diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h
index 65b9ba874368f392bd409054cf6729d3847fdb60..fcc578f35e322a97c5398825d2bfa9dc2c357728 100644
--- a/bolt/include/bolt/Profile/BoltAddressTranslation.h
+++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h
@@ -141,15 +141,13 @@ private:
                          uint64_t FuncOutputAddress) const;
 
   /// Write the serialized address translation table for a function.
-  template <bool Cold>
-  void writeMaps(std::map<uint64_t, MapTy> &Maps, uint64_t &PrevAddress,
-                 raw_ostream &OS);
+  template <bool Cold> void writeMaps(uint64_t &PrevAddress, raw_ostream &OS);
 
   /// Read the serialized address translation table for a function.
   /// Return a parse error if failed.
   template <bool Cold>
-  void parseMaps(std::vector<uint64_t> &HotFuncs, uint64_t &PrevAddress,
-                 DataExtractor &DE, uint64_t &Offset, Error &Err);
+  void parseMaps(uint64_t &PrevAddress, DataExtractor &DE, uint64_t &Offset,
+                 Error &Err);
 
   /// Returns the bitmask with set bits corresponding to indices of BRANCHENTRY
   /// entries in function address translation map.
@@ -161,6 +159,9 @@ private:
 
   std::map<uint64_t, MapTy> Maps;
 
+  /// Ordered vector with addresses of hot functions.
+  std::vector<uint64_t> HotFuncs;
+
   /// Map a function to its basic blocks count
   std::unordered_map<uint64_t, size_t> NumBasicBlocksMap;
 
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 36c42fced93d08bc9e9944d1a3eba6b5ae40d64f..ef3fba37817daa541f28528f62555dd2df8676f9 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -3684,9 +3684,8 @@ BinaryFunction::BasicBlockListType BinaryFunction::dfs() const {
     BinaryBasicBlock *BB = Stack.top();
     Stack.pop();
 
-    if (Visited.find(BB) != Visited.end())
+    if (!Visited.insert(BB).second)
       continue;
-    Visited.insert(BB);
     DFS.push_back(BB);
 
     for (BinaryBasicBlock *SuccBB : BB->landing_pads()) {
@@ -3879,11 +3878,8 @@ void BinaryFunction::disambiguateJumpTables(
       JumpTable *JT = getJumpTable(Inst);
       if (!JT)
         continue;
-      auto Iter = JumpTables.find(JT);
-      if (Iter == JumpTables.end()) {
-        JumpTables.insert(JT);
+      if (JumpTables.insert(JT).second)
         continue;
-      }
       // This instruction is an indirect jump using a jump table, but it is
       // using the same jump table of another jump. Try all our tricks to
       // extract the jump table symbol and make it point to a new, duplicated JT
diff --git a/bolt/lib/Core/DIEBuilder.cpp b/bolt/lib/Core/DIEBuilder.cpp
index 69cfd58a1df04bcaff7dd0893a81dec73f4a05cc..8d781fc5e04ff72b36de20d20c598eab4bee5233 100644
--- a/bolt/lib/Core/DIEBuilder.cpp
+++ b/bolt/lib/Core/DIEBuilder.cpp
@@ -57,11 +57,9 @@ getDWOName(llvm::DWARFUnit &CU,
          "DW_AT_dwo_name/DW_AT_GNU_dwo_name does not exist.");
   if (DwarfOutputPath) {
     DWOName = std::string(sys::path::filename(DWOName));
-    auto Iter = NameToIndexMap.find(DWOName);
-    if (Iter == NameToIndexMap.end())
-      Iter = NameToIndexMap.insert({DWOName, 0}).first;
-    DWOName.append(std::to_string(Iter->second));
-    ++Iter->second;
+    uint32_t &Index = NameToIndexMap[DWOName];
+    DWOName.append(std::to_string(Index));
+    ++Index;
   }
   DWOName.append(".dwo");
   return DWOName;
diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp
index ebb3925749b4d2775818937e0e7be83ebcd32fb9..76766b05b9176071fcbaf77931fd39d8a42d3942 100644
--- a/bolt/lib/Passes/Instrumentation.cpp
+++ b/bolt/lib/Passes/Instrumentation.cpp
@@ -109,9 +109,8 @@ static bool hasAArch64ExclusiveMemop(
     BinaryBasicBlock *BB = BBQueue.front().first;
     bool IsLoad = BBQueue.front().second;
     BBQueue.pop();
-    if (Visited.find(BB) != Visited.end())
+    if (!Visited.insert(BB).second)
       continue;
-    Visited.insert(BB);
 
     for (const MCInst &Inst : *BB) {
       // Two loads one after another - skip whole function
@@ -126,8 +125,7 @@ static bool hasAArch64ExclusiveMemop(
       if (BC.MIB->isAArch64ExclusiveLoad(Inst))
         IsLoad = true;
 
-      if (IsLoad && BBToSkip.find(BB) == BBToSkip.end()) {
-        BBToSkip.insert(BB);
+      if (IsLoad && BBToSkip.insert(BB).second) {
         if (opts::Verbosity >= 2) {
           outs() << "BOLT-INSTRUMENTER: skip BB " << BB->getName()
                  << " due to exclusive instruction in function "
diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index ec7e303c0f52e83dcae6bbfa80407afb649e3435..4d005f942699e10631a7e44b39ccc368e7c5f82f 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -143,8 +143,8 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
 
   // Output addresses are delta-encoded
   uint64_t PrevAddress = 0;
-  writeMaps</*Cold=*/false>(Maps, PrevAddress, OS);
-  writeMaps</*Cold=*/true>(Maps, PrevAddress, OS);
+  writeMaps</*Cold=*/false>(PrevAddress, OS);
+  writeMaps</*Cold=*/true>(PrevAddress, OS);
 
   BC.outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n";
   BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.getNumFunctions()
@@ -182,8 +182,7 @@ size_t BoltAddressTranslation::getNumEqualOffsets(const MapTy &Map,
 }
 
 template <bool Cold>
-void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
-                                       uint64_t &PrevAddress, raw_ostream &OS) {
+void BoltAddressTranslation::writeMaps(uint64_t &PrevAddress, raw_ostream &OS) {
   const uint32_t NumFuncs =
       llvm::count_if(llvm::make_first_range(Maps), [&](const uint64_t Address) {
         return Cold == ColdPartSource.count(Address);
@@ -213,9 +212,9 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
             : 0;
     uint32_t Skew = 0;
     if (Cold) {
-      auto HotEntryIt = Maps.find(ColdPartSource[Address]);
-      assert(HotEntryIt != Maps.end());
-      size_t HotIndex = std::distance(Maps.begin(), HotEntryIt);
+      auto HotEntryIt = llvm::lower_bound(HotFuncs, ColdPartSource[Address]);
+      assert(HotEntryIt != HotFuncs.end());
+      size_t HotIndex = std::distance(HotFuncs.begin(), HotEntryIt);
       encodeULEB128(HotIndex - PrevIndex, OS);
       PrevIndex = HotIndex;
       // Skew of all input offsets for cold fragments is simply the first input
@@ -223,6 +222,7 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
       Skew = Map.begin()->second >> 1;
       encodeULEB128(Skew, OS);
     } else {
+      HotFuncs.push_back(Address);
       // Function hash
       size_t BFHash = getBFHash(HotInputAddress);
       LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", BFHash));
@@ -311,17 +311,15 @@ std::error_code BoltAddressTranslation::parse(raw_ostream &OS, StringRef Buf) {
     return make_error_code(llvm::errc::io_error);
 
   Error Err(Error::success());
-  std::vector<uint64_t> HotFuncs;
   uint64_t PrevAddress = 0;
-  parseMaps</*Cold=*/false>(HotFuncs, PrevAddress, DE, Offset, Err);
-  parseMaps</*Cold=*/true>(HotFuncs, PrevAddress, DE, Offset, Err);
+  parseMaps</*Cold=*/false>(PrevAddress, DE, Offset, Err);
+  parseMaps</*Cold=*/true>(PrevAddress, DE, Offset, Err);
   OS << "BOLT-INFO: Parsed " << Maps.size() << " BAT entries\n";
   return errorToErrorCode(std::move(Err));
 }
 
 template <bool Cold>
-void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
-                                       uint64_t &PrevAddress, DataExtractor &DE,
+void BoltAddressTranslation::parseMaps(uint64_t &PrevAddress, DataExtractor &DE,
                                        uint64_t &Offset, Error &Err) {
   const uint32_t NumFunctions = DE.getULEB128(&Offset, &Err);
   LLVM_DEBUG(dbgs() << "Parsing " << NumFunctions << (Cold ? " cold" : "")
diff --git a/bolt/tools/driver/llvm-bolt.cpp b/bolt/tools/driver/llvm-bolt.cpp
index 9b03524e9f18e8771f38a7926b3b9c50945d1153..a8d1ac6480893001917c84e7275c97dea2fa3cab 100644
--- a/bolt/tools/driver/llvm-bolt.cpp
+++ b/bolt/tools/driver/llvm-bolt.cpp
@@ -202,9 +202,9 @@ int main(int argc, char **argv) {
 
   ToolName = argv[0];
 
-  if (llvm::sys::path::filename(ToolName) == "perf2bolt")
+  if (llvm::sys::path::filename(ToolName).starts_with("perf2bolt"))
     perf2boltMode(argc, argv);
-  else if (llvm::sys::path::filename(ToolName) == "llvm-boltdiff")
+  else if (llvm::sys::path::filename(ToolName).starts_with("llvm-boltdiff"))
     boltDiffMode(argc, argv);
   else
     boltMode(argc, argv);
diff --git a/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp
index 6175fcdfd229c3a1409217786a95e44dc539034a..8eaf54fe0088a4d27d2153427ab24b6110eb1a90 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp
@@ -165,7 +165,7 @@ void CrtpConstructorAccessibilityCheck::check(
 
     WithFriendHintIfNeeded(
         diag(Ctor->getLocation(),
-             "%0 contructor allows the CRTP to be %select{inherited "
+             "%0 constructor allows the CRTP to be %select{inherited "
              "from|constructed}1 as a regular template class; consider making "
              "it private%select{| and declaring the derived class as friend}2")
             << Access << IsPublic << NeedsFriend
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
index a30e63f9b0fd6a75231b71819b60430ff29a878d..628d30ce7f73fe382ed9585a7ee3078213eda120 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
@@ -70,7 +70,9 @@ SizeofExpressionCheck::SizeofExpressionCheck(StringRef Name,
           Options.get("WarnOnSizeOfCompareToConstant", true)),
       WarnOnSizeOfPointerToAggregate(
           Options.get("WarnOnSizeOfPointerToAggregate", true)),
-      WarnOnSizeOfPointer(Options.get("WarnOnSizeOfPointer", false)) {}
+      WarnOnSizeOfPointer(Options.get("WarnOnSizeOfPointer", false)),
+      WarnOnOffsetDividedBySizeOf(
+          Options.get("WarnOnOffsetDividedBySizeOf", true)) {}
 
 void SizeofExpressionCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "WarnOnSizeOfConstant", WarnOnSizeOfConstant);
@@ -82,6 +84,8 @@ void SizeofExpressionCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "WarnOnSizeOfPointerToAggregate",
                 WarnOnSizeOfPointerToAggregate);
   Options.store(Opts, "WarnOnSizeOfPointer", WarnOnSizeOfPointer);
+  Options.store(Opts, "WarnOnOffsetDividedBySizeOf",
+                WarnOnOffsetDividedBySizeOf);
 }
 
 void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
@@ -307,7 +311,8 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
                  offsetOfExpr()))
           .bind("sizeof-in-ptr-arithmetic-scale-expr");
   const auto PtrArithmeticIntegerScaleExpr = binaryOperator(
-      hasAnyOperatorName("*", "/"),
+      WarnOnOffsetDividedBySizeOf ? binaryOperator(hasAnyOperatorName("*", "/"))
+                                  : binaryOperator(hasOperatorName("*")),
       // sizeof(...) * sizeof(...) and sizeof(...) / sizeof(...) is handled
       // by this check on another path.
       hasOperands(expr(hasType(isInteger()), unless(SizeofLikeScaleExpr)),
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
index 66d7c34cc9e94010d73e2a273069aa2b603e84ae..fbd62cb80fb2d02a715300f07d0010af0c2f9416 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
@@ -31,6 +31,7 @@ private:
   const bool WarnOnSizeOfCompareToConstant;
   const bool WarnOnSizeOfPointerToAggregate;
   const bool WarnOnSizeOfPointer;
+  const bool WarnOnOffsetDividedBySizeOf;
 };
 
 } // namespace clang::tidy::bugprone
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
index 5b1b1cd152fffba36e4f328eab925c07c676b3bd..6df565c9a9d691db5c72067b4bcb54cf5e00bb23 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
@@ -137,11 +137,11 @@ static bool mayShadow(const NamedDecl *ND0,
 const ConfusableIdentifierCheck::ContextInfo *
 ConfusableIdentifierCheck::getContextInfo(const DeclContext *DC) {
   const DeclContext *PrimaryContext = DC->getPrimaryContext();
-  auto It = ContextInfos.find(PrimaryContext);
-  if (It != ContextInfos.end())
+  auto [It, Inserted] = ContextInfos.try_emplace(PrimaryContext);
+  if (!Inserted)
     return &It->second;
 
-  ContextInfo &Info = ContextInfos[PrimaryContext];
+  ContextInfo &Info = It->second;
   Info.PrimaryContext = PrimaryContext;
   Info.NonTransparentContext = PrimaryContext;
 
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
index 89ee45faecd7f33e8f80189ee59fad02849695aa..5eb3267adb0799dd8fb02addc6b580ba242b7b1f 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.cpp
@@ -8,6 +8,7 @@
 
 #include "UseStartsEndsWithCheck.h"
 
+#include "../utils/ASTUtils.h"
 #include "../utils/OptionsUtils.h"
 #include "clang/Lex/Lexer.h"
 
@@ -16,6 +17,63 @@
 using namespace clang::ast_matchers;
 
 namespace clang::tidy::modernize {
+struct NotLengthExprForStringNode {
+  NotLengthExprForStringNode(std::string ID, DynTypedNode Node,
+                             ASTContext *Context)
+      : ID(std::move(ID)), Node(std::move(Node)), Context(Context) {}
+  bool operator()(const internal::BoundNodesMap &Nodes) const {
+    // Match a string literal and an integer size or strlen() call.
+    if (const auto *StringLiteralNode = Nodes.getNodeAs<StringLiteral>(ID)) {
+      if (const auto *IntegerLiteralSizeNode = Node.get<IntegerLiteral>()) {
+        return StringLiteralNode->getLength() !=
+               IntegerLiteralSizeNode->getValue().getZExtValue();
+      }
+
+      if (const auto *StrlenNode = Node.get<CallExpr>()) {
+        if (StrlenNode->getDirectCallee()->getName() != "strlen" ||
+            StrlenNode->getNumArgs() != 1) {
+          return true;
+        }
+
+        if (const auto *StrlenArgNode = dyn_cast<StringLiteral>(
+                StrlenNode->getArg(0)->IgnoreParenImpCasts())) {
+          return StrlenArgNode->getLength() != StringLiteralNode->getLength();
+        }
+      }
+    }
+
+    // Match a string variable and a call to length() or size().
+    if (const auto *ExprNode = Nodes.getNodeAs<Expr>(ID)) {
+      if (const auto *MemberCallNode = Node.get<CXXMemberCallExpr>()) {
+        const CXXMethodDecl *MethodDeclNode = MemberCallNode->getMethodDecl();
+        const StringRef Name = MethodDeclNode->getName();
+        if (!MethodDeclNode->isConst() || MethodDeclNode->getNumParams() != 0 ||
+            (Name != "size" && Name != "length")) {
+          return true;
+        }
+
+        if (const auto *OnNode =
+                dyn_cast<Expr>(MemberCallNode->getImplicitObjectArgument())) {
+          return !utils::areStatementsIdentical(OnNode->IgnoreParenImpCasts(),
+                                                ExprNode->IgnoreParenImpCasts(),
+                                                *Context);
+        }
+      }
+    }
+
+    return true;
+  }
+
+private:
+  std::string ID;
+  DynTypedNode Node;
+  ASTContext *Context;
+};
+
+AST_MATCHER_P(Expr, lengthExprForStringNode, std::string, ID) {
+  return Builder->removeBindings(NotLengthExprForStringNode(
+      ID, DynTypedNode::create(Node), &(Finder->getASTContext())));
+}
 
 UseStartsEndsWithCheck::UseStartsEndsWithCheck(StringRef Name,
                                                ClangTidyContext *Context)
@@ -23,6 +81,7 @@ UseStartsEndsWithCheck::UseStartsEndsWithCheck(StringRef Name,
 
 void UseStartsEndsWithCheck::registerMatchers(MatchFinder *Finder) {
   const auto ZeroLiteral = integerLiteral(equals(0));
+
   const auto HasStartsWithMethodWithName = [](const std::string &Name) {
     return hasMethod(
         cxxMethodDecl(hasName(Name), isConst(), parameterCountIs(1))
@@ -32,119 +91,104 @@ void UseStartsEndsWithCheck::registerMatchers(MatchFinder *Finder) {
       anyOf(HasStartsWithMethodWithName("starts_with"),
             HasStartsWithMethodWithName("startsWith"),
             HasStartsWithMethodWithName("startswith"));
-  const auto ClassWithStartsWithFunction = cxxRecordDecl(anyOf(
-      HasStartsWithMethod, hasAnyBase(hasType(hasCanonicalType(hasDeclaration(
-                               cxxRecordDecl(HasStartsWithMethod)))))));
+  const auto OnClassWithStartsWithFunction =
+      on(hasType(hasCanonicalType(hasDeclaration(cxxRecordDecl(
+          anyOf(HasStartsWithMethod,
+                hasAnyBase(hasType(hasCanonicalType(
+                    hasDeclaration(cxxRecordDecl(HasStartsWithMethod)))))))))));
 
+  const auto HasEndsWithMethodWithName = [](const std::string &Name) {
+    return hasMethod(
+        cxxMethodDecl(hasName(Name), isConst(), parameterCountIs(1))
+            .bind("ends_with_fun"));
+  };
+  const auto HasEndsWithMethod = anyOf(HasEndsWithMethodWithName("ends_with"),
+                                       HasEndsWithMethodWithName("endsWith"),
+                                       HasEndsWithMethodWithName("endswith"));
+  const auto OnClassWithEndsWithFunction =
+      on(expr(hasType(hasCanonicalType(hasDeclaration(cxxRecordDecl(
+                  anyOf(HasEndsWithMethod,
+                        hasAnyBase(hasType(hasCanonicalType(hasDeclaration(
+                            cxxRecordDecl(HasEndsWithMethod)))))))))))
+             .bind("haystack"));
+
+  // Case 1: X.find(Y) [!=]= 0 -> starts_with.
   const auto FindExpr = cxxMemberCallExpr(
-      // A method call with no second argument or the second argument is zero...
       anyOf(argumentCountIs(1), hasArgument(1, ZeroLiteral)),
-      // ... named find...
       callee(cxxMethodDecl(hasName("find")).bind("find_fun")),
-      // ... on a class with a starts_with function.
-      on(hasType(
-          hasCanonicalType(hasDeclaration(ClassWithStartsWithFunction)))),
-      // Bind search expression.
-      hasArgument(0, expr().bind("search_expr")));
+      OnClassWithStartsWithFunction, hasArgument(0, expr().bind("needle")));
 
+  // Case 2: X.rfind(Y, 0) [!=]= 0 -> starts_with.
   const auto RFindExpr = cxxMemberCallExpr(
-      // A method call with a second argument of zero...
       hasArgument(1, ZeroLiteral),
-      // ... named rfind...
       callee(cxxMethodDecl(hasName("rfind")).bind("find_fun")),
-      // ... on a class with a starts_with function.
-      on(hasType(
-          hasCanonicalType(hasDeclaration(ClassWithStartsWithFunction)))),
-      // Bind search expression.
-      hasArgument(0, expr().bind("search_expr")));
-
-  // Match a string literal and an integer or strlen() call matching the length.
-  const auto HasStringLiteralAndLengthArgs = [](const auto StringArgIndex,
-                                                const auto LengthArgIndex) {
-    return allOf(
-        hasArgument(StringArgIndex, stringLiteral().bind("string_literal_arg")),
-        hasArgument(LengthArgIndex,
-                    anyOf(integerLiteral().bind("integer_literal_size_arg"),
-                          callExpr(callee(functionDecl(parameterCountIs(1),
-                                                       hasName("strlen"))),
-                                   hasArgument(0, stringLiteral().bind(
-                                                      "strlen_arg"))))));
-  };
-
-  // Match a string variable and a call to length() or size().
-  const auto HasStringVariableAndSizeCallArgs = [](const auto StringArgIndex,
-                                                   const auto LengthArgIndex) {
-    return allOf(
-        hasArgument(StringArgIndex, declRefExpr(hasDeclaration(
-                                        decl().bind("string_var_decl")))),
-        hasArgument(LengthArgIndex,
-                    cxxMemberCallExpr(
-                        callee(cxxMethodDecl(isConst(), parameterCountIs(0),
-                                             hasAnyName("size", "length"))),
-                        on(declRefExpr(
-                            to(decl(equalsBoundNode("string_var_decl"))))))));
-  };
-
-  // Match either one of the two cases above.
-  const auto HasStringAndLengthArgs =
-      [HasStringLiteralAndLengthArgs, HasStringVariableAndSizeCallArgs](
-          const auto StringArgIndex, const auto LengthArgIndex) {
-        return anyOf(
-            HasStringLiteralAndLengthArgs(StringArgIndex, LengthArgIndex),
-            HasStringVariableAndSizeCallArgs(StringArgIndex, LengthArgIndex));
-      };
+      OnClassWithStartsWithFunction, hasArgument(0, expr().bind("needle")));
 
+  // Case 3: X.compare(0, LEN(Y), Y) [!=]= 0 -> starts_with.
   const auto CompareExpr = cxxMemberCallExpr(
-      // A method call with three arguments...
-      argumentCountIs(3),
-      // ... where the first argument is zero...
-      hasArgument(0, ZeroLiteral),
-      // ... named compare...
+      argumentCountIs(3), hasArgument(0, ZeroLiteral),
       callee(cxxMethodDecl(hasName("compare")).bind("find_fun")),
-      // ... on a class with a starts_with function...
-      on(hasType(
-          hasCanonicalType(hasDeclaration(ClassWithStartsWithFunction)))),
-      // ... where the third argument is some string and the second a length.
-      HasStringAndLengthArgs(2, 1),
-      // Bind search expression.
-      hasArgument(2, expr().bind("search_expr")));
+      OnClassWithStartsWithFunction, hasArgument(2, expr().bind("needle")),
+      hasArgument(1, lengthExprForStringNode("needle")));
 
+  // Case 4: X.compare(LEN(X) - LEN(Y), LEN(Y), Y) [!=]= 0 -> ends_with.
+  const auto CompareEndsWithExpr = cxxMemberCallExpr(
+      argumentCountIs(3),
+      callee(cxxMethodDecl(hasName("compare")).bind("find_fun")),
+      OnClassWithEndsWithFunction, hasArgument(2, expr().bind("needle")),
+      hasArgument(1, lengthExprForStringNode("needle")),
+      hasArgument(0,
+                  binaryOperator(hasOperatorName("-"),
+                                 hasLHS(lengthExprForStringNode("haystack")),
+                                 hasRHS(lengthExprForStringNode("needle")))));
+
+  // All cases comparing to 0.
   Finder->addMatcher(
-      // Match [=!]= with a zero on one side and (r?)find|compare on the other.
       binaryOperator(
           hasAnyOperatorName("==", "!="),
-          hasOperands(cxxMemberCallExpr(anyOf(FindExpr, RFindExpr, CompareExpr))
+          hasOperands(cxxMemberCallExpr(anyOf(FindExpr, RFindExpr, CompareExpr,
+                                              CompareEndsWithExpr))
                           .bind("find_expr"),
                       ZeroLiteral))
           .bind("expr"),
       this);
+
+  // Case 5: X.rfind(Y) [!=]= LEN(X) - LEN(Y) -> ends_with.
+  Finder->addMatcher(
+      binaryOperator(
+          hasAnyOperatorName("==", "!="),
+          hasOperands(
+              cxxMemberCallExpr(
+                  anyOf(
+                      argumentCountIs(1),
+                      allOf(argumentCountIs(2),
+                            hasArgument(
+                                1,
+                                anyOf(declRefExpr(to(varDecl(hasName("npos")))),
+                                      memberExpr(member(hasName("npos"))))))),
+                  callee(cxxMethodDecl(hasName("rfind")).bind("find_fun")),
+                  OnClassWithEndsWithFunction,
+                  hasArgument(0, expr().bind("needle")))
+                  .bind("find_expr"),
+              binaryOperator(hasOperatorName("-"),
+                             hasLHS(lengthExprForStringNode("haystack")),
+                             hasRHS(lengthExprForStringNode("needle")))))
+          .bind("expr"),
+      this);
 }
 
 void UseStartsEndsWithCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *ComparisonExpr = Result.Nodes.getNodeAs<BinaryOperator>("expr");
   const auto *FindExpr = Result.Nodes.getNodeAs<CXXMemberCallExpr>("find_expr");
   const auto *FindFun = Result.Nodes.getNodeAs<CXXMethodDecl>("find_fun");
-  const auto *SearchExpr = Result.Nodes.getNodeAs<Expr>("search_expr");
+  const auto *SearchExpr = Result.Nodes.getNodeAs<Expr>("needle");
   const auto *StartsWithFunction =
       Result.Nodes.getNodeAs<CXXMethodDecl>("starts_with_fun");
-
-  const auto *StringLiteralArg =
-      Result.Nodes.getNodeAs<StringLiteral>("string_literal_arg");
-  const auto *IntegerLiteralSizeArg =
-      Result.Nodes.getNodeAs<IntegerLiteral>("integer_literal_size_arg");
-  const auto *StrlenArg = Result.Nodes.getNodeAs<StringLiteral>("strlen_arg");
-
-  // Filter out compare cases where the length does not match string literal.
-  if (StringLiteralArg && IntegerLiteralSizeArg &&
-      StringLiteralArg->getLength() !=
-          IntegerLiteralSizeArg->getValue().getZExtValue()) {
-    return;
-  }
-
-  if (StringLiteralArg && StrlenArg &&
-      StringLiteralArg->getLength() != StrlenArg->getLength()) {
-    return;
-  }
+  const auto *EndsWithFunction =
+      Result.Nodes.getNodeAs<CXXMethodDecl>("ends_with_fun");
+  assert(bool(StartsWithFunction) != bool(EndsWithFunction));
+  const CXXMethodDecl *ReplacementFunction =
+      StartsWithFunction ? StartsWithFunction : EndsWithFunction;
 
   if (ComparisonExpr->getBeginLoc().isMacroID()) {
     return;
@@ -154,9 +198,9 @@ void UseStartsEndsWithCheck::check(const MatchFinder::MatchResult &Result) {
 
   auto Diagnostic =
       diag(FindExpr->getExprLoc(), "use %0 instead of %1() %select{==|!=}2 0")
-      << StartsWithFunction->getName() << FindFun->getName() << Neg;
+      << ReplacementFunction->getName() << FindFun->getName() << Neg;
 
-  // Remove possible arguments after search expression and ' [!=]= 0' suffix.
+  // Remove possible arguments after search expression and ' [!=]= .+' suffix.
   Diagnostic << FixItHint::CreateReplacement(
       CharSourceRange::getTokenRange(
           Lexer::getLocForEndOfToken(SearchExpr->getEndLoc(), 0,
@@ -164,16 +208,16 @@ void UseStartsEndsWithCheck::check(const MatchFinder::MatchResult &Result) {
           ComparisonExpr->getEndLoc()),
       ")");
 
-  // Remove possible '0 [!=]= ' prefix.
+  // Remove possible '.+ [!=]= ' prefix.
   Diagnostic << FixItHint::CreateRemoval(CharSourceRange::getCharRange(
       ComparisonExpr->getBeginLoc(), FindExpr->getBeginLoc()));
 
-  // Replace method name by 'starts_with'.
+  // Replace method name by '(starts|ends)_with'.
   // Remove possible arguments before search expression.
   Diagnostic << FixItHint::CreateReplacement(
       CharSourceRange::getCharRange(FindExpr->getExprLoc(),
                                     SearchExpr->getBeginLoc()),
-      (StartsWithFunction->getName() + "(").str());
+      (ReplacementFunction->getName() + "(").str());
 
   // Add possible negation '!'.
   if (Neg) {
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.h b/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.h
index 840191f321493fe49e440b4bb7bbfc5407c64b3b..17c2999bda84cd0e936e2989828871a5552f666f 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/UseStartsEndsWithCheck.h
@@ -14,7 +14,7 @@
 namespace clang::tidy::modernize {
 
 /// Checks for common roundabout ways to express ``starts_with`` and
-/// ``ends_with`` and suggests replacing with ``starts_with`` when the method is
+/// ``ends_with`` and suggests replacing with the simpler method when it is
 /// available. Notably, this will work with ``std::string`` and
 /// ``std::string_view``.
 ///
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index 828f13805a69802689cbe91d7af8a1c6a893ab5a..3f63eec2c51a8c34c5d151b0dc554630643ca2b4 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -1135,6 +1135,9 @@ StyleKind IdentifierNamingCheck::findStyleKind(
   if (isa<TypeAliasDecl>(D) && NamingStyles[SK_TypeAlias])
     return SK_TypeAlias;
 
+  if (isa<NamespaceAliasDecl>(D) && NamingStyles[SK_Namespace])
+    return SK_Namespace;
+
   if (const auto *Decl = dyn_cast<NamespaceDecl>(D)) {
     if (Decl->isAnonymousNamespace())
       return SK_Invalid;
diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index 333fc10f17d7b43731ab7043ebf481d569c87ca3..f3eee1c6335f9889a446db7024977a85d2c43182 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -144,8 +144,13 @@ getQualification(ASTContext &Context, const DeclContext *DestContext,
   // since we stored inner-most parent first.
   std::string Result;
   llvm::raw_string_ostream OS(Result);
-  for (const auto *Parent : llvm::reverse(Parents))
+  for (const auto *Parent : llvm::reverse(Parents)) {
+    if (Parent != *Parents.rbegin() && Parent->isDependent() &&
+        Parent->getAsRecordDecl() &&
+        Parent->getAsRecordDecl()->getDescribedClassTemplate())
+      OS << "template ";
     Parent->print(OS, Context.getPrintingPolicy());
+  }
   return OS.str();
 }
 
diff --git a/clang-tools-extra/clangd/Headers.cpp b/clang-tools-extra/clangd/Headers.cpp
index 75f8668e7bef06077f52b3b5404f3dba3ac15cd5..b537417bd10568cadde99d9b4d664adaaca701b3 100644
--- a/clang-tools-extra/clangd/Headers.cpp
+++ b/clang-tools-extra/clangd/Headers.cpp
@@ -75,8 +75,8 @@ public:
               IDs.push_back(HID);
           }
       }
-      Out->MainFileIncludesBySpelling.try_emplace(Inc.Written)
-          .first->second.push_back(Out->MainFileIncludes.size() - 1);
+      Out->MainFileIncludesBySpelling[Inc.Written].push_back(
+          Out->MainFileIncludes.size() - 1);
     }
 
     // Record include graph (not just for main-file includes)
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index f94cadeffaa298ab4bd5d6a583d604913dc34116..b34aba603b53065eb6a37ecba5e49065311222ee 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -2282,8 +2282,7 @@ incomingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index) {
       elog("incomingCalls failed to convert location: {0}", Loc.takeError());
       return;
     }
-    auto It = CallsIn.try_emplace(R.Container, std::vector<Range>{}).first;
-    It->second.push_back(Loc->range);
+    CallsIn[R.Container].push_back(Loc->range);
 
     ContainerLookup.IDs.insert(R.Container);
   });
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp b/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
index f43f2417df8fcef8cc686f65e09eb18a89523c6a..591a8b245260eab6c43a1bb7e385d3733e65623d 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
@@ -128,7 +128,28 @@ getFunctionSourceAfterReplacements(const FunctionDecl *FD,
       SM.getBufferData(SM.getMainFileID()), Replacements);
   if (!QualifiedFunc)
     return QualifiedFunc.takeError();
-  return QualifiedFunc->substr(FuncBegin, FuncEnd - FuncBegin + 1);
+
+  std::string TemplatePrefix;
+  if (auto *MD = llvm::dyn_cast<CXXMethodDecl>(FD)) {
+    for (const CXXRecordDecl *Parent = MD->getParent(); Parent;
+         Parent =
+             llvm::dyn_cast_or_null<const CXXRecordDecl>(Parent->getParent())) {
+      if (const TemplateParameterList *Params =
+              Parent->getDescribedTemplateParams()) {
+        std::string S;
+        llvm::raw_string_ostream Stream(S);
+        Params->print(Stream, FD->getASTContext());
+        if (!S.empty())
+          *S.rbegin() = '\n'; // Replace space with newline
+        TemplatePrefix.insert(0, S);
+      }
+    }
+  }
+
+  auto Source = QualifiedFunc->substr(FuncBegin, FuncEnd - FuncBegin + 1);
+  if (!TemplatePrefix.empty())
+    Source.insert(0, TemplatePrefix);
+  return Source;
 }
 
 // Returns replacements to delete tokens with kind `Kind` in the range
@@ -212,9 +233,13 @@ getFunctionSourceCode(const FunctionDecl *FD, const DeclContext *TargetContext,
           }
         }
         const NamedDecl *ND = Ref.Targets.front();
-        const std::string Qualifier =
+        std::string Qualifier =
             getQualification(AST, TargetContext,
                              SM.getLocForStartOfFile(SM.getMainFileID()), ND);
+        if (ND->getDeclContext()->isDependentContext() &&
+            llvm::isa<TypeDecl>(ND)) {
+          Qualifier.insert(0, "typename ");
+        }
         if (auto Err = DeclarationCleanups.add(
                 tooling::Replacement(SM, Ref.NameLoc, 0, Qualifier)))
           Errors = llvm::joinErrors(std::move(Errors), std::move(Err));
@@ -407,10 +432,23 @@ public:
       return !SameFile;
     }
 
-    // Bail out in templated classes, as it is hard to spell the class name,
-    // i.e if the template parameter is unnamed.
-    if (MD->getParent()->isTemplated())
-      return false;
+    for (const CXXRecordDecl *Parent = MD->getParent(); Parent;
+         Parent =
+             llvm::dyn_cast_or_null<const CXXRecordDecl>(Parent->getParent())) {
+      if (const TemplateParameterList *Params =
+              Parent->getDescribedTemplateParams()) {
+
+        // Class template member functions must be defined in the
+        // same file.
+        SameFile = true;
+
+        // Bail out if the template parameter is unnamed.
+        for (NamedDecl *P : *Params) {
+          if (!P->getIdentifier())
+            return false;
+        }
+      }
+    }
 
     // The refactoring is meaningless for unnamed classes and namespaces,
     // unless we're outlining in the same file
diff --git a/clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp
index 906ff33db8734414fc08b8e82170ec2afa2b380d..6a9e90c3bfa70fc55a61e4bafcc83ab20017c59b 100644
--- a/clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp
+++ b/clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp
@@ -105,8 +105,8 @@ TEST_F(DefineOutlineTest, TriggersOnFunctionDecl) {
       F^oo(const Foo&) = delete;
     };)cpp");
 
-  // Not available within templated classes, as it is hard to spell class name
-  // out-of-line in such cases.
+  // Not available within templated classes with unnamed parameters, as it is
+  // hard to spell class name out-of-line in such cases.
   EXPECT_UNAVAILABLE(R"cpp(
     template <typename> struct Foo { void fo^o(){} };
     )cpp");
@@ -154,7 +154,6 @@ TEST_F(DefineOutlineTest, FailsWithoutSource) {
 }
 
 TEST_F(DefineOutlineTest, ApplyTest) {
-  llvm::StringMap<std::string> EditedFiles;
   ExtraFiles["Test.cpp"] = "";
   FileName = "Test.hpp";
 
@@ -229,17 +228,18 @@ TEST_F(DefineOutlineTest, ApplyTest) {
       // Ctor initializer with attribute.
       {
           R"cpp(
-              class Foo {
-                F^oo(int z) __attribute__((weak)) : bar(2){}
+              template <typename T> class Foo {
+                F^oo(T z) __attribute__((weak)) : bar(2){}
                 int bar;
               };)cpp",
           R"cpp(
-              class Foo {
-                Foo(int z) __attribute__((weak)) ;
+              template <typename T> class Foo {
+                Foo(T z) __attribute__((weak)) ;
                 int bar;
-              };)cpp",
-          "Foo::Foo(int z) __attribute__((weak)) : bar(2){}\n",
-      },
+              };template <typename T>
+Foo<T>::Foo(T z) __attribute__((weak)) : bar(2){}
+)cpp",
+          ""},
       // Virt specifiers.
       {
           R"cpp(
@@ -369,7 +369,31 @@ TEST_F(DefineOutlineTest, ApplyTest) {
             };)cpp",
           " void A::foo(int) {}\n",
       },
-      // Destrctors
+      // Complex class template
+      {
+          R"cpp(
+            template <typename T, typename ...U> struct O1 {
+              template <class V, int A> struct O2 {
+                enum E { E1, E2 };
+                struct I {
+                  E f^oo(T, U..., V, E) { return E1; }
+                };
+              };
+            };)cpp",
+          R"cpp(
+            template <typename T, typename ...U> struct O1 {
+              template <class V, int A> struct O2 {
+                enum E { E1, E2 };
+                struct I {
+                  E foo(T, U..., V, E) ;
+                };
+              };
+            };template <typename T, typename ...U>
+template <class V, int A>
+typename O1<T, U...>::template O2<V, A>::E O1<T, U...>::template O2<V, A>::I::foo(T, U..., V, E) { return E1; }
+)cpp",
+          ""},
+      // Destructors
       {
           "class A { ~A^(){} };",
           "class A { ~A(); };",
@@ -378,9 +402,14 @@ TEST_F(DefineOutlineTest, ApplyTest) {
   };
   for (const auto &Case : Cases) {
     SCOPED_TRACE(Case.Test);
+    llvm::StringMap<std::string> EditedFiles;
     EXPECT_EQ(apply(Case.Test, &EditedFiles), Case.ExpectedHeader);
-    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                                 testPath("Test.cpp"), Case.ExpectedSource)));
+    if (Case.ExpectedSource.empty()) {
+      EXPECT_TRUE(EditedFiles.empty());
+    } else {
+      EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                                   testPath("Test.cpp"), Case.ExpectedSource)));
+    }
   }
 }
 
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 3f7bcde1eb3014fc7389bca420f29fc1fc21683e..95be0a89cd6c93a5ba88d76e3e9131f378b45638 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -88,6 +88,8 @@ Objective-C
 Miscellaneous
 ^^^^^^^^^^^^^
 
+- The DefineOutline tweak now handles member functions of class templates.
+
 Improvements to clang-doc
 -------------------------
 
@@ -156,7 +158,7 @@ Changes in existing checks
 - Improved :doc:`bugprone-sizeof-expression
   <clang-tidy/checks/bugprone/sizeof-expression>` check to find suspicious
   usages of ``sizeof()``, ``alignof()``, and ``offsetof()`` when adding or
-  subtracting from a pointer.
+  subtracting from a pointer directly or when used to scale a numeric value.
 
 - Improved :doc:`bugprone-unchecked-optional-access
   <clang-tidy/checks/bugprone/unchecked-optional-access>` to support
@@ -202,6 +204,10 @@ Changes in existing checks
   <clang-tidy/checks/modernize/use-nullptr>` check to also recognize
   ``NULL``/``__null`` (but not ``0``) when used with a templated type.
 
+- Improved :doc:`modernize-use-starts-ends-with
+  <clang-tidy/checks/modernize/use-starts-ends-with>` check to handle two cases
+  that can be replaced with ``ends_with``
+
 - Improved :doc:`modernize-use-std-format
   <clang-tidy/checks/modernize/use-std-format>` check to support replacing
   member function calls too and to only expand macros starting with ``PRI``
@@ -238,6 +244,10 @@ Changes in existing checks
   <clang-tidy/checks/readability/redundant-smartptr-get>` check to
   remove `->`, when redundant `get()` is removed.
 
+- Improved :doc:`readability-identifier-naming
+  <clang-tidy/checks/readability/identifier-naming>` check to
+  validate ``namespace`` aliases.
+
 Removed checks
 ^^^^^^^^^^^^^^
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
index 89c88d2740dba29ea1257d56ed5559a54c4d495e..29edb26ed7aa20e368dca56349390a45270d1a7f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
@@ -221,6 +221,17 @@ and the result is typically unintended, often out of bounds.
 ``Ptr + sizeof(T)`` will offset the pointer by ``sizeof(T)`` elements,
 effectively exponentiating the scaling factor to the power of 2.
 
+Similarly, multiplying or dividing a numeric value with the ``sizeof`` of an
+element or the whole buffer is suspicious, because the dimensional connection
+between the numeric value and the actual ``sizeof`` result can not always be
+deduced.
+While scaling an integer up (multiplying) with ``sizeof`` is likely **always**
+an issue, a scaling down (division) is not always inherently dangerous, in case
+the developer is aware that the division happens between an appropriate number
+of _bytes_ and a ``sizeof`` value.
+Turning :option:`WarnOnOffsetDividedBySizeOf` off will restrict the
+warnings to the multiplication case.
+
 This case also checks suspicious ``alignof`` and ``offsetof`` usages in
 pointer arithmetic, as both return the "size" in bytes and not elements,
 potentially resulting in doubly-scaled offsets.
@@ -299,3 +310,9 @@ Options
    ``sizeof`` is an expression that produces a pointer (except for a few
    idiomatic expressions that are probably intentional and correct).
    This detects occurrences of CWE 467. Default is `false`.
+
+.. option:: WarnOnOffsetDividedBySizeOf
+
+   When `true`, the check will warn on pointer arithmetic where the
+   element count is obtained from a division with ``sizeof(...)``,
+   e.g., ``Ptr + Bytes / sizeof(*T)``. Default is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-starts-ends-with.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-starts-ends-with.rst
index 34237ede30a30b51f4c35703d7a2151171c356c4..721e927e29849f47520569d12891f6da5351627a 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-starts-ends-with.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-starts-ends-with.rst
@@ -4,8 +4,8 @@ modernize-use-starts-ends-with
 ==============================
 
 Checks for common roundabout ways to express ``starts_with`` and ``ends_with``
-and suggests replacing with ``starts_with`` when the method is available.
-Notably, this will work with ``std::string`` and ``std::string_view``.
+and suggests replacing with the simpler method when it is available. Notably, 
+this will work with ``std::string`` and ``std::string_view``.
 
 .. code-block:: c++
 
@@ -13,6 +13,12 @@ Notably, this will work with ``std::string`` and ``std::string_view``.
   if (s.find("prefix") == 0) { /* do something */ }
   if (s.rfind("prefix", 0) == 0) { /* do something */ }
   if (s.compare(0, strlen("prefix"), "prefix") == 0) { /* do something */ }
+  if (s.compare(s.size() - strlen("suffix"), strlen("suffix"), "suffix") == 0) {
+    /* do something */
+  }
+  if (s.rfind("suffix") == (s.length() - 6)) {
+    /* do something */
+  }
 
 becomes
 
@@ -22,3 +28,5 @@ becomes
   if (s.starts_with("prefix")) { /* do something */ }
   if (s.starts_with("prefix")) { /* do something */ }
   if (s.starts_with("prefix")) { /* do something */ }
+  if (s.ends_with("suffix")) { /* do something */ }
+  if (s.ends_with("suffix")) { /* do something */ }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
index 0c160bc182b6ebdf4ddce7f460ad38c8fd8a99a4..a0e8ebbb267cd0b8a8a0bea3d10c949245b5eb7a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
@@ -64,6 +64,10 @@ struct basic_string {
   constexpr bool starts_with(C ch) const noexcept;
   constexpr bool starts_with(const C* s) const;
 
+  constexpr bool ends_with(std::basic_string_view<C, T> sv) const noexcept;
+  constexpr bool ends_with(C ch) const noexcept;
+  constexpr bool ends_with(const C* s) const;
+
   _Type& operator[](size_type);
   const _Type& operator[](size_type) const;
 
@@ -108,6 +112,10 @@ struct basic_string_view {
   constexpr bool starts_with(C ch) const noexcept;
   constexpr bool starts_with(const C* s) const;
 
+  constexpr bool ends_with(basic_string_view sv) const noexcept;
+  constexpr bool ends_with(C ch) const noexcept;
+  constexpr bool ends_with(const C* s) const;
+
   constexpr int compare(basic_string_view sv) const noexcept;
 
   static constexpr size_t npos = -1;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp
index cb41923df157cf83a0c6d2539fab5cb57c787988..f33b8457cc8af528a5e73900083d8176bf981510 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp
@@ -26,7 +26,7 @@ template <typename T>
 class CRTP {
 public:
     CRTP() = default;
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public constructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}public:
     // CHECK-FIXES: friend T;
 };
@@ -39,7 +39,7 @@ template <typename T>
 class CRTP {
 public:
     CRTP(int) {}
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public constructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(int) {}{{[[:space:]]*}}public:
     // CHECK-FIXES: friend T;
 };
@@ -52,10 +52,10 @@ template <typename T>
 class CRTP {
 public:
     CRTP(int) {}
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public constructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(int) {}{{[[:space:]]*}}public:
     CRTP(float) {}
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public constructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(float) {}{{[[:space:]]*}}public:
     
     // CHECK-FIXES: friend T;
@@ -70,13 +70,13 @@ template <typename T>
 class CRTP {
 protected:
     CRTP(int) {}
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected contructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected constructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(int) {}{{[[:space:]]*}}protected:
     CRTP() = default;
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected contructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected constructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}protected:
     CRTP(float) {}
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected contructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected constructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(float) {}{{[[:space:]]*}}protected:
     
     // CHECK-FIXES: friend T;
@@ -101,7 +101,7 @@ namespace struct_default_ctor {
 template <typename T>
 struct CRTP {
     CRTP() = default;
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public constructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility]
     // CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}public:
     // CHECK-FIXES: friend T;
 };
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics-no-division.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics-no-division.c
new file mode 100644
index 0000000000000000000000000000000000000000..b0cfb8d1cfa456407e4cb0c7900e3ec6734246f0
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics-no-division.c
@@ -0,0 +1,14 @@
+// RUN: %check_clang_tidy %s bugprone-sizeof-expression %t -- \
+// RUN:   -config='{CheckOptions: { \
+// RUN:     bugprone-sizeof-expression.WarnOnOffsetDividedBySizeOf: false \
+// RUN:   }}'
+
+typedef __SIZE_TYPE__ size_t;
+
+void situational14(int *Buffer, size_t BufferSize) {
+  int *P = &Buffer[0];
+  while (P < Buffer + BufferSize / sizeof(*Buffer)) {
+    // NO-WARNING: This test opted out of "P +- N */ sizeof(...)" warnings.
+    ++P;
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics.c
index 360ce00a6889d71275dd6c6f0db3c651747972d9..712141c5f266aad562868230f31d4833fb00aeb2 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics.c
@@ -352,21 +352,30 @@ void good13(void) {
   int Buffer[BufferSize];
 
   int *P = &Buffer[0];
-  while (P < (Buffer + sizeof(Buffer) / sizeof(int))) {
+  while (P < Buffer + sizeof(Buffer) / sizeof(int)) {
     // NO-WARNING: Calculating the element count of the buffer here, which is
     // safe with this idiom (as long as the types don't change).
     ++P;
   }
 
-  while (P < (Buffer + sizeof(Buffer) / sizeof(Buffer[0]))) {
+  while (P < Buffer + sizeof(Buffer) / sizeof(Buffer[0])) {
     // NO-WARNING: Calculating the element count of the buffer here, which is
     // safe with this idiom.
     ++P;
   }
 
-  while (P < (Buffer + sizeof(Buffer) / sizeof(*P))) {
+  while (P < Buffer + sizeof(Buffer) / sizeof(*P)) {
     // NO-WARNING: Calculating the element count of the buffer here, which is
     // safe with this idiom.
     ++P;
   }
 }
+
+void situational14(int *Buffer, size_t BufferSize) {
+  int *P = &Buffer[0];
+  while (P < Buffer + BufferSize / sizeof(*Buffer)) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:21: note: '+' in pointer arithmetic internally scales with 'sizeof(int)' == {{[0-9]+}}
+    ++P;
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp
index c5b2c86befd1fec8c42c9f980367d07de51b4c69..798af260a3b66cd21e940204dde418fd29eb6136 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-starts-ends-with.cpp
@@ -208,6 +208,62 @@ void test(std::string s, std::string_view sv, sub_string ss, sub_sub_string sss,
   // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use starts_with
   // CHECK-FIXES: !s.starts_with(sv);
 
+  s.compare(s.size() - 6, 6, "suffix") == 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with("suffix");
+
+  s.compare(s.size() - 6, strlen("abcdef"), "suffix") == 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with("suffix");
+
+  std::string suffix = "suffix";
+  s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with(suffix);
+
+  s.rfind("suffix") == s.size() - 6;
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with("suffix");
+
+  s.rfind("suffix") == s.size() - strlen("suffix");
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with("suffix");
+
+  s.rfind(suffix) == s.size() - suffix.size();
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with(suffix);
+
+  s.rfind(suffix, std::string::npos) == s.size() - suffix.size();
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with(suffix);
+
+  s.rfind(suffix) == (s.size() - suffix.size());
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with(suffix);
+
+  s.rfind(suffix, s.npos) == (s.size() - suffix.size());
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with(suffix);
+
+  s.rfind(suffix, s.npos) == (((s.size()) - (suffix.size())));
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with(suffix);
+
+  s.rfind(suffix) != s.size() - suffix.size();
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: !s.ends_with(suffix);
+
+  (s.size() - suffix.size()) == s.rfind(suffix);
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: s.ends_with(suffix);
+
+  struct S {
+    std::string s;
+  } t;
+  t.s.rfind(suffix) == (t.s.size() - suffix.size());
+  // CHECK-MESSAGES: :[[@LINE-1]]:{{[0-9]+}}: warning: use ends_with
+  // CHECK-FIXES: t.s.ends_with(suffix);
+
   // Expressions that don't trigger the check are here.
   #define EQ(x, y) ((x) == (y))
   EQ(s.find("a"), 0);
@@ -219,4 +275,5 @@ void test(std::string s, std::string_view sv, sub_string ss, sub_sub_string sss,
   STARTS_WITH_COMPARE(s, s) == 0;
 
   s.compare(0, 1, "ab") == 0;
+  s.rfind(suffix, 1) == s.size() - suffix.size();
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming.cpp
index 99149fe86aceecf662ead7f674103230bb263630..be5ba54513c672e064ed259e599cb739ea32718f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming.cpp
@@ -101,6 +101,10 @@ inline namespace InlineNamespace {
 // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for inline namespace 'InlineNamespace'
 // CHECK-FIXES: {{^}}inline namespace inline_namespace {{{$}}
 
+namespace FOO_ALIAS = FOO_NS;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for namespace 'FOO_ALIAS' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}namespace foo_alias = FOO_NS;{{$}}
+
 SYSTEM_NS::structure g_s1;
 // NO warnings or fixes expected as SYSTEM_NS and structure are declared in a header file
 
diff --git a/clang/bindings/python/tests/CMakeLists.txt b/clang/bindings/python/tests/CMakeLists.txt
index 2543cf739463d93ceb169e97b26354e2eb0c61b2..a0ddabc21bb411b392b7c846308fbf9b3eaaf411 100644
--- a/clang/bindings/python/tests/CMakeLists.txt
+++ b/clang/bindings/python/tests/CMakeLists.txt
@@ -47,6 +47,19 @@ if(${LLVM_NATIVE_ARCH} MATCHES "^(AArch64|Hexagon|Sparc|SystemZ)$")
   set(RUN_PYTHON_TESTS FALSE)
 endif()
 
+# Tests will fail if cross-compiling for a different target, as tests will try
+# to use the host Python3_EXECUTABLE and make FFI calls to functions in target
+# libraries.
+if(CMAKE_CROSSCOMPILING)
+  # FIXME: Consider a solution that allows better control over these tests in
+  # a crosscompiling scenario. e.g. registering them with lit to allow them to
+  # be explicitly skipped via appropriate LIT_ARGS, or adding a mechanism to
+  # allow specifying a python interpreter compiled for the target that could
+  # be executed using qemu-user.
+  message(WARNING "check-clang-python not added to check-all as these tests fail in a cross-build setup")
+  set(RUN_PYTHON_TESTS FALSE)
+endif()
+
 if(RUN_PYTHON_TESTS)
     set_property(GLOBAL APPEND PROPERTY
                  LLVM_ALL_ADDITIONAL_TEST_TARGETS check-clang-python)
diff --git a/clang/cmake/modules/AddClang.cmake b/clang/cmake/modules/AddClang.cmake
index 5327b5d2f08928820ecae68a70fca1e921b5ca23..091aec98e93ca337ec5b000b7eb3ea8427a261a9 100644
--- a/clang/cmake/modules/AddClang.cmake
+++ b/clang/cmake/modules/AddClang.cmake
@@ -108,6 +108,17 @@ macro(add_clang_library name)
   endif()
   llvm_add_library(${name} ${LIBTYPE} ${ARG_UNPARSED_ARGUMENTS} ${srcs})
 
+  if(MSVC AND NOT CLANG_LINK_CLANG_DYLIB)
+    # Make sure all consumers also turn off visibility macros so there not trying to dllimport symbols.
+    target_compile_definitions(${name} PUBLIC CLANG_BUILD_STATIC)
+    if(TARGET "obj.${name}")
+      target_compile_definitions("obj.${name}" PUBLIC CLANG_BUILD_STATIC)
+    endif()
+  elseif(NOT ARG_SHARED AND NOT ARG_STATIC)
+    # Clang component libraries linked in to clang-cpp are declared without SHARED or STATIC
+    target_compile_definitions("obj.${name}" PUBLIC CLANG_EXPORTS)
+  endif()
+
   set(libs ${name})
   if(ARG_SHARED AND ARG_STATIC)
     list(APPEND libs ${name}_static)
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index a9bfb4c4a0fcb2ca6ef42be2eca61e3560d01f87..8add0a53e5be136c83b14af36ffc0033135e942d 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -5406,22 +5406,46 @@ the configuration (without a prefix: ``Auto``).
 
 .. _ReflowComments:
 
-**ReflowComments** (``Boolean``) :versionbadge:`clang-format 3.8` :ref:`¶ <ReflowComments>`
-  If ``true``, clang-format will attempt to re-flow comments. That is it
-  will touch a comment and *reflow* long comments into new lines, trying to
-  obey the ``ColumnLimit``.
+**ReflowComments** (``ReflowCommentsStyle``) :versionbadge:`clang-format 3.8` :ref:`¶ <ReflowComments>`
+  Comment reformatting style.
 
-  .. code-block:: c++
+  Possible values:
+
+  * ``RCS_Never`` (in configuration: ``Never``)
+    Leave comments untouched.
+
+    .. code-block:: c++
+
+       // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+       /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information */
+       /* third veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+            * and a misaligned second line */
+
+  * ``RCS_IndentOnly`` (in configuration: ``IndentOnly``)
+    Only apply indentation rules, moving comments left or right, without
+    changing formatting inside the comments.
+
+    .. code-block:: c++
+
+       // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+       /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information */
+       /* third veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+        * and a misaligned second line */
+
+  * ``RCS_Always`` (in configuration: ``Always``)
+    Apply indentation rules and reflow long comments into new lines, trying
+    to obey the ``ColumnLimit``.
+
+    .. code-block:: c++
+
+       // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
+       // information
+       /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
+        * information */
+       /* third veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
+        * information and a misaligned second line */
 
-     false:
-     // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
-     /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information */
 
-     true:
-     // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
-     // information
-     /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
-      * information */
 
 .. _RemoveBracesLLVM:
 
diff --git a/clang/docs/ClangPlugins.rst b/clang/docs/ClangPlugins.rst
index 001e66e434efb15e44b045e6296b43808b9f3960..92e41fb5877fe8b449020b3cd23a55ac7794ec25 100644
--- a/clang/docs/ClangPlugins.rst
+++ b/clang/docs/ClangPlugins.rst
@@ -92,11 +92,6 @@ The members of ``ParsedAttrInfo`` that a plugin attribute must define are:
    attribute, each of which consists of an attribute syntax and how the
    attribute name is spelled for that syntax. If the syntax allows a scope then
    the spelling must be "scope::attr" if a scope is present or "::attr" if not.
- * ``handleDeclAttribute``, which is the function that applies the attribute to
-   a declaration. It is responsible for checking that the attribute's arguments
-   are valid, and typically applies the attribute by adding an ``Attr`` to the
-   ``Decl``. It returns either ``AttributeApplied``, to indicate that the
-   attribute was successfully applied, or ``AttributeNotApplied`` if it wasn't.
 
 The members of ``ParsedAttrInfo`` that may need to be defined, depending on the
 attribute, are:
@@ -105,6 +100,18 @@ attribute, are:
    arguments to the attribute.
  * ``diagAppertainsToDecl``, which checks if the attribute has been used on the
    right kind of declaration and issues a diagnostic if not.
+ * ``handleDeclAttribute``, which is the function that applies the attribute to
+   a declaration. It is responsible for checking that the attribute's arguments
+   are valid, and typically applies the attribute by adding an ``Attr`` to the
+   ``Decl``. It returns either ``AttributeApplied``, to indicate that the
+   attribute was successfully applied, or ``AttributeNotApplied`` if it wasn't.
+ * ``diagAppertainsToStmt``, which checks if the attribute has been used on the
+   right kind of statement and issues a diagnostic if not.
+ * ``handleStmtAttribute``, which is the function that applies the attribute to
+   a statement. It is responsible for checking that the attribute's arguments
+   are valid, and typically applies the attribute by adding an ``Attr`` to the
+   ``Stmt``. It returns either ``AttributeApplied``, to indicate that the
+   attribute was successfully applied, or ``AttributeNotApplied`` if it wasn't.
  * ``diagLangOpts``, which checks if the attribute is permitted for the current
    language mode and issues a diagnostic if not.
  * ``existsInTarget``, which checks if the attribute is permitted for the given
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index fe41742a25c64236fa984d2e9320084115b74b15..10232ff41da15a704fc1cbb775256f7fc5e1ec48 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -5883,3 +5883,26 @@ specify the starting offset to begin embedding from. The resources is treated
 as being empty if the specified offset is larger than the number of bytes in
 the resource. The offset will be applied *before* any ``limit`` parameters are
 applied.
+
+Union and aggregate initialization in C
+=======================================
+
+In C23 (N2900), when an object is initialized from initializer ``= {}``, all
+elements of arrays, all members of structs, and the first members of unions are
+empty-initialized recursively. In addition, all padding bits are initialized to
+zero.
+
+Clang guarantees the following behaviors:
+
+* ``1:`` Clang supports initializer ``= {}`` mentioned above in all C
+  standards.
+
+* ``2:`` When unions are initialized from initializer ``= {}``, bytes outside
+  of the first members of unions are also initialized to zero.
+
+* ``3:`` When unions, structures and arrays are initialized from initializer
+  ``= { initializer-list }``, all members not explicitly initialized in
+  the initializer list are empty-initialized recursively. In addition, all
+  padding bits are initialized to zero.
+
+Currently, the above extension only applies to C source code, not C++.
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index df165b9125250581250a4d9c614230cfe883df38..817e3abef8d5669cd29601cef58c31c0febaf73f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -99,6 +99,20 @@ C++ Specific Potentially Breaking Changes
     // Was error, now evaluates to false.
     constexpr bool b = f() == g();
 
+- The warning ``-Wdeprecated-literal-operator`` is now on by default, as this is
+  something that WG21 has shown interest in removing from the language. The
+  result is that anyone who is compiling with ``-Werror`` should see this
+  diagnostic.  To fix this diagnostic, simply removing the space character from
+  between the ``operator""`` and the user defined literal name will make the
+  source no longer deprecated. This is consistent with `CWG2521 <https://cplusplus.github.io/CWG/issues/2521.html>_`.
+
+  .. code-block:: c++
+
+    // Now diagnoses by default.
+    unsigned operator"" _udl_name(unsigned long long);
+    // Fixed version:
+    unsigned operator""_udl_name(unsigned long long);
+
 ABI Changes in This Version
 ---------------------------
 
@@ -177,10 +191,6 @@ C++23 Feature Support
 C++20 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
-C++17 Feature Support
-^^^^^^^^^^^^^^^^^^^^^
-- The implementation of the relaxed template template argument matching rules is
-  more complete and reliable, and should provide more accurate diagnostics.
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -207,8 +217,7 @@ Resolutions to C++ Defect Reports
   (`CWG2351: void{} <https://cplusplus.github.io/CWG/issues/2351.html>`_).
 
 - Clang now has improved resolution to CWG2398, allowing class templates to have
-  default arguments deduced when partial ordering, and better backwards compatibility
-  in overload resolution.
+  default arguments deduced when partial ordering.
 
 - Clang now allows comparing unequal object pointers that have been cast to ``void *``
   in constant expressions. These comparisons always worked in non-constant expressions.
@@ -220,6 +229,10 @@ Resolutions to C++ Defect Reports
 - Clang now allows trailing requires clause on explicit deduction guides.
   (`CWG2707: Deduction guides cannot have a trailing requires-clause <https://cplusplus.github.io/CWG/issues/2707.html>`_).
 
+- Clang now diagnoses a space in the first production of a ``literal-operator-id``
+  by default.
+  (`CWG2521: User-defined literals and reserved identifiers <https://cplusplus.github.io/CWG/issues/2521.html>`_).
+
 C Language Changes
 ------------------
 
@@ -237,6 +250,8 @@ Non-comprehensive list of changes in this release
 - The floating point comparison builtins (``__builtin_isgreater``,
   ``__builtin_isgreaterequal``, ``__builtin_isless``, etc.) and
   ``__builtin_signbit`` can now be used in constant expressions.
+- Plugins can now define custom attributes that apply to statements
+  as well as declarations.
 
 New Compiler Flags
 ------------------
@@ -339,10 +354,6 @@ Improvements to Clang's diagnostics
 
 - Clang now diagnoses when the result of a [[nodiscard]] function is discarded after being cast in C. Fixes #GH104391.
 
-- Clang now properly explains the reason a template template argument failed to
-  match a template template parameter, in terms of the C++17 relaxed matching rules
-  instead of the old ones.
-
 - Don't emit duplicated dangling diagnostics. (#GH93386).
 
 - Improved diagnostic when trying to befriend a concept. (#GH45182).
@@ -387,6 +398,10 @@ Improvements to Clang's diagnostics
 - The warning for an unsupported type for a named register variable is now phrased ``unsupported type for named register variable``,
   instead of ``bad type for named register variable``. This makes it clear that the type is not supported at all, rather than being
   suboptimal in some way the error fails to mention (#GH111550).
+  
+- Clang now emits a ``-Wdepredcated-literal-operator`` diagnostic, even if the
+  name was a reserved name, which we improperly allowed to suppress the
+  diagnostic.
 
 Improvements to Clang's time-trace
 ----------------------------------
@@ -452,8 +467,6 @@ Bug Fixes to C++ Support
 - Correctly check constraints of explicit instantiations of member functions. (#GH46029)
 - When performing partial ordering of function templates, clang now checks that
   the deduction was consistent. Fixes (#GH18291).
-- Fixes to several issues in partial ordering of template template parameters, which
-  were documented in the test suite.
 - Fixed an assertion failure about a constraint of a friend function template references to a value with greater
   template depth than the friend function template. (#GH98258)
 - Clang now rebuilds the template parameters of out-of-line declarations and specializations in the context
@@ -491,9 +504,19 @@ Bug Fixes to C++ Support
   in certain friend declarations. (#GH93099)
 - Clang now instantiates the correct lambda call operator when a lambda's class type is
   merged across modules. (#GH110401)
+- Clang now uses the correct set of template argument lists when comparing the constraints of
+  out-of-line definitions and member templates explicitly specialized for a given implicit instantiation of
+  a class template. (#GH102320)
 - Fix a crash when parsing a pseudo destructor involving an invalid type. (#GH111460)
 - Fixed an assertion failure when invoking recovery call expressions with explicit attributes
-  and undeclared templates. (#GH107047, #GH49093)
+  and undeclared templates. (#GH107047), (#GH49093)
+- Clang no longer crashes when a lambda contains an invalid block declaration that contains an unexpanded
+  parameter pack. (#GH109148)
+- Fixed overload handling for object parameters with top-level cv-qualifiers in explicit member functions (#GH100394)
+- Fixed a bug in lambda captures where ``constexpr`` class-type objects were not properly considered ODR-used in
+  certain situations. (#GH47400), (#GH90896)
+- Fix erroneous templated array size calculation leading to crashes in generated code. (#GH41441)
+- During the lookup for a base class name, non-type names are ignored. (#GH16855)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -601,6 +624,8 @@ CUDA/HIP Language Changes
 
 CUDA Support
 ^^^^^^^^^^^^
+- Clang now supports CUDA SDK up to 12.6
+- Added support for sm_100
 
 AIX Support
 ^^^^^^^^^^^
@@ -614,6 +639,8 @@ WebAssembly Support
 AVR Support
 ^^^^^^^^^^^
 
+- Reject C/C++ compilation for avr1 devices which have no SRAM.
+
 DWARF Support in Clang
 ----------------------
 
@@ -636,12 +663,17 @@ AST Matchers
 
 - Fixed a crash when traverse lambda expr with invalid captures. (#GH106444)
 
+- Ensure ``hasName`` matches template specializations across inline namespaces,
+  making `matchesNodeFullSlow` and `matchesNodeFullFast` consistent.
+
 clang-format
 ------------
 
 - Adds ``BreakBinaryOperations`` option.
 - Adds ``TemplateNames`` option.
 - Adds ``AlignFunctionDeclarations`` option to ``AlignConsecutiveDeclarations``.
+- Adds ``IndentOnly`` suboption to ``ReflowComments`` to fix the indentation of multi-line comments
+  without touching their contents, renames ``false`` to ``Never``, and ``true`` to ``Always``.
 
 libclang
 --------
diff --git a/clang/examples/Attribute/Attribute.cpp b/clang/examples/Attribute/Attribute.cpp
index 9d6cf9ae36c6a6e31fd8544e7cf200a17f23dcea..3b90724ad222052ce74c5ec0152f05fd26c7bb14 100644
--- a/clang/examples/Attribute/Attribute.cpp
+++ b/clang/examples/Attribute/Attribute.cpp
@@ -94,6 +94,54 @@ struct ExampleAttrInfo : public ParsedAttrInfo {
     }
     return AttributeApplied;
   }
+
+  bool diagAppertainsToStmt(Sema &S, const ParsedAttr &Attr,
+                            const Stmt *St) const override {
+    // This attribute appertains to for loop statements only.
+    if (!isa<ForStmt>(St)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type_str)
+          << Attr << Attr.isRegularKeywordAttribute() << "for loop statements";
+      return false;
+    }
+    return true;
+  }
+
+  AttrHandling handleStmtAttribute(Sema &S, Stmt *St, const ParsedAttr &Attr,
+                                   class Attr *&Result) const override {
+    // We make some rules here:
+    // 1. Only accept at most 3 arguments here.
+    // 2. The first argument must be a string literal if it exists.
+    if (Attr.getNumArgs() > 3) {
+      unsigned ID = S.getDiagnostics().getCustomDiagID(
+          DiagnosticsEngine::Error,
+          "'example' attribute only accepts at most three arguments");
+      S.Diag(Attr.getLoc(), ID);
+      return AttributeNotApplied;
+    }
+    // If there are arguments, the first argument should be a string literal.
+    if (Attr.getNumArgs() > 0) {
+      auto *Arg0 = Attr.getArgAsExpr(0);
+      StringLiteral *Literal =
+          dyn_cast<StringLiteral>(Arg0->IgnoreParenCasts());
+      if (!Literal) {
+        unsigned ID = S.getDiagnostics().getCustomDiagID(
+            DiagnosticsEngine::Error, "first argument to the 'example' "
+                                      "attribute must be a string literal");
+        S.Diag(Attr.getLoc(), ID);
+        return AttributeNotApplied;
+      }
+      SmallVector<Expr *, 16> ArgsBuf;
+      for (unsigned i = 0; i < Attr.getNumArgs(); i++) {
+        ArgsBuf.push_back(Attr.getArgAsExpr(i));
+      }
+      Result = AnnotateAttr::Create(S.Context, "example", ArgsBuf.data(),
+                                    ArgsBuf.size(), Attr.getRange());
+    } else {
+      Result = AnnotateAttr::Create(S.Context, "example", nullptr, 0,
+                                    Attr.getRange());
+    }
+    return AttributeApplied;
+  }
 };
 
 } // namespace
diff --git a/clang/include/clang/AST/ComputeDependence.h b/clang/include/clang/AST/ComputeDependence.h
index 1a8507cfbf9872335d8147c8b3fcc55dcf3f6e12..e96275e5f2e07370ad7e9a680bb4c40d113e6e8e 100644
--- a/clang/include/clang/AST/ComputeDependence.h
+++ b/clang/include/clang/AST/ComputeDependence.h
@@ -133,7 +133,8 @@ ExprDependence computeDependence(ArrayInitLoopExpr *E);
 ExprDependence computeDependence(ImplicitValueInitExpr *E);
 ExprDependence computeDependence(InitListExpr *E);
 ExprDependence computeDependence(ExtVectorElementExpr *E);
-ExprDependence computeDependence(BlockExpr *E);
+ExprDependence computeDependence(BlockExpr *E,
+                                 bool ContainsUnexpandedParameterPack);
 ExprDependence computeDependence(AsTypeExpr *E);
 ExprDependence computeDependence(DeclRefExpr *E, const ASTContext &Ctx);
 ExprDependence computeDependence(RecoveryExpr *E);
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 687715a22e9fd312e25b0ae2d07ccdfd780105da..141f58c4600af0be3c555d8c92add43d86597f00 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -781,15 +781,11 @@ protected:
                              EntryType *Entry, void *InsertPos);
 
   struct CommonBase {
-    CommonBase() : InstantiatedFromMember(nullptr, false) {}
+    CommonBase() {}
 
     /// The template from which this was most
     /// directly instantiated (or null).
-    ///
-    /// The boolean value indicates whether this template
-    /// was explicitly specialized.
-    llvm::PointerIntPair<RedeclarableTemplateDecl*, 1, bool>
-      InstantiatedFromMember;
+    RedeclarableTemplateDecl *InstantiatedFromMember = nullptr;
 
     /// If non-null, points to an array of specializations (including
     /// partial specializations) known only by their external declaration IDs.
@@ -809,14 +805,19 @@ protected:
   };
 
   /// Pointer to the common data shared by all declarations of this
-  /// template.
-  mutable CommonBase *Common = nullptr;
+  /// template, and a flag indicating if the template is a member
+  /// specialization.
+  mutable llvm::PointerIntPair<CommonBase *, 1, bool> Common;
+
+  CommonBase *getCommonPtrInternal() const { return Common.getPointer(); }
 
   /// Retrieves the "common" pointer shared by all (re-)declarations of
   /// the same template. Calling this routine may implicitly allocate memory
   /// for the common pointer.
   CommonBase *getCommonPtr() const;
 
+  void setCommonPtr(CommonBase *C) const { Common.setPointer(C); }
+
   virtual CommonBase *newCommon(ASTContext &C) const = 0;
 
   // Construct a template decl with name, parameters, and templated element.
@@ -857,15 +858,22 @@ public:
   /// template<> template<typename T>
   /// struct X<int>::Inner { /* ... */ };
   /// \endcode
-  bool isMemberSpecialization() const {
-    return getCommonPtr()->InstantiatedFromMember.getInt();
+  bool isMemberSpecialization() const { return Common.getInt(); }
+
+  /// Determines whether any redeclaration of this template was
+  /// a specialization of a member template.
+  bool hasMemberSpecialization() const {
+    for (const auto *D : redecls()) {
+      if (D->isMemberSpecialization())
+        return true;
+    }
+    return false;
   }
 
   /// Note that this member template is a specialization.
   void setMemberSpecialization() {
-    assert(getCommonPtr()->InstantiatedFromMember.getPointer() &&
-           "Only member templates can be member template specializations");
-    getCommonPtr()->InstantiatedFromMember.setInt(true);
+    assert(!isMemberSpecialization() && "already a member specialization");
+    Common.setInt(true);
   }
 
   /// Retrieve the member template from which this template was
@@ -905,12 +913,12 @@ public:
   /// void X<T>::f(T, U);
   /// \endcode
   RedeclarableTemplateDecl *getInstantiatedFromMemberTemplate() const {
-    return getCommonPtr()->InstantiatedFromMember.getPointer();
+    return getCommonPtr()->InstantiatedFromMember;
   }
 
   void setInstantiatedFromMemberTemplate(RedeclarableTemplateDecl *TD) {
-    assert(!getCommonPtr()->InstantiatedFromMember.getPointer());
-    getCommonPtr()->InstantiatedFromMember.setPointer(TD);
+    assert(!getCommonPtr()->InstantiatedFromMember);
+    getCommonPtr()->InstantiatedFromMember = TD;
   }
 
   /// Retrieve the "injected" template arguments that correspond to the
@@ -1989,6 +1997,8 @@ public:
   /// template arguments have been deduced.
   void setInstantiationOf(ClassTemplatePartialSpecializationDecl *PartialSpec,
                           const TemplateArgumentList *TemplateArgs) {
+    assert(!isa<ClassTemplatePartialSpecializationDecl>(this) &&
+           "A partial specialization cannot be instantiated from a template");
     assert(!SpecializedTemplate.is<SpecializedPartialSpecialization*>() &&
            "Already set to a class template partial specialization!");
     auto *PS = new (getASTContext()) SpecializedPartialSpecialization();
@@ -2000,6 +2010,8 @@ public:
   /// Note that this class template specialization is an instantiation
   /// of the given class template.
   void setInstantiationOf(ClassTemplateDecl *TemplDecl) {
+    assert(!isa<ClassTemplatePartialSpecializationDecl>(this) &&
+           "A partial specialization cannot be instantiated from a template");
     assert(!SpecializedTemplate.is<SpecializedPartialSpecialization*>() &&
            "Previously set to a class template partial specialization!");
     SpecializedTemplate = TemplDecl;
@@ -2187,19 +2199,23 @@ public:
   /// struct X<int>::Inner<T*> { /* ... */ };
   /// \endcode
   bool isMemberSpecialization() const {
-    const auto *First =
-        cast<ClassTemplatePartialSpecializationDecl>(getFirstDecl());
-    return First->InstantiatedFromMember.getInt();
+    return InstantiatedFromMember.getInt();
   }
 
-  /// Note that this member template is a specialization.
-  void setMemberSpecialization() {
-    auto *First = cast<ClassTemplatePartialSpecializationDecl>(getFirstDecl());
-    assert(First->InstantiatedFromMember.getPointer() &&
-           "Only member templates can be member template specializations");
-    return First->InstantiatedFromMember.setInt(true);
+  /// Determines whether any redeclaration of this this class template partial
+  /// specialization was a specialization of a member partial specialization.
+  bool hasMemberSpecialization() const {
+    for (const auto *D : redecls()) {
+      if (cast<ClassTemplatePartialSpecializationDecl>(D)
+              ->isMemberSpecialization())
+        return true;
+    }
+    return false;
   }
 
+  /// Note that this member template is a specialization.
+  void setMemberSpecialization() { return InstantiatedFromMember.setInt(true); }
+
   /// Retrieves the injected specialization type for this partial
   /// specialization.  This is not the same as the type-decl-type for
   /// this partial specialization, which is an InjectedClassNameType.
@@ -2268,10 +2284,6 @@ protected:
     return static_cast<Common *>(RedeclarableTemplateDecl::getCommonPtr());
   }
 
-  void setCommonPtr(Common *C) {
-    RedeclarableTemplateDecl::Common = C;
-  }
-
 public:
 
   friend class ASTDeclReader;
@@ -2754,6 +2766,8 @@ public:
   /// template arguments have been deduced.
   void setInstantiationOf(VarTemplatePartialSpecializationDecl *PartialSpec,
                           const TemplateArgumentList *TemplateArgs) {
+    assert(!isa<VarTemplatePartialSpecializationDecl>(this) &&
+           "A partial specialization cannot be instantiated from a template");
     assert(!SpecializedTemplate.is<SpecializedPartialSpecialization *>() &&
            "Already set to a variable template partial specialization!");
     auto *PS = new (getASTContext()) SpecializedPartialSpecialization();
@@ -2765,6 +2779,8 @@ public:
   /// Note that this variable template specialization is an instantiation
   /// of the given variable template.
   void setInstantiationOf(VarTemplateDecl *TemplDecl) {
+    assert(!isa<VarTemplatePartialSpecializationDecl>(this) &&
+           "A partial specialization cannot be instantiated from a template");
     assert(!SpecializedTemplate.is<SpecializedPartialSpecialization *>() &&
            "Previously set to a variable template partial specialization!");
     SpecializedTemplate = TemplDecl;
@@ -2949,19 +2965,24 @@ public:
   /// U* X<int>::Inner<T*> = (T*)(0) + 1;
   /// \endcode
   bool isMemberSpecialization() const {
-    const auto *First =
-        cast<VarTemplatePartialSpecializationDecl>(getFirstDecl());
-    return First->InstantiatedFromMember.getInt();
+    return InstantiatedFromMember.getInt();
   }
 
-  /// Note that this member template is a specialization.
-  void setMemberSpecialization() {
-    auto *First = cast<VarTemplatePartialSpecializationDecl>(getFirstDecl());
-    assert(First->InstantiatedFromMember.getPointer() &&
-           "Only member templates can be member template specializations");
-    return First->InstantiatedFromMember.setInt(true);
+  /// Determines whether any redeclaration of this this variable template
+  /// partial specialization was a specialization of a member partial
+  /// specialization.
+  bool hasMemberSpecialization() const {
+    for (const auto *D : redecls()) {
+      if (cast<VarTemplatePartialSpecializationDecl>(D)
+              ->isMemberSpecialization())
+        return true;
+    }
+    return false;
   }
 
+  /// Note that this member template is a specialization.
+  void setMemberSpecialization() { return InstantiatedFromMember.setInt(true); }
+
   SourceRange getSourceRange() const override LLVM_READONLY;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 607bf313c4d95e52384278be45277bb87d0ac2c4..cbe62411d11bffe543b582e397a0494e23c80bd8 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -6413,9 +6413,9 @@ class BlockExpr : public Expr {
 protected:
   BlockDecl *TheBlock;
 public:
-  BlockExpr(BlockDecl *BD, QualType ty)
+  BlockExpr(BlockDecl *BD, QualType ty, bool ContainsUnexpandedParameterPack)
       : Expr(BlockExprClass, ty, VK_PRValue, OK_Ordinary), TheBlock(BD) {
-    setDependence(computeDependence(this));
+    setDependence(computeDependence(this, ContainsUnexpandedParameterPack));
   }
 
   /// Build an empty block expression.
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index e4f2e07222a338ff12f9279d8152ab0f69d096bd..e8b8f477f91ae7cab511ec6d32881d765b51e65c 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -119,32 +119,6 @@ public:
   }
 };
 
-// Not yet implemented, but the type name is necessary for 'seq' diagnostics, so
-// this provides a basic, do-nothing implementation. We still need to add this
-// type to the visitors/etc, as well as get it to take its proper arguments.
-class OpenACCGangClause : public OpenACCClause {
-protected:
-  OpenACCGangClause(SourceLocation BeginLoc, SourceLocation EndLoc)
-      : OpenACCClause(OpenACCClauseKind::Gang, BeginLoc, EndLoc) {
-    llvm_unreachable("Not yet implemented");
-  }
-
-public:
-  static bool classof(const OpenACCClause *C) {
-    return C->getClauseKind() == OpenACCClauseKind::Gang;
-  }
-
-  static OpenACCGangClause *
-  Create(const ASTContext &Ctx, SourceLocation BeginLoc, SourceLocation EndLoc);
-
-  child_range children() {
-    return child_range(child_iterator(), child_iterator());
-  }
-  const_child_range children() const {
-    return const_child_range(const_child_iterator(), const_child_iterator());
-  }
-};
-
 // Not yet implemented, but the type name is necessary for 'seq' diagnostics, so
 // this provides a basic, do-nothing implementation. We still need to add this
 // type to the visitors/etc, as well as get it to take its proper arguments.
@@ -157,7 +131,7 @@ protected:
 
 public:
   static bool classof(const OpenACCClause *C) {
-    return C->getClauseKind() == OpenACCClauseKind::Gang;
+    return C->getClauseKind() == OpenACCClauseKind::Vector;
   }
 
   static OpenACCVectorClause *
@@ -171,32 +145,6 @@ public:
   }
 };
 
-// Not yet implemented, but the type name is necessary for 'seq' diagnostics, so
-// this provides a basic, do-nothing implementation. We still need to add this
-// type to the visitors/etc, as well as get it to take its proper arguments.
-class OpenACCWorkerClause : public OpenACCClause {
-protected:
-  OpenACCWorkerClause(SourceLocation BeginLoc, SourceLocation EndLoc)
-      : OpenACCClause(OpenACCClauseKind::Gang, BeginLoc, EndLoc) {
-    llvm_unreachable("Not yet implemented");
-  }
-
-public:
-  static bool classof(const OpenACCClause *C) {
-    return C->getClauseKind() == OpenACCClauseKind::Gang;
-  }
-
-  static OpenACCWorkerClause *
-  Create(const ASTContext &Ctx, SourceLocation BeginLoc, SourceLocation EndLoc);
-
-  child_range children() {
-    return child_range(child_iterator(), child_iterator());
-  }
-  const_child_range children() const {
-    return const_child_range(const_child_iterator(), const_child_iterator());
-  }
-};
-
 /// Represents a clause that has a list of parameters.
 class OpenACCClauseWithParams : public OpenACCClause {
   /// Location of the '('.
@@ -535,6 +483,54 @@ public:
   Expr *getIntExpr() { return hasIntExpr() ? getExprs()[0] : nullptr; };
 };
 
+class OpenACCGangClause final
+    : public OpenACCClauseWithExprs,
+      public llvm::TrailingObjects<OpenACCGangClause, Expr *, OpenACCGangKind> {
+protected:
+  OpenACCGangClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                    ArrayRef<OpenACCGangKind> GangKinds,
+                    ArrayRef<Expr *> IntExprs, SourceLocation EndLoc);
+
+  OpenACCGangKind getGangKind(unsigned I) const {
+    return getTrailingObjects<OpenACCGangKind>()[I];
+  }
+
+public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Gang;
+  }
+
+  size_t numTrailingObjects(OverloadToken<Expr *>) const {
+    return getNumExprs();
+  }
+
+  unsigned getNumExprs() const { return getExprs().size(); }
+  std::pair<OpenACCGangKind, const Expr *> getExpr(unsigned I) const {
+    return {getGangKind(I), getExprs()[I]};
+  }
+
+  static OpenACCGangClause *
+  Create(const ASTContext &Ctx, SourceLocation BeginLoc,
+         SourceLocation LParenLoc, ArrayRef<OpenACCGangKind> GangKinds,
+         ArrayRef<Expr *> IntExprs, SourceLocation EndLoc);
+};
+
+class OpenACCWorkerClause : public OpenACCClauseWithSingleIntExpr {
+protected:
+  OpenACCWorkerClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                      Expr *IntExpr, SourceLocation EndLoc);
+
+public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Worker;
+  }
+
+  static OpenACCWorkerClause *Create(const ASTContext &Ctx,
+                                     SourceLocation BeginLoc,
+                                     SourceLocation LParenLoc, Expr *IntExpr,
+                                     SourceLocation EndLoc);
+};
+
 class OpenACCNumWorkersClause : public OpenACCClauseWithSingleIntExpr {
   OpenACCNumWorkersClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
                           Expr *IntExpr, SourceLocation EndLoc);
diff --git a/clang/include/clang/AST/PrettyPrinter.h b/clang/include/clang/AST/PrettyPrinter.h
index 332ac3c6a004a95678f117f31ddad80d0dc7527e..91818776b770cf2f10a0d90cf1b485ee2b204309 100644
--- a/clang/include/clang/AST/PrettyPrinter.h
+++ b/clang/include/clang/AST/PrettyPrinter.h
@@ -55,15 +55,17 @@ public:
 /// This type is intended to be small and suitable for passing by value.
 /// It is very frequently copied.
 struct PrintingPolicy {
+  enum SuppressInlineNamespaceMode : uint8_t { None, Redundant, All };
+
   /// Create a default printing policy for the specified language.
   PrintingPolicy(const LangOptions &LO)
       : Indentation(2), SuppressSpecifiers(false),
         SuppressTagKeyword(LO.CPlusPlus), IncludeTagDefinition(false),
         SuppressScope(false), SuppressUnwrittenScope(false),
-        SuppressInlineNamespace(true), SuppressElaboration(false),
-        SuppressInitializers(false), ConstantArraySizeAsWritten(false),
-        AnonymousTagLocations(true), SuppressStrongLifetime(false),
-        SuppressLifetimeQualifiers(false),
+        SuppressInlineNamespace(SuppressInlineNamespaceMode::Redundant),
+        SuppressElaboration(false), SuppressInitializers(false),
+        ConstantArraySizeAsWritten(false), AnonymousTagLocations(true),
+        SuppressStrongLifetime(false), SuppressLifetimeQualifiers(false),
         SuppressTemplateArgsInCXXConstructors(false),
         SuppressDefaultTemplateArgs(true), Bool(LO.Bool),
         Nullptr(LO.CPlusPlus11 || LO.C23), NullptrTypeInNamespace(LO.CPlusPlus),
@@ -141,10 +143,12 @@ struct PrintingPolicy {
   unsigned SuppressUnwrittenScope : 1;
 
   /// Suppress printing parts of scope specifiers that correspond
-  /// to inline namespaces, where the name is unambiguous with the specifier
+  /// to inline namespaces.
+  /// If Redudant, where the name is unambiguous with the specifier removed.
+  /// If All, even if the name is ambiguous with the specifier
   /// removed.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned SuppressInlineNamespace : 1;
+  LLVM_PREFERRED_TYPE(SuppressInlineNamespaceMode)
+  unsigned SuppressInlineNamespace : 2;
 
   /// Ignore qualifiers and tag keywords as specified by elaborated type sugar,
   /// instead letting the underlying type print as normal.
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 7068473a0e12ac4f12373e700271bf2cee66d342..bda8a48be92bda81fe596f6410f651e9b8733947 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -215,6 +215,18 @@ def FminF16F128 : Builtin, F16F128MathTemplate {
   let Prototype = "T(T, T)";
 }
 
+def FmaximumNumF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_fmaximum_num"];
+  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const, Constexpr];
+  let Prototype = "T(T, T)";
+}
+
+def FminimumNumF16F128 : Builtin, F16F128MathTemplate {
+  let Spellings = ["__builtin_fminimum_num"];
+  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const, Constexpr];
+  let Prototype = "T(T, T)";
+}
+
 def Atan2F128 : Builtin {
   let Spellings = ["__builtin_atan2f128"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, ConstIgnoringErrnoAndExceptions];
@@ -1186,13 +1198,13 @@ def AllowRuntimeCheck : Builtin {
 
 def ShuffleVector : Builtin {
   let Spellings = ["__builtin_shufflevector"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
 def ConvertVector : Builtin {
   let Spellings = ["__builtin_convertvector"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
@@ -3728,6 +3740,22 @@ def Fmin : FPMathTemplate, LibBuiltin<"math.h"> {
   let OnlyBuiltinPrefixedAliasIsConstexpr = 1;
 }
 
+def FmaximumNum : FPMathTemplate, LibBuiltin<"math.h"> {
+  let Spellings = ["fmaximum_num"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "T(T, T)";
+  let AddBuiltinPrefixedAlias = 1;
+  let OnlyBuiltinPrefixedAliasIsConstexpr = 1;
+}
+
+def FminimumNum : FPMathTemplate, LibBuiltin<"math.h"> {
+  let Spellings = ["fminimum_num"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "T(T, T)";
+  let AddBuiltinPrefixedAlias = 1;
+  let OnlyBuiltinPrefixedAliasIsConstexpr = 1;
+}
+
 def Hypot : FPMathTemplate, LibBuiltin<"math.h"> {
   let Spellings = ["hypot"];
   let Attributes = [NoThrow, ConstIgnoringErrnoAndExceptions];
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def
index 6b7bce5bc00d4f3bf57eb549054210161d12033d..969dd9e41ebfa3a0021ea31c331969a97b36c290 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.def
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.def
@@ -27,8 +27,10 @@
 #pragma push_macro("SM_89")
 #pragma push_macro("SM_90")
 #pragma push_macro("SM_90a")
+#pragma push_macro("SM_100")
+#define SM_100 "sm_100"
 #define SM_90a "sm_90a"
-#define SM_90 "sm_90|" SM_90a
+#define SM_90 "sm_90|" SM_90a "|" SM_100
 #define SM_89 "sm_89|" SM_90
 #define SM_87 "sm_87|" SM_89
 #define SM_86 "sm_86|" SM_87
@@ -63,7 +65,9 @@
 #pragma push_macro("PTX83")
 #pragma push_macro("PTX84")
 #pragma push_macro("PTX85")
-#define PTX85 "ptx85"
+#pragma push_macro("PTX86")
+#define PTX86 "ptx86"
+#define PTX85 "ptx85|" PTX86
 #define PTX84 "ptx84|" PTX85
 #define PTX83 "ptx83|" PTX84
 #define PTX82 "ptx82|" PTX83
@@ -1086,6 +1090,7 @@ TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78))
 #pragma pop_macro("SM_89")
 #pragma pop_macro("SM_90")
 #pragma pop_macro("SM_90a")
+#pragma pop_macro("SM_100")
 #pragma pop_macro("PTX42")
 #pragma pop_macro("PTX60")
 #pragma pop_macro("PTX61")
@@ -1108,3 +1113,4 @@ TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78))
 #pragma pop_macro("PTX83")
 #pragma pop_macro("PTX84")
 #pragma pop_macro("PTX85")
+#pragma pop_macro("PTX86")
diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index 83699f8897f6630d4015667dac6961d7bf0e0bad..a18e62620dd5d0b72e6499638c98237f9b61b4b5 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -43,9 +43,10 @@ enum class CudaVersion {
   CUDA_123,
   CUDA_124,
   CUDA_125,
+  CUDA_126,
   FULLY_SUPPORTED = CUDA_123,
   PARTIALLY_SUPPORTED =
-      CUDA_125, // Partially supported. Proceed with a warning.
+      CUDA_126, // Partially supported. Proceed with a warning.
   NEW = 10000,  // Too new. Issue a warning, but allow using it.
 };
 const char *CudaVersionToString(CudaVersion V);
@@ -78,6 +79,7 @@ enum class OffloadArch {
   SM_89,
   SM_90,
   SM_90a,
+  SM_100,
   GFX600,
   GFX601,
   GFX602,
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index ae08a9160d3cef8e8b3de87f5900813fdacc7544..7c702358287aa20e8200e7d94c06dc6116ca020b 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -250,7 +250,12 @@ def ext_designated_init_brace_elision : ExtWarn<
 // Declarations.
 def ext_plain_complex : ExtWarn<
   "plain '_Complex' requires a type specifier; assuming '_Complex double'">;
-def ext_imaginary_constant : Extension<
+def warn_c23_compat_imaginary_constant : Warning<
+  "imaginary constants are incompatible with C standards before C2y">,
+  DefaultIgnore, InGroup<CPre2yCompat>;
+def ext_c2y_imaginary_constant : Extension<
+  "imaginary constants are a C2y extension">, InGroup<C2y>;
+def ext_gnu_imaginary_constant : Extension<
   "imaginary constants are a GNU extension">, InGroup<GNUImaginaryConstant>;
 def ext_integer_complex : Extension<
   "complex integer types are a GNU extension">, InGroup<GNUComplexInteger>;
@@ -439,7 +444,7 @@ def warn_reserved_extern_symbol: Warning<
   InGroup<ReservedIdentifier>, DefaultIgnore;
 def warn_deprecated_literal_operator_id: Warning<
   "identifier %0 preceded by whitespace in a literal operator declaration "
-  "is deprecated">, InGroup<DeprecatedLiteralOperator>, DefaultIgnore;
+  "is deprecated">, InGroup<DeprecatedLiteralOperator>;
 def warn_reserved_module_name : Warning<
   "%0 is a reserved name for a module">, InGroup<ReservedModuleIdentifier>;
 def warn_import_implementation_partition_unit_in_interface_unit : Warning<
@@ -5262,13 +5267,6 @@ def note_template_arg_refers_here_func : Note<
 def err_template_arg_template_params_mismatch : Error<
   "template template argument has different template parameters than its "
   "corresponding template template parameter">;
-def note_template_arg_template_params_mismatch : Note<
-  "template template argument has different template parameters than its "
-  "corresponding template template parameter">;
-def err_non_deduced_mismatch : Error<
-  "could not match %diff{$ against $|types}0,1">;
-def err_inconsistent_deduction : Error<
-  "conflicting deduction %diff{$ against $|types}0,1 for parameter">;
 def err_template_arg_not_integral_or_enumeral : Error<
   "non-type template argument of type %0 must have an integral or enumeration"
   " type">;
@@ -12609,6 +12607,7 @@ def err_acc_duplicate_clause_disallowed
     : Error<"OpenACC '%1' clause cannot appear more than once on a '%0' "
             "directive">;
 def note_acc_previous_clause_here : Note<"previous clause is here">;
+def note_acc_previous_expr_here : Note<"previous expression is here">;
 def err_acc_branch_in_out_compute_construct
     : Error<"invalid %select{branch|return|throw}0 %select{out of|into}1 "
             "OpenACC Compute Construct">;
@@ -12715,6 +12714,25 @@ def err_acc_insufficient_loops
 def err_acc_intervening_code
     : Error<"inner loops must be tightly nested inside a '%0' clause on "
             "a 'loop' construct">;
+def err_acc_gang_multiple_elt
+    : Error<"OpenACC 'gang' clause may have at most one %select{unnamed or "
+            "'num'|'dim'|'static'}0 argument">;
+def err_acc_int_arg_invalid
+    : Error<"'%1' argument on '%0' clause is not permitted on a%select{n "
+            "orphaned|||}2 'loop' construct %select{|associated with a "
+            "'parallel' compute construct|associated with a 'kernels' compute "
+            "construct|associated with a 'serial' compute construct}2">;
+def err_acc_gang_dim_value
+    : Error<"argument to 'gang' clause dimension must be %select{a constant "
+            "expression|1, 2, or 3: evaluated to %1}0">;
+def err_acc_num_arg_conflict
+    : Error<"'num' argument to '%0' clause not allowed on a 'loop' construct "
+            "associated with a 'kernels' construct that has a "
+            "'%select{num_gangs|num_workers}1' "
+            "clause">;
+def err_acc_clause_in_clause_region
+    : Error<"loop with a '%0' clause may not exist in the region of a '%1' "
+            "clause%select{| on a 'kernels' compute construct}2">;
 
 // AMDGCN builtins diagnostics
 def err_amdgcn_global_load_lds_size_invalid_value : Error<"invalid size value">;
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index a380e5ae69c41820b835106d0d21cee884207449..4c0b56dc13e62555c36c66547007daf91a2f854a 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -42,6 +42,7 @@ VISIT_CLAUSE(DevicePtr)
 VISIT_CLAUSE(DeviceType)
 CLAUSE_ALIAS(DType, DeviceType, false)
 VISIT_CLAUSE(FirstPrivate)
+VISIT_CLAUSE(Gang)
 VISIT_CLAUSE(If)
 VISIT_CLAUSE(Independent)
 VISIT_CLAUSE(NoCreate)
@@ -55,6 +56,7 @@ VISIT_CLAUSE(Seq)
 VISIT_CLAUSE(Tile)
 VISIT_CLAUSE(VectorLength)
 VISIT_CLAUSE(Wait)
+VISIT_CLAUSE(Worker)
 
 #undef VISIT_CLAUSE
 #undef CLAUSE_ALIAS
diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h
index 7b9d619a8aec6d5b6764493db13cef5937f9fc2c..4310c28bc935e8719a8a73e7a76cb3b5a91da34a 100644
--- a/clang/include/clang/Basic/OpenACCKinds.h
+++ b/clang/include/clang/Basic/OpenACCKinds.h
@@ -22,7 +22,7 @@ namespace clang {
 // Represents the Construct/Directive kind of a pragma directive. Note the
 // OpenACC standard is inconsistent between calling these Construct vs
 // Directive, but we're calling it a Directive to be consistent with OpenMP.
-enum class OpenACCDirectiveKind {
+enum class OpenACCDirectiveKind : uint8_t {
   // Compute Constructs.
   Parallel,
   Serial,
@@ -152,7 +152,7 @@ inline bool isOpenACCComputeDirectiveKind(OpenACCDirectiveKind K) {
          K == OpenACCDirectiveKind::Kernels;
 }
 
-enum class OpenACCAtomicKind {
+enum class OpenACCAtomicKind : uint8_t {
   Read,
   Write,
   Update,
@@ -161,7 +161,7 @@ enum class OpenACCAtomicKind {
 };
 
 /// Represents the kind of an OpenACC clause.
-enum class OpenACCClauseKind {
+enum class OpenACCClauseKind : uint8_t {
   /// 'finalize' clause, allowed on 'exit data' directive.
   Finalize,
   /// 'if_present' clause, allowed on 'host_data' and 'update' directives.
@@ -459,7 +459,7 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
   return printOpenACCClauseKind(Out, K);
 }
 
-enum class OpenACCDefaultClauseKind {
+enum class OpenACCDefaultClauseKind : uint8_t {
   /// 'none' option.
   None,
   /// 'present' option.
@@ -492,7 +492,7 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
   return printOpenACCDefaultClauseKind(Out, K);
 }
 
-enum class OpenACCReductionOperator {
+enum class OpenACCReductionOperator : uint8_t {
   /// '+'.
   Addition,
   /// '*'.
@@ -550,6 +550,36 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
                                      OpenACCReductionOperator Op) {
   return printOpenACCReductionOperator(Out, Op);
 }
+
+enum class OpenACCGangKind : uint8_t {
+  /// num:
+  Num,
+  /// dim:
+  Dim,
+  /// static:
+  Static
+};
+
+template <typename StreamTy>
+inline StreamTy &printOpenACCGangKind(StreamTy &Out, OpenACCGangKind GK) {
+  switch (GK) {
+  case OpenACCGangKind::Num:
+    return Out << "num";
+  case OpenACCGangKind::Dim:
+    return Out << "dim";
+  case OpenACCGangKind::Static:
+    return Out << "static";
+  }
+  llvm_unreachable("unknown gang kind");
+}
+inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &Out,
+                                             OpenACCGangKind Op) {
+  return printOpenACCGangKind(Out, Op);
+}
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
+                                     OpenACCGangKind Op) {
+  return printOpenACCGangKind(Out, Op);
+}
 } // namespace clang
 
 #endif // LLVM_CLANG_BASIC_OPENACCKINDS_H
diff --git a/clang/include/clang/Basic/ParsedAttrInfo.h b/clang/include/clang/Basic/ParsedAttrInfo.h
index 537d8f3391d5898705317fd5637c450ec8c00625..fab5c6f1377d270b1c2ec84593c2f6f273a9f7f9 100644
--- a/clang/include/clang/Basic/ParsedAttrInfo.h
+++ b/clang/include/clang/Basic/ParsedAttrInfo.h
@@ -24,6 +24,7 @@
 
 namespace clang {
 
+class Attr;
 class Decl;
 class LangOptions;
 class ParsedAttr;
@@ -154,6 +155,15 @@ public:
                                            const ParsedAttr &Attr) const {
     return NotHandled;
   }
+  /// If this ParsedAttrInfo knows how to handle this ParsedAttr applied to this
+  /// Stmt then do so (referencing the resulting Attr in Result) and return
+  /// either AttributeApplied if it was applied or AttributeNotApplied if it
+  /// wasn't. Otherwise return NotHandled.
+  virtual AttrHandling handleStmtAttribute(Sema &S, Stmt *St,
+                                           const ParsedAttr &Attr,
+                                           class Attr *&Result) const {
+    return NotHandled;
+  }
 
   static const ParsedAttrInfo &get(const AttributeCommonInfo &A);
   static ArrayRef<const ParsedAttrInfo *> getAllBuiltin();
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index b18965dd7fe1aab62f8e4b26141d449429e57817..5c2204a3c4d3180ed0c41e735969da32709a4d76 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2002,16 +2002,18 @@ def fparse_all_comments : Flag<["-"], "fparse-all-comments">, Group<f_clang_Grou
   Visibility<[ClangOption, CC1Option]>,
   MarshallingInfoFlag<LangOpts<"CommentOpts.ParseAllComments">>;
 def frecord_command_line : Flag<["-"], "frecord-command-line">,
-  DocBrief<[{Generate a section named ".GCC.command.line" containing the clang
+  DocBrief<[{Generate a section named ".GCC.command.line" containing the
 driver command-line. After linking, the section may contain multiple command
 lines, which will be individually terminated by null bytes. Separate arguments
 within a command line are combined with spaces; spaces and backslashes within an
 argument are escaped with backslashes. This format differs from the format of
 the equivalent section produced by GCC with the -frecord-gcc-switches flag.
 This option is currently only supported on ELF targets.}]>,
-  Group<f_clang_Group>;
+  Group<f_Group>,
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>;
 def fno_record_command_line : Flag<["-"], "fno-record-command-line">,
-  Group<f_clang_Group>;
+  Group<f_Group>,
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>;
 def : Flag<["-"], "frecord-gcc-switches">, Alias<frecord_command_line>;
 def : Flag<["-"], "fno-record-gcc-switches">, Alias<fno_record_command_line>;
 def fcommon : Flag<["-"], "fcommon">, Group<f_Group>,
@@ -7173,6 +7175,9 @@ def mrelocation_model : Separate<["-"], "mrelocation-model">,
   NormalizedValues<["Static", "PIC_", "ROPI", "RWPI", "ROPI_RWPI", "DynamicNoPIC"]>,
   MarshallingInfoEnum<CodeGenOpts<"RelocationModel">, "PIC_">;
 def debug_info_kind_EQ : Joined<["-"], "debug-info-kind=">;
+def record_command_line : Separate<["-"], "record-command-line">,
+  HelpText<"The string to embed in the .LLVM.command.line section.">,
+  MarshallingInfoString<CodeGenOpts<"RecordCommandLine">>;
 
 } // let Visibility = [CC1Option, CC1AsOption, FC1Option]
 
@@ -7193,9 +7198,6 @@ def debugger_tuning_EQ : Joined<["-"], "debugger-tuning=">,
 def dwarf_debug_flags : Separate<["-"], "dwarf-debug-flags">,
   HelpText<"The string to embed in the Dwarf debug flags record.">,
   MarshallingInfoString<CodeGenOpts<"DwarfDebugFlags">>;
-def record_command_line : Separate<["-"], "record-command-line">,
-  HelpText<"The string to embed in the .LLVM.command.line section.">,
-  MarshallingInfoString<CodeGenOpts<"RecordCommandLine">>;
 def compress_debug_sections_EQ : Joined<["-", "--"], "compress-debug-sections=">,
     HelpText<"DWARF debug sections compression type">, Values<"none,zlib,zstd">,
     NormalizedValuesScope<"llvm::DebugCompressionType">, NormalizedValues<["None", "Zlib", "Zstd"]>,
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 3d3e4330902a300b8ebee568713973e1b4c8f05f..a0762b088b68ef3d94e1415b742370a4b2bd5103 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -3847,24 +3847,43 @@ struct FormatStyle {
   ReferenceAlignmentStyle ReferenceAlignment;
 
   // clang-format off
-  /// If ``true``, clang-format will attempt to re-flow comments. That is it
-  /// will touch a comment and *reflow* long comments into new lines, trying to
-  /// obey the ``ColumnLimit``.
-  /// \code
-  ///    false:
-  ///    // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
-  ///    /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information */
-  ///
-  ///    true:
-  ///    // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
-  ///    // information
-  ///    /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
-  ///     * information */
-  /// \endcode
-  /// \version 3.8
-  bool ReflowComments;
+  /// \brief Types of comment reflow style.
+  enum ReflowCommentsStyle : int8_t {
+    /// Leave comments untouched.
+    /// \code
+    ///    // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+    ///    /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information */
+    ///    /* third veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+    ///         * and a misaligned second line */
+    /// \endcode
+    RCS_Never,
+    /// Only apply indentation rules, moving comments left or right, without
+    /// changing formatting inside the comments.
+    /// \code
+    ///    // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+    ///    /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information */
+    ///    /* third veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
+    ///     * and a misaligned second line */
+    /// \endcode
+    RCS_IndentOnly,
+    /// Apply indentation rules and reflow long comments into new lines, trying
+    /// to obey the ``ColumnLimit``.
+    /// \code
+    ///    // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
+    ///    // information
+    ///    /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
+    ///     * information */
+    ///    /* third veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of
+    ///     * information and a misaligned second line */
+    /// \endcode
+    RCS_Always
+  };
   // clang-format on
 
+  /// \brief Comment reformatting style.
+  /// \version 3.8
+  ReflowCommentsStyle ReflowComments;
+
   /// Remove optional braces of control statements (``if``, ``else``, ``for``,
   /// and ``while``) in C++ according to the LLVM coding style.
   /// \warning
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index f21a63a6c3f2e6fe6044f0d1cdaf451a8f5c9c58..acd1da7fa9c39d3dc1f7d5703c32281900ee1f90 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3838,9 +3838,15 @@ private:
   bool ParseOpenACCSizeExprList(OpenACCClauseKind CK,
                                 llvm::SmallVectorImpl<Expr *> &SizeExprs);
   /// Parses a 'gang-arg-list', used for the 'gang' clause.
-  bool ParseOpenACCGangArgList(SourceLocation GangLoc);
-  /// Parses a 'gang-arg', used for the 'gang' clause.
-  bool ParseOpenACCGangArg(SourceLocation GangLoc);
+  bool ParseOpenACCGangArgList(SourceLocation GangLoc,
+                               llvm::SmallVectorImpl<OpenACCGangKind> &GKs,
+                               llvm::SmallVectorImpl<Expr *> &IntExprs);
+
+  using OpenACCGangArgRes = std::pair<OpenACCGangKind, ExprResult>;
+  /// Parses a 'gang-arg', used for the 'gang' clause. Returns a pair of the
+  /// ExprResult (which contains the validity of the expression), plus the gang
+  /// kind for the current argument.
+  OpenACCGangArgRes ParseOpenACCGangArg(SourceLocation GangLoc);
   /// Parses a 'condition' expr, ensuring it results in a
   ExprResult ParseOpenACCConditionExpr();
 
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index d38278c5041118fa71db6d65b70a0575d5a8ab3f..c716a25bb673b8b36a743004d6901771f2a89454 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -925,11 +925,6 @@ class Sema;
 
     bool TookAddressOfOverload : 1;
 
-    /// Have we matched any packs on the parameter side, versus any non-packs on
-    /// the argument side, in a context where the opposite matching is also
-    /// allowed?
-    bool HasMatchedPackOnParmToNonPackOnArg : 1;
-
     /// True if the candidate was found using ADL.
     CallExpr::ADLCallKind IsADLCandidate : 1;
 
@@ -1004,9 +999,8 @@ class Sema;
     friend class OverloadCandidateSet;
     OverloadCandidate()
         : IsSurrogate(false), IgnoreObjectArgument(false),
-          TookAddressOfOverload(false),
-          HasMatchedPackOnParmToNonPackOnArg(false),
-          IsADLCandidate(CallExpr::NotADL), RewriteKind(CRK_None) {}
+          TookAddressOfOverload(false), IsADLCandidate(CallExpr::NotADL),
+          RewriteKind(CRK_None) {}
   };
 
   /// OverloadCandidateSet - A set of overload candidates, used in C++
diff --git a/clang/include/clang/Sema/ScopeInfo.h b/clang/include/clang/Sema/ScopeInfo.h
index 700e361ef83f13c0265ab54c0a1e2161fdb425eb..958d65055fa9b2a6718592cec6324274fb3af810 100644
--- a/clang/include/clang/Sema/ScopeInfo.h
+++ b/clang/include/clang/Sema/ScopeInfo.h
@@ -724,10 +724,16 @@ public:
   /// is deduced (e.g. a lambda or block with omitted return type).
   bool HasImplicitReturnType = false;
 
+  /// Whether this contains an unexpanded parameter pack.
+  bool ContainsUnexpandedParameterPack = false;
+
   /// ReturnType - The target type of return statements in this context,
   /// or null if unknown.
   QualType ReturnType;
 
+  /// Packs introduced by this, if any.
+  SmallVector<NamedDecl *, 4> LocalPacks;
+
   void addCapture(ValueDecl *Var, bool isBlock, bool isByref, bool isNested,
                   SourceLocation Loc, SourceLocation EllipsisLoc,
                   QualType CaptureType, bool Invalid) {
@@ -895,12 +901,6 @@ public:
   /// Whether any of the capture expressions requires cleanups.
   CleanupInfo Cleanup;
 
-  /// Whether the lambda contains an unexpanded parameter pack.
-  bool ContainsUnexpandedParameterPack = false;
-
-  /// Packs introduced by this lambda, if any.
-  SmallVector<NamedDecl*, 4> LocalPacks;
-
   /// Source range covering the explicit template parameter list (if it exists).
   SourceRange ExplicitTemplateParamsRange;
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index f8118ca64ad3f2218d122ff2cb3d5da4cd9a1c81..0faa5aed4eec3bc8bb36d4a6f83694179304f5a9 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -702,10 +702,10 @@ public:
   /// Retrieve the current block, if any.
   sema::BlockScopeInfo *getCurBlock();
 
-  /// Get the innermost lambda enclosing the current location, if any. This
-  /// looks through intervening non-lambda scopes such as local functions and
-  /// blocks.
-  sema::LambdaScopeInfo *getEnclosingLambda() const;
+  /// Get the innermost lambda or block enclosing the current location, if any.
+  /// This looks through intervening non-lambda, non-block scopes such as local
+  /// functions.
+  sema::CapturingScopeInfo *getEnclosingLambdaOrBlock() const;
 
   /// Retrieve the current lambda scope info, if any.
   /// \param IgnoreNonLambdaCapturingScope true if should find the top-most
@@ -872,8 +872,6 @@ public:
   /// For example, user-defined classes, built-in "id" type, etc.
   Scope *TUScope;
 
-  bool WarnedStackExhausted = false;
-
   void incrementMSManglingNumber() const {
     return CurScope->incrementMSManglingNumber();
   }
@@ -1185,6 +1183,8 @@ private:
   std::optional<std::unique_ptr<DarwinSDKInfo>> CachedDarwinSDKInfo;
   bool WarnedDarwinSDKInfoMissing = false;
 
+  bool WarnedStackExhausted = false;
+
   Sema(const Sema &) = delete;
   void operator=(const Sema &) = delete;
 
@@ -10134,8 +10134,7 @@ public:
       ADLCallKind IsADLCandidate = ADLCallKind::NotADL,
       ConversionSequenceList EarlyConversions = std::nullopt,
       OverloadCandidateParamOrder PO = {},
-      bool AggregateCandidateDeduction = false,
-      bool HasMatchedPackOnParmToNonPackOnArg = false);
+      bool AggregateCandidateDeduction = false);
 
   /// Add all of the function declarations in the given function set to
   /// the overload candidate set.
@@ -10170,8 +10169,7 @@ public:
                      bool SuppressUserConversions = false,
                      bool PartialOverloading = false,
                      ConversionSequenceList EarlyConversions = std::nullopt,
-                     OverloadCandidateParamOrder PO = {},
-                     bool HasMatchedPackOnParmToNonPackOnArg = false);
+                     OverloadCandidateParamOrder PO = {});
 
   /// Add a C++ member function template as a candidate to the candidate
   /// set, using template argument deduction to produce an appropriate member
@@ -10217,8 +10215,7 @@ public:
       CXXConversionDecl *Conversion, DeclAccessPair FoundDecl,
       CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
       OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-      bool AllowExplicit, bool AllowResultConversion = true,
-      bool HasMatchedPackOnParmToNonPackOnArg = false);
+      bool AllowExplicit, bool AllowResultConversion = true);
 
   /// Adds a conversion function template specialization
   /// candidate to the overload set, using template argument deduction
@@ -11330,9 +11327,9 @@ public:
       CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc,
       const ParsedAttributesView &Attr, TemplateParameterList *TemplateParams,
       AccessSpecifier AS, SourceLocation ModulePrivateLoc,
-      SourceLocation FriendLoc, unsigned NumOuterTemplateParamLists,
-      TemplateParameterList **OuterTemplateParamLists,
-      SkipBodyInfo *SkipBody = nullptr);
+      SourceLocation FriendLoc,
+      ArrayRef<TemplateParameterList *> OuterTemplateParamLists,
+      bool IsMemberSpecialization, SkipBodyInfo *SkipBody = nullptr);
 
   /// Translates template arguments as provided by the parser
   /// into template arguments used by semantic analysis.
@@ -11371,7 +11368,8 @@ public:
   DeclResult ActOnVarTemplateSpecialization(
       Scope *S, Declarator &D, TypeSourceInfo *DI, LookupResult &Previous,
       SourceLocation TemplateKWLoc, TemplateParameterList *TemplateParams,
-      StorageClass SC, bool IsPartialSpecialization);
+      StorageClass SC, bool IsPartialSpecialization,
+      bool IsMemberSpecialization);
 
   /// Get the specialization of the given variable template corresponding to
   /// the specified argument list, or a null-but-valid result if the arguments
@@ -11641,8 +11639,7 @@ public:
                         SourceLocation RAngleLoc, unsigned ArgumentPackIndex,
                         SmallVectorImpl<TemplateArgument> &SugaredConverted,
                         SmallVectorImpl<TemplateArgument> &CanonicalConverted,
-                        CheckTemplateArgumentKind CTAK, bool PartialOrdering,
-                        bool *MatchedPackOnParmToNonPackOnArg);
+                        CheckTemplateArgumentKind CTAK);
 
   /// Check that the given template arguments can be provided to
   /// the given template, converting the arguments along the way.
@@ -11689,8 +11686,7 @@ public:
       SmallVectorImpl<TemplateArgument> &SugaredConverted,
       SmallVectorImpl<TemplateArgument> &CanonicalConverted,
       bool UpdateArgsWithConversions = true,
-      bool *ConstraintsNotSatisfied = nullptr, bool PartialOrderingTTP = false,
-      bool *MatchedPackOnParmToNonPackOnArg = nullptr);
+      bool *ConstraintsNotSatisfied = nullptr, bool PartialOrderingTTP = false);
 
   bool CheckTemplateTypeArgument(
       TemplateTypeParmDecl *Param, TemplateArgumentLoc &Arg,
@@ -11724,9 +11720,7 @@ public:
   /// It returns true if an error occurred, and false otherwise.
   bool CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
                                      TemplateParameterList *Params,
-                                     TemplateArgumentLoc &Arg,
-                                     bool PartialOrdering,
-                                     bool *MatchedPackOnParmToNonPackOnArg);
+                                     TemplateArgumentLoc &Arg, bool IsDeduced);
 
   void NoteTemplateLocation(const NamedDecl &Decl,
                             std::optional<SourceRange> ParamRange = {});
@@ -12237,8 +12231,8 @@ public:
       SmallVectorImpl<DeducedTemplateArgument> &Deduced,
       unsigned NumExplicitlySpecified, FunctionDecl *&Specialization,
       sema::TemplateDeductionInfo &Info,
-      SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs,
-      bool PartialOverloading, bool PartialOrdering,
+      SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs = nullptr,
+      bool PartialOverloading = false,
       llvm::function_ref<bool()> CheckNonDependent = [] { return false; });
 
   /// Perform template argument deduction from a function call
@@ -12272,8 +12266,7 @@ public:
       TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
       FunctionDecl *&Specialization, sema::TemplateDeductionInfo &Info,
       bool PartialOverloading, bool AggregateDeductionCandidate,
-      bool PartialOrdering, QualType ObjectType,
-      Expr::Classification ObjectClassification,
+      QualType ObjectType, Expr::Classification ObjectClassification,
       llvm::function_ref<bool(ArrayRef<QualType>)> CheckNonDependent);
 
   /// Deduce template arguments when taking the address of a function
@@ -12426,9 +12419,8 @@ public:
                                     sema::TemplateDeductionInfo &Info);
 
   bool isTemplateTemplateParameterAtLeastAsSpecializedAs(
-      TemplateParameterList *PParam, TemplateDecl *PArg, TemplateDecl *AArg,
-      const DefaultArguments &DefaultArgs, SourceLocation ArgLoc,
-      bool PartialOrdering, bool *MatchedPackOnParmToNonPackOnArg);
+      TemplateParameterList *PParam, TemplateDecl *AArg,
+      const DefaultArguments &DefaultArgs, SourceLocation Loc, bool IsDeduced);
 
   /// Mark which template parameters are used in a given expression.
   ///
@@ -12737,9 +12729,6 @@ public:
 
       /// We are instantiating a type alias template declaration.
       TypeAliasTemplateInstantiation,
-
-      /// We are performing partial ordering for template template parameters.
-      PartialOrderingTTP,
     } Kind;
 
     /// Was the enclosing context a non-instantiation SFINAE context?
@@ -12961,12 +12950,6 @@ public:
                           TemplateDecl *Entity, BuildingDeductionGuidesTag,
                           SourceRange InstantiationRange = SourceRange());
 
-    struct PartialOrderingTTP {};
-    /// \brief Note that we are partial ordering template template parameters.
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation ArgLoc,
-                          PartialOrderingTTP, TemplateDecl *PArg,
-                          SourceRange InstantiationRange = SourceRange());
-
     /// Note that we have finished instantiating this template.
     void Clear();
 
@@ -13027,28 +13010,14 @@ public:
   /// dealing with a specialization. This is only relevant for function
   /// template specializations.
   ///
-  /// \param Pattern If non-NULL, indicates the pattern from which we will be
-  /// instantiating the definition of the given declaration, \p ND. This is
-  /// used to determine the proper set of template instantiation arguments for
-  /// friend function template specializations.
-  ///
   /// \param ForConstraintInstantiation when collecting arguments,
   /// ForConstraintInstantiation indicates we should continue looking when
   /// encountering a lambda generic call operator, and continue looking for
   /// arguments on an enclosing class template.
-  ///
-  /// \param SkipForSpecialization when specified, any template specializations
-  /// in a traversal would be ignored.
-  /// \param ForDefaultArgumentSubstitution indicates we should continue looking
-  /// when encountering a specialized member function template, rather than
-  /// returning immediately.
   MultiLevelTemplateArgumentList getTemplateInstantiationArgs(
       const NamedDecl *D, const DeclContext *DC = nullptr, bool Final = false,
       std::optional<ArrayRef<TemplateArgument>> Innermost = std::nullopt,
-      bool RelativeToPrimary = false, const FunctionDecl *Pattern = nullptr,
-      bool ForConstraintInstantiation = false,
-      bool SkipForSpecialization = false,
-      bool ForDefaultArgumentSubstitution = false);
+      bool RelativeToPrimary = false, bool ForConstraintInstantiation = false);
 
   /// RAII object to handle the state changes required to synthesize
   /// a function body.
@@ -13427,8 +13396,7 @@ public:
   bool InstantiateClassTemplateSpecialization(
       SourceLocation PointOfInstantiation,
       ClassTemplateSpecializationDecl *ClassTemplateSpec,
-      TemplateSpecializationKind TSK, bool Complain = true,
-      bool PrimaryHasMatchedPackOnParmToNonPackOnArg = false);
+      TemplateSpecializationKind TSK, bool Complain = true);
 
   /// Instantiates the definitions of all of the member
   /// of the given class, which is an instantiation of a class template
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 97386d2378b7583b2a2fda290b7e9dac4e4e0a87..e253610a84b0bf5c4da13c29c9a01f4aba59f867 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -38,9 +38,20 @@ private:
   /// haven't had their 'parent' compute construct set yet. Entires will only be
   /// made to this list in the case where we know the loop isn't an orphan.
   llvm::SmallVector<OpenACCLoopConstruct *> ParentlessLoopConstructs;
-  /// Whether we are inside of a compute construct, and should add loops to the
-  /// above collection.
-  bool InsideComputeConstruct = false;
+
+  struct ComputeConstructInfo {
+    /// Which type of compute construct we are inside of, which we can use to
+    /// determine whether we should add loops to the above collection.  We can
+    /// also use it to diagnose loop construct clauses.
+    OpenACCDirectiveKind Kind = OpenACCDirectiveKind::Invalid;
+    // If we have an active compute construct, stores the list of clauses we've
+    // prepared for it, so that we can diagnose limitations on child constructs.
+    ArrayRef<OpenACCClause *> Clauses;
+  } ActiveComputeConstructInfo;
+
+  bool isInComputeConstruct() const {
+    return ActiveComputeConstructInfo.Kind != OpenACCDirectiveKind::Invalid;
+  }
 
   /// Certain clauses care about the same things that aren't specific to the
   /// individual clause, but can be shared by a few, so store them here. All
@@ -99,6 +110,20 @@ private:
   } TileInfo;
 
 public:
+  ComputeConstructInfo &getActiveComputeConstructInfo() {
+    return ActiveComputeConstructInfo;
+  }
+
+  /// If there is a current 'active' loop construct with a 'gang' clause on a
+  /// 'kernel' construct, this will have the source location for it. This
+  /// permits us to implement the restriction of no further 'gang' clauses.
+  SourceLocation LoopGangClauseOnKernelLoc;
+  /// If there is a current 'active' loop construct with a 'worker' clause on it
+  /// (on any sort of construct), this has the source location for it.  This
+  /// permits us to implement the restriction of no further 'gang' or 'worker'
+  /// clauses.
+  SourceLocation LoopWorkerClauseLoc;
+
   // Redeclaration of the version in OpenACCClause.h.
   using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
 
@@ -149,9 +174,14 @@ public:
       Expr *LoopCount;
     };
 
+    struct GangDetails {
+      SmallVector<OpenACCGangKind> GangKinds;
+      SmallVector<Expr *> IntExprs;
+    };
+
     std::variant<std::monostate, DefaultDetails, ConditionDetails,
                  IntExprDetails, VarListDetails, WaitDetails, DeviceTypeDetails,
-                 ReductionDetails, CollapseDetails>
+                 ReductionDetails, CollapseDetails, GangDetails>
         Details = std::monostate{};
 
   public:
@@ -199,11 +229,15 @@ public:
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::Tile ||
+              ClauseKind == OpenACCClauseKind::Worker ||
+              ClauseKind == OpenACCClauseKind::Vector ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
 
       // 'async' and 'wait' have an optional IntExpr, so be tolerant of that.
       if ((ClauseKind == OpenACCClauseKind::Async ||
+           ClauseKind == OpenACCClauseKind::Worker ||
+           ClauseKind == OpenACCClauseKind::Vector ||
            ClauseKind == OpenACCClauseKind::Wait) &&
           std::holds_alternative<std::monostate>(Details))
         return 0;
@@ -245,9 +279,20 @@ public:
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::Tile ||
+              ClauseKind == OpenACCClauseKind::Gang ||
+              ClauseKind == OpenACCClauseKind::Worker ||
+              ClauseKind == OpenACCClauseKind::Vector ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
 
+      if (ClauseKind == OpenACCClauseKind::Gang) {
+        // There might not be any gang int exprs, as this is an optional
+        // argument.
+        if (std::holds_alternative<std::monostate>(Details))
+          return {};
+        return std::get<GangDetails>(Details).IntExprs;
+      }
+
       return std::get<IntExprDetails>(Details).IntExprs;
     }
 
@@ -259,6 +304,16 @@ public:
       return std::get<ReductionDetails>(Details).Op;
     }
 
+    ArrayRef<OpenACCGangKind> getGangKinds() const {
+      assert(ClauseKind == OpenACCClauseKind::Gang &&
+             "Parsed clause kind does not have gang kind");
+      // The args on gang are optional, so this might not actually hold
+      // anything.
+      if (std::holds_alternative<std::monostate>(Details))
+        return {};
+      return std::get<GangDetails>(Details).GangKinds;
+    }
+
     ArrayRef<Expr *> getVarList() {
       assert((ClauseKind == OpenACCClauseKind::Private ||
               ClauseKind == OpenACCClauseKind::NoCreate ||
@@ -357,6 +412,8 @@ public:
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::Tile ||
+              ClauseKind == OpenACCClauseKind::Worker ||
+              ClauseKind == OpenACCClauseKind::Vector ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
       Details = IntExprDetails{{IntExprs.begin(), IntExprs.end()}};
@@ -366,11 +423,32 @@ public:
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::Tile ||
+              ClauseKind == OpenACCClauseKind::Worker ||
+              ClauseKind == OpenACCClauseKind::Vector ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
       Details = IntExprDetails{std::move(IntExprs)};
     }
 
+    void setGangDetails(ArrayRef<OpenACCGangKind> GKs,
+                        ArrayRef<Expr *> IntExprs) {
+      assert(ClauseKind == OpenACCClauseKind::Gang &&
+             "Parsed Clause kind does not have gang details");
+      assert(GKs.size() == IntExprs.size() && "Mismatched kind/size?");
+
+      Details = GangDetails{{GKs.begin(), GKs.end()},
+                            {IntExprs.begin(), IntExprs.end()}};
+    }
+
+    void setGangDetails(llvm::SmallVector<OpenACCGangKind> &&GKs,
+                        llvm::SmallVector<Expr *> &&IntExprs) {
+      assert(ClauseKind == OpenACCClauseKind::Gang &&
+             "Parsed Clause kind does not have gang details");
+      assert(GKs.size() == IntExprs.size() && "Mismatched kind/size?");
+
+      Details = GangDetails{std::move(GKs), std::move(IntExprs)};
+    }
+
     void setVarListDetails(ArrayRef<Expr *> VarList, bool IsReadOnly,
                            bool IsZero) {
       assert((ClauseKind == OpenACCClauseKind::Private ||
@@ -545,10 +623,12 @@ public:
                                    SourceLocation RBLoc);
   /// Checks the loop depth value for a collapse clause.
   ExprResult CheckCollapseLoopCount(Expr *LoopCount);
-  /// Checks a single size expr for a tile clause. 'gang' could possibly call
-  /// this, but has slightly stricter rules as to valid values.
+  /// Checks a single size expr for a tile clause.
   ExprResult CheckTileSizeExpr(Expr *SizeExpr);
 
+  // Check a single expression on a gang clause.
+  ExprResult CheckGangExpr(OpenACCGangKind GK, Expr *E);
+
   ExprResult BuildOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc);
   ExprResult ActOnOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc);
 
@@ -595,8 +675,10 @@ public:
   /// Loop needing its parent construct.
   class AssociatedStmtRAII {
     SemaOpenACC &SemaRef;
-    bool WasInsideComputeConstruct;
+    ComputeConstructInfo OldActiveComputeConstructInfo;
     OpenACCDirectiveKind DirKind;
+    SourceLocation OldLoopGangClauseOnKernelLoc;
+    SourceLocation OldLoopWorkerClauseLoc;
     llvm::SmallVector<OpenACCLoopConstruct *> ParentlessLoopConstructs;
     LoopInConstructRAII LoopRAII;
 
diff --git a/clang/include/clang/Sema/Template.h b/clang/include/clang/Sema/Template.h
index fe27290efdbfc591a7308fba5d90902db6532175..6872d04cc4dfb9bf02caf931a1f5f5f30a93d3b0 100644
--- a/clang/include/clang/Sema/Template.h
+++ b/clang/include/clang/Sema/Template.h
@@ -411,10 +411,10 @@ enum class TemplateSubstitutionKind : char {
     /// lookup will search our outer scope.
     bool CombineWithOuterScope;
 
-    /// Whether this scope is being used to instantiate a lambda expression,
-    /// in which case it should be reused for instantiating the lambda's
-    /// FunctionProtoType.
-    bool InstantiatingLambda = false;
+    /// Whether this scope is being used to instantiate a lambda or block
+    /// expression, in which case it should be reused for instantiating the
+    /// lambda's FunctionProtoType.
+    bool InstantiatingLambdaOrBlock = false;
 
     /// If non-NULL, the template parameter pack that has been
     /// partially substituted per C++0x [temp.arg.explicit]p9.
@@ -431,10 +431,10 @@ enum class TemplateSubstitutionKind : char {
 
   public:
     LocalInstantiationScope(Sema &SemaRef, bool CombineWithOuterScope = false,
-                            bool InstantiatingLambda = false)
+                            bool InstantiatingLambdaOrBlock = false)
         : SemaRef(SemaRef), Outer(SemaRef.CurrentInstantiationScope),
           CombineWithOuterScope(CombineWithOuterScope),
-          InstantiatingLambda(InstantiatingLambda) {
+          InstantiatingLambdaOrBlock(InstantiatingLambdaOrBlock) {
       SemaRef.CurrentInstantiationScope = this;
     }
 
@@ -561,8 +561,8 @@ enum class TemplateSubstitutionKind : char {
     /// Determine whether D is a pack expansion created in this scope.
     bool isLocalPackExpansion(const Decl *D);
 
-    /// Determine whether this scope is for instantiating a lambda.
-    bool isLambda() const { return InstantiatingLambda; }
+    /// Determine whether this scope is for instantiating a lambda or block.
+    bool isLambdaOrBlock() const { return InstantiatingLambdaOrBlock; }
   };
 
   class TemplateDeclInstantiator
diff --git a/clang/include/clang/Sema/TemplateDeduction.h b/clang/include/clang/Sema/TemplateDeduction.h
index 9c12eef5c42a064f4df08ae827fb3b89dd3c8433..28b014fd84e4b35a641f925080cf4551208e9870 100644
--- a/clang/include/clang/Sema/TemplateDeduction.h
+++ b/clang/include/clang/Sema/TemplateDeduction.h
@@ -51,11 +51,6 @@ class TemplateDeductionInfo {
   /// Have we suppressed an error during deduction?
   bool HasSFINAEDiagnostic = false;
 
-  /// Have we matched any packs on the parameter side, versus any non-packs on
-  /// the argument side, in a context where the opposite matching is also
-  /// allowed?
-  bool MatchedPackOnParmToNonPackOnArg = false;
-
   /// The template parameter depth for which we're performing deduction.
   unsigned DeducedDepth;
 
@@ -92,14 +87,6 @@ public:
     return DeducedDepth;
   }
 
-  bool hasMatchedPackOnParmToNonPackOnArg() const {
-    return MatchedPackOnParmToNonPackOnArg;
-  }
-
-  void setMatchedPackOnParmToNonPackOnArg() {
-    MatchedPackOnParmToNonPackOnArg = true;
-  }
-
   /// Get the number of explicitly-specified arguments.
   unsigned getNumExplicitArgs() const {
     return ExplicitArgs;
diff --git a/clang/include/clang/Support/Compiler.h b/clang/include/clang/Support/Compiler.h
new file mode 100644
index 0000000000000000000000000000000000000000..c35815e106d2e2323f78e31840d2af3ae4dabf05
--- /dev/null
+++ b/clang/include/clang/Support/Compiler.h
@@ -0,0 +1,63 @@
+//===-- clang/Support/Compiler.h - Compiler abstraction support -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines explicit visibility macros used to export symbols from
+// clang-cpp
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_SUPPORT_COMPILER_H
+#define CLANG_SUPPORT_COMPILER_H
+
+#include "llvm/Support/Compiler.h"
+
+/// CLANG_ABI is the main export/visibility macro to mark something as
+/// explicitly exported when clang is built as a shared library with everything
+/// else that is unannotated having hidden visibility.
+///
+/// CLANG_EXPORT_TEMPLATE is used on explicit template instantiations in source
+/// files that were declared extern in a header. This macro is only set as a
+/// compiler export attribute on windows, on other platforms it does nothing.
+///
+/// CLANG_TEMPLATE_ABI is for annotating extern template declarations in headers
+/// for both functions and classes. On windows its turned in to dllimport for
+/// library consumers, for other platforms its a default visibility attribute.
+#ifndef CLANG_ABI_GENERATING_ANNOTATIONS
+// Marker to add to classes or functions in public headers that should not have
+// export macros added to them by the clang tool
+#define CLANG_ABI_NOT_EXPORTED
+// Some libraries like those for tablegen are linked in to tools that used
+// in the build so can't depend on the llvm shared library. If export macros
+// were left enabled when building these we would get duplicate or
+// missing symbol linker errors on windows.
+#if defined(CLANG_BUILD_STATIC)
+#define CLANG_ABI
+#define CLANG_TEMPLATE_ABI
+#define CLANG_EXPORT_TEMPLATE
+#elif defined(_WIN32) && !defined(__MINGW32__)
+#if defined(CLANG_EXPORTS)
+#define CLANG_ABI __declspec(dllexport)
+#define CLANG_TEMPLATE_ABI
+#define CLANG_EXPORT_TEMPLATE __declspec(dllexport)
+#else
+#define CLANG_ABI __declspec(dllimport)
+#define CLANG_TEMPLATE_ABI __declspec(dllimport)
+#define CLANG_EXPORT_TEMPLATE
+#endif
+#elif defined(__ELF__) || defined(__MINGW32__) || defined(_AIX)
+#define CLANG_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
+#define CLANG_TEMPLATE_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
+#define CLANG_EXPORT_TEMPLATE
+#elif defined(__MACH__) || defined(__WASM__)
+#define CLANG_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
+#define CLANG_TEMPLATE_ABI
+#define CLANG_EXPORT_TEMPLATE
+#endif
+#endif
+
+#endif
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 0a3b38b0dc6e5730889b558191bbbabcb10e321a..8ca63bf64aa0ef996c7134d6eaacdb518b690bc3 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -2869,6 +2869,11 @@ bool Compiler<Emitter>::VisitPredefinedExpr(const PredefinedExpr *E) {
   if (DiscardResult)
     return true;
 
+  if (!Initializing) {
+    unsigned StringIndex = P.createGlobalString(E->getFunctionName(), E);
+    return this->emitGetPtrGlobal(StringIndex, E);
+  }
+
   return this->delegate(E->getFunctionName());
 }
 
@@ -3411,6 +3416,9 @@ bool Compiler<Emitter>::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
 
 template <class Emitter>
 bool Compiler<Emitter>::VisitBlockExpr(const BlockExpr *E) {
+  if (DiscardResult)
+    return true;
+
   const Function *Func = nullptr;
   if (auto F = Compiler<ByteCodeEmitter>(Ctx, P).compileObjCBlock(E))
     Func = F;
@@ -3537,8 +3545,8 @@ bool Compiler<Emitter>::VisitConvertVectorExpr(const ConvertVectorExpr *E) {
   QualType ElemType = VT->getElementType();
   PrimType ElemT = classifyPrim(ElemType);
   const Expr *Src = E->getSrcExpr();
-  PrimType SrcElemT =
-      classifyPrim(Src->getType()->castAs<VectorType>()->getElementType());
+  QualType SrcType = Src->getType();
+  PrimType SrcElemT = classifyVectorElementType(SrcType);
 
   unsigned SrcOffset = this->allocateLocalPrimitive(Src, PT_Ptr, true, false);
   if (!this->visit(Src))
@@ -3551,9 +3559,15 @@ bool Compiler<Emitter>::VisitConvertVectorExpr(const ConvertVectorExpr *E) {
       return false;
     if (!this->emitArrayElemPop(SrcElemT, I, E))
       return false;
+
+    // Cast to the desired result element type.
     if (SrcElemT != ElemT) {
       if (!this->emitPrimCast(SrcElemT, ElemT, ElemType, E))
         return false;
+    } else if (ElemType->isFloatingType() && SrcType != ElemType) {
+      const auto *TargetSemantics = &Ctx.getFloatSemantics(ElemType);
+      if (!this->emitCastFP(TargetSemantics, getRoundingMode(E), E))
+        return false;
     }
     if (!this->emitInitElem(ElemT, I, E))
       return false;
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 74e9e1cf62937298009d6728c30548577c6cf0e0..65c7b4e5306d7203c284a5e1d3e0ee5cb74fae50 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -38,7 +38,6 @@ static T getParam(const InterpFrame *Frame, unsigned Index) {
   return Frame->getParam<T>(Offset);
 }
 
-// static APSInt getAPSIntParam(InterpStack &Stk, size_t Offset = 0) {
 static APSInt getAPSIntParam(const InterpFrame *Frame, unsigned Index) {
   APSInt R;
   unsigned Offset = Frame->getFunction()->getParamOffset(Index);
@@ -334,39 +333,48 @@ static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
 }
 
 static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
-                                 const InterpFrame *Frame, const Function *F) {
+                                 const InterpFrame *Frame, const Function *F,
+                                 bool IsNumBuiltin) {
   const Floating &LHS = getParam<Floating>(Frame, 0);
   const Floating &RHS = getParam<Floating>(Frame, 1);
 
   Floating Result;
 
-  // When comparing zeroes, return -0.0 if one of the zeroes is negative.
-  if (LHS.isZero() && RHS.isZero() && RHS.isNegative())
-    Result = RHS;
-  else if (LHS.isNan() || RHS < LHS)
-    Result = RHS;
-  else
-    Result = LHS;
+  if (IsNumBuiltin) {
+    Result = llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat());
+  } else {
+    // When comparing zeroes, return -0.0 if one of the zeroes is negative.
+    if (LHS.isZero() && RHS.isZero() && RHS.isNegative())
+      Result = RHS;
+    else if (LHS.isNan() || RHS < LHS)
+      Result = RHS;
+    else
+      Result = LHS;
+  }
 
   S.Stk.push<Floating>(Result);
   return true;
 }
 
 static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
-                                 const InterpFrame *Frame,
-                                 const Function *Func) {
+                                 const InterpFrame *Frame, const Function *Func,
+                                 bool IsNumBuiltin) {
   const Floating &LHS = getParam<Floating>(Frame, 0);
   const Floating &RHS = getParam<Floating>(Frame, 1);
 
   Floating Result;
 
-  // When comparing zeroes, return +0.0 if one of the zeroes is positive.
-  if (LHS.isZero() && RHS.isZero() && LHS.isNegative())
-    Result = RHS;
-  else if (LHS.isNan() || RHS > LHS)
-    Result = RHS;
-  else
-    Result = LHS;
+  if (IsNumBuiltin) {
+    Result = llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat());
+  } else {
+    // When comparing zeroes, return +0.0 if one of the zeroes is positive.
+    if (LHS.isZero() && RHS.isZero() && LHS.isNegative())
+      Result = RHS;
+    else if (LHS.isNan() || RHS > LHS)
+      Result = RHS;
+    else
+      Result = LHS;
+  }
 
   S.Stk.push<Floating>(Result);
   return true;
@@ -1162,6 +1170,71 @@ static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
   return false;
 }
 
+/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
+static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
+                                           const InterpFrame *Frame,
+                                           const Function *Func,
+                                           const CallExpr *Call) {
+  assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
+
+  // Might be called with function pointers in C.
+  std::optional<PrimType> PtrT = S.Ctx.classify(Call->getArg(0));
+  if (PtrT != PT_Ptr)
+    return false;
+
+  unsigned ArgSize = callArgSize(S, Call);
+  const Pointer &Ptr = S.Stk.peek<Pointer>(ArgSize);
+  std::optional<APSInt> ExtraOffset;
+  APSInt Alignment;
+  if (Call->getNumArgs() == 2) {
+    Alignment = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)));
+  } else {
+    PrimType AlignmentT = *S.Ctx.classify(Call->getArg(1));
+    PrimType ExtraOffsetT = *S.Ctx.classify(Call->getArg(2));
+    Alignment = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)),
+                             align(primSize(AlignmentT)) +
+                                 align(primSize(ExtraOffsetT)));
+    ExtraOffset = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(2)));
+  }
+
+  CharUnits Align = CharUnits::fromQuantity(Alignment.getZExtValue());
+
+  // If there is a base object, then it must have the correct alignment.
+  if (Ptr.isBlockPointer()) {
+    CharUnits BaseAlignment;
+    if (const auto *VD = Ptr.getDeclDesc()->asValueDecl())
+      BaseAlignment = S.getASTContext().getDeclAlign(VD);
+    else if (const auto *E = Ptr.getDeclDesc()->asExpr())
+      BaseAlignment = GetAlignOfExpr(S.getASTContext(), E, UETT_AlignOf);
+
+    if (BaseAlignment < Align) {
+      S.CCEDiag(Call->getArg(0),
+                diag::note_constexpr_baa_insufficient_alignment)
+          << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
+      return false;
+    }
+  }
+
+  APValue AV = Ptr.toAPValue(S.getASTContext());
+  CharUnits AVOffset = AV.getLValueOffset();
+  if (ExtraOffset)
+    AVOffset -= CharUnits::fromQuantity(ExtraOffset->getZExtValue());
+  if (AVOffset.alignTo(Align) != AVOffset) {
+    if (Ptr.isBlockPointer())
+      S.CCEDiag(Call->getArg(0),
+                diag::note_constexpr_baa_insufficient_alignment)
+          << 1 << AVOffset.getQuantity() << Align.getQuantity();
+    else
+      S.CCEDiag(Call->getArg(0),
+                diag::note_constexpr_baa_value_insufficient_alignment)
+          << AVOffset.getQuantity() << Align.getQuantity();
+    return false;
+  }
+
+  S.Stk.push<Pointer>(Ptr);
+  return true;
+}
+
 static bool interp__builtin_ia32_bextr(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const Function *Func,
@@ -1637,7 +1710,16 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
   case Builtin::BI__builtin_fminl:
   case Builtin::BI__builtin_fminf16:
   case Builtin::BI__builtin_fminf128:
-    if (!interp__builtin_fmin(S, OpPC, Frame, F))
+    if (!interp__builtin_fmin(S, OpPC, Frame, F, /*IsNumBuiltin=*/false))
+      return false;
+    break;
+
+  case Builtin::BI__builtin_fminimum_num:
+  case Builtin::BI__builtin_fminimum_numf:
+  case Builtin::BI__builtin_fminimum_numl:
+  case Builtin::BI__builtin_fminimum_numf16:
+  case Builtin::BI__builtin_fminimum_numf128:
+    if (!interp__builtin_fmin(S, OpPC, Frame, F, /*IsNumBuiltin=*/true))
       return false;
     break;
 
@@ -1646,7 +1728,16 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
   case Builtin::BI__builtin_fmaxl:
   case Builtin::BI__builtin_fmaxf16:
   case Builtin::BI__builtin_fmaxf128:
-    if (!interp__builtin_fmax(S, OpPC, Frame, F))
+    if (!interp__builtin_fmax(S, OpPC, Frame, F, /*IsNumBuiltin=*/false))
+      return false;
+    break;
+
+  case Builtin::BI__builtin_fmaximum_num:
+  case Builtin::BI__builtin_fmaximum_numf:
+  case Builtin::BI__builtin_fmaximum_numl:
+  case Builtin::BI__builtin_fmaximum_numf16:
+  case Builtin::BI__builtin_fmaximum_numf128:
+    if (!interp__builtin_fmax(S, OpPC, Frame, F, /*IsNumBuiltin=*/true))
       return false;
     break;
 
@@ -1905,6 +1996,11 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
       return false;
     break;
 
+  case Builtin::BI__builtin_assume_aligned:
+    if (!interp__builtin_assume_aligned(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
   case clang::X86::BI__builtin_ia32_bextr_u32:
   case clang::X86::BI__builtin_ia32_bextr_u64:
   case clang::X86::BI__builtin_ia32_bextri_u32:
diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
index 23245a66b578ae45696975a4ffdf0c3892ff122d..cd2665f755d7cbaf9c3e84240760c3345e4e66df 100644
--- a/clang/lib/AST/ByteCode/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -33,7 +33,7 @@ const void *Program::getNativePointer(unsigned Idx) {
   return NativePointers[Idx];
 }
 
-unsigned Program::createGlobalString(const StringLiteral *S) {
+unsigned Program::createGlobalString(const StringLiteral *S, const Expr *Base) {
   const size_t CharWidth = S->getCharByteWidth();
   const size_t BitWidth = CharWidth * Ctx.getCharBit();
 
@@ -52,12 +52,15 @@ unsigned Program::createGlobalString(const StringLiteral *S) {
     llvm_unreachable("unsupported character width");
   }
 
+  if (!Base)
+    Base = S;
+
   // Create a descriptor for the string.
-  Descriptor *Desc =
-      allocateDescriptor(S, CharType, Descriptor::GlobalMD, S->getLength() + 1,
-                         /*isConst=*/true,
-                         /*isTemporary=*/false,
-                         /*isMutable=*/false);
+  Descriptor *Desc = allocateDescriptor(Base, CharType, Descriptor::GlobalMD,
+                                        S->getLength() + 1,
+                                        /*isConst=*/true,
+                                        /*isTemporary=*/false,
+                                        /*isMutable=*/false);
 
   // Allocate storage for the string.
   // The byte length does not include the null terminator.
diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h
index be84c40714a60bca8479938134bfb5ea741a6ed2..f676672fb7ced5a6824dd8ca80db72b53057f5cb 100644
--- a/clang/lib/AST/ByteCode/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -64,7 +64,8 @@ public:
   const void *getNativePointer(unsigned Idx);
 
   /// Emits a string literal among global data.
-  unsigned createGlobalString(const StringLiteral *S);
+  unsigned createGlobalString(const StringLiteral *S,
+                              const Expr *Base = nullptr);
 
   /// Returns a pointer to a global.
   Pointer getPtrGlobal(unsigned Idx) const;
diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index 8c79df8317a2ec18d0bf2a6e85fba1bb62078c57..e37ebec0851951087f1e960ad1739102ecb0e2a1 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -252,10 +252,13 @@ ExprDependence clang::computeDependence(ExtVectorElementExpr *E) {
   return E->getBase()->getDependence();
 }
 
-ExprDependence clang::computeDependence(BlockExpr *E) {
+ExprDependence clang::computeDependence(BlockExpr *E,
+                                        bool ContainsUnexpandedParameterPack) {
   auto D = toExprDependenceForImpliedType(E->getType()->getDependence());
   if (E->getBlockDecl()->isDependentContext())
     D |= ExprDependence::Instantiation;
+  if (ContainsUnexpandedParameterPack)
+    D |= ExprDependence::UnexpandedPack;
   return D;
 }
 
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index a70a5efb8ca01fef1fb90b4381978236eed9ae9f..8affa7cf619f9fcc07b89d2242e9e5a791be887a 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -1737,9 +1737,17 @@ void NamedDecl::printNestedNameSpecifier(raw_ostream &OS,
       continue;
 
     // Suppress inline namespace if it doesn't make the result ambiguous.
-    if (P.SuppressInlineNamespace && Ctx->isInlineNamespace() && NameInScope &&
-        cast<NamespaceDecl>(Ctx)->isRedundantInlineQualifierFor(NameInScope))
-      continue;
+    if (Ctx->isInlineNamespace() && NameInScope) {
+      bool isRedundant =
+          cast<NamespaceDecl>(Ctx)->isRedundantInlineQualifierFor(NameInScope);
+      if (P.SuppressInlineNamespace ==
+              PrintingPolicy::SuppressInlineNamespaceMode::All ||
+          (P.SuppressInlineNamespace ==
+               PrintingPolicy::SuppressInlineNamespaceMode::Redundant &&
+           isRedundant)) {
+        continue;
+      }
+    }
 
     // Skip non-named contexts such as linkage specifications and ExportDecls.
     const NamedDecl *ND = dyn_cast<NamedDecl>(Ctx);
@@ -2696,21 +2704,21 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
     if (isTemplateInstantiation(VDTemplSpec->getTemplateSpecializationKind())) {
       auto From = VDTemplSpec->getInstantiatedFrom();
       if (auto *VTD = From.dyn_cast<VarTemplateDecl *>()) {
-        while (!VTD->isMemberSpecialization()) {
-          auto *NewVTD = VTD->getInstantiatedFromMemberTemplate();
-          if (!NewVTD)
+        while (!VTD->hasMemberSpecialization()) {
+          if (auto *NewVTD = VTD->getInstantiatedFromMemberTemplate())
+            VTD = NewVTD;
+          else
             break;
-          VTD = NewVTD;
         }
         return getDefinitionOrSelf(VTD->getTemplatedDecl());
       }
       if (auto *VTPSD =
               From.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
-        while (!VTPSD->isMemberSpecialization()) {
-          auto *NewVTPSD = VTPSD->getInstantiatedFromMember();
-          if (!NewVTPSD)
+        while (!VTPSD->hasMemberSpecialization()) {
+          if (auto *NewVTPSD = VTPSD->getInstantiatedFromMember())
+            VTPSD = NewVTPSD;
+          else
             break;
-          VTPSD = NewVTPSD;
         }
         return getDefinitionOrSelf<VarDecl>(VTPSD);
       }
@@ -2719,15 +2727,14 @@ VarDecl *VarDecl::getTemplateInstantiationPattern() const {
 
   // If this is the pattern of a variable template, find where it was
   // instantiated from. FIXME: Is this necessary?
-  if (VarTemplateDecl *VarTemplate = VD->getDescribedVarTemplate()) {
-    while (!VarTemplate->isMemberSpecialization()) {
-      auto *NewVT = VarTemplate->getInstantiatedFromMemberTemplate();
-      if (!NewVT)
+  if (VarTemplateDecl *VTD = VD->getDescribedVarTemplate()) {
+    while (!VTD->hasMemberSpecialization()) {
+      if (auto *NewVTD = VTD->getInstantiatedFromMemberTemplate())
+        VTD = NewVTD;
+      else
         break;
-      VarTemplate = NewVT;
     }
-
-    return getDefinitionOrSelf(VarTemplate->getTemplatedDecl());
+    return getDefinitionOrSelf(VTD->getTemplatedDecl());
   }
 
   if (VD == this)
@@ -4142,11 +4149,11 @@ FunctionDecl::getTemplateInstantiationPattern(bool ForDefinition) const {
   if (FunctionTemplateDecl *Primary = getPrimaryTemplate()) {
     // If we hit a point where the user provided a specialization of this
     // template, we're done looking.
-    while (!ForDefinition || !Primary->isMemberSpecialization()) {
-      auto *NewPrimary = Primary->getInstantiatedFromMemberTemplate();
-      if (!NewPrimary)
+    while (!ForDefinition || !Primary->hasMemberSpecialization()) {
+      if (auto *NewPrimary = Primary->getInstantiatedFromMemberTemplate())
+        Primary = NewPrimary;
+      else
         break;
-      Primary = NewPrimary;
     }
 
     return getDefinitionOrSelf(Primary->getTemplatedDecl());
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 1364ccc745ba01627b8c87e8125529aca11a0244..407ec14bbc00d554048f3b4880247b37c6308032 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -2023,19 +2023,21 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
   if (auto *TD = dyn_cast<ClassTemplateSpecializationDecl>(this)) {
     auto From = TD->getInstantiatedFrom();
     if (auto *CTD = From.dyn_cast<ClassTemplateDecl *>()) {
-      while (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate()) {
-        if (NewCTD->isMemberSpecialization())
+      while (!CTD->hasMemberSpecialization()) {
+        if (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate())
+          CTD = NewCTD;
+        else
           break;
-        CTD = NewCTD;
       }
       return GetDefinitionOrSelf(CTD->getTemplatedDecl());
     }
     if (auto *CTPSD =
             From.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-      while (auto *NewCTPSD = CTPSD->getInstantiatedFromMember()) {
-        if (NewCTPSD->isMemberSpecialization())
+      while (!CTPSD->hasMemberSpecialization()) {
+        if (auto *NewCTPSD = CTPSD->getInstantiatedFromMemberTemplate())
+          CTPSD = NewCTPSD;
+        else
           break;
-        CTPSD = NewCTPSD;
       }
       return GetDefinitionOrSelf(CTPSD);
     }
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 6fe817c5ef1c6b44b33465fc607c7b3466386529..d9b67b7bedf5a5d9a800ffbc4752b0d47e877da9 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -309,16 +309,16 @@ bool TemplateDecl::isTypeAlias() const {
 void RedeclarableTemplateDecl::anchor() {}
 
 RedeclarableTemplateDecl::CommonBase *RedeclarableTemplateDecl::getCommonPtr() const {
-  if (Common)
-    return Common;
+  if (CommonBase *C = getCommonPtrInternal())
+    return C;
 
   // Walk the previous-declaration chain until we either find a declaration
   // with a common pointer or we run out of previous declarations.
   SmallVector<const RedeclarableTemplateDecl *, 2> PrevDecls;
   for (const RedeclarableTemplateDecl *Prev = getPreviousDecl(); Prev;
        Prev = Prev->getPreviousDecl()) {
-    if (Prev->Common) {
-      Common = Prev->Common;
+    if (CommonBase *C = Prev->getCommonPtrInternal()) {
+      setCommonPtr(C);
       break;
     }
 
@@ -326,18 +326,18 @@ RedeclarableTemplateDecl::CommonBase *RedeclarableTemplateDecl::getCommonPtr() c
   }
 
   // If we never found a common pointer, allocate one now.
-  if (!Common) {
+  if (!getCommonPtrInternal()) {
     // FIXME: If any of the declarations is from an AST file, we probably
     // need an update record to add the common data.
 
-    Common = newCommon(getASTContext());
+    setCommonPtr(newCommon(getASTContext()));
   }
 
   // Update any previous declarations we saw with the common pointer.
   for (const RedeclarableTemplateDecl *Prev : PrevDecls)
-    Prev->Common = Common;
+    Prev->setCommonPtr(getCommonPtrInternal());
 
-  return Common;
+  return getCommonPtrInternal();
 }
 
 void RedeclarableTemplateDecl::loadLazySpecializationsImpl() const {
@@ -463,19 +463,17 @@ void FunctionTemplateDecl::addSpecialization(
 }
 
 void FunctionTemplateDecl::mergePrevDecl(FunctionTemplateDecl *Prev) {
-  using Base = RedeclarableTemplateDecl;
-
   // If we haven't created a common pointer yet, then it can just be created
   // with the usual method.
-  if (!Base::Common)
+  if (!getCommonPtrInternal())
     return;
 
-  Common *ThisCommon = static_cast<Common *>(Base::Common);
+  Common *ThisCommon = static_cast<Common *>(getCommonPtrInternal());
   Common *PrevCommon = nullptr;
   SmallVector<FunctionTemplateDecl *, 8> PreviousDecls;
   for (; Prev; Prev = Prev->getPreviousDecl()) {
-    if (Prev->Base::Common) {
-      PrevCommon = static_cast<Common *>(Prev->Base::Common);
+    if (CommonBase *C = Prev->getCommonPtrInternal()) {
+      PrevCommon = static_cast<Common *>(C);
       break;
     }
     PreviousDecls.push_back(Prev);
@@ -485,7 +483,7 @@ void FunctionTemplateDecl::mergePrevDecl(FunctionTemplateDecl *Prev) {
   // use this common pointer.
   if (!PrevCommon) {
     for (auto *D : PreviousDecls)
-      D->Base::Common = ThisCommon;
+      D->setCommonPtr(ThisCommon);
     return;
   }
 
@@ -493,7 +491,7 @@ void FunctionTemplateDecl::mergePrevDecl(FunctionTemplateDecl *Prev) {
   assert(ThisCommon->Specializations.size() == 0 &&
          "Can't merge incompatible declarations!");
 
-  Base::Common = PrevCommon;
+  setCommonPtr(PrevCommon);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/AST/ExprConstShared.h b/clang/lib/AST/ExprConstShared.h
index efe8ee986d29b3da7915bae6c320889b14c5fd47..401ae629c86bfdedef4fa3be1860313c29d84fdd 100644
--- a/clang/lib/AST/ExprConstShared.h
+++ b/clang/lib/AST/ExprConstShared.h
@@ -14,12 +14,17 @@
 #ifndef LLVM_CLANG_LIB_AST_EXPRCONSTSHARED_H
 #define LLVM_CLANG_LIB_AST_EXPRCONSTSHARED_H
 
+#include "clang/Basic/TypeTraits.h"
+
 namespace llvm {
 class APFloat;
 }
 namespace clang {
 class QualType;
 class LangOptions;
+class ASTContext;
+class CharUnits;
+class Expr;
 } // namespace clang
 using namespace clang;
 /// Values returned by __builtin_classify_type, chosen to match the values
@@ -66,4 +71,7 @@ void HandleComplexComplexDiv(llvm::APFloat A, llvm::APFloat B, llvm::APFloat C,
                              llvm::APFloat D, llvm::APFloat &ResR,
                              llvm::APFloat &ResI);
 
+CharUnits GetAlignOfExpr(const ASTContext &Ctx, const Expr *E,
+                         UnaryExprOrTypeTrait ExprKind);
+
 #endif
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 4d5af96093cfeb684f3fb61535de239e5a7d4299..51956c631786b533c7ca87c9a4aa01cafbebdf7e 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -9620,7 +9620,7 @@ bool PointerExprEvaluator::VisitCastExpr(const CastExpr *E) {
   return ExprEvaluatorBaseTy::VisitCastExpr(E);
 }
 
-static CharUnits GetAlignOfType(EvalInfo &Info, QualType T,
+static CharUnits GetAlignOfType(const ASTContext &Ctx, QualType T,
                                 UnaryExprOrTypeTrait ExprKind) {
   // C++ [expr.alignof]p3:
   //     When alignof is applied to a reference type, the result is the
@@ -9631,23 +9631,22 @@ static CharUnits GetAlignOfType(EvalInfo &Info, QualType T,
     return CharUnits::One();
 
   const bool AlignOfReturnsPreferred =
-      Info.Ctx.getLangOpts().getClangABICompat() <= LangOptions::ClangABI::Ver7;
+      Ctx.getLangOpts().getClangABICompat() <= LangOptions::ClangABI::Ver7;
 
   // __alignof is defined to return the preferred alignment.
   // Before 8, clang returned the preferred alignment for alignof and _Alignof
   // as well.
   if (ExprKind == UETT_PreferredAlignOf || AlignOfReturnsPreferred)
-    return Info.Ctx.toCharUnitsFromBits(
-      Info.Ctx.getPreferredTypeAlign(T.getTypePtr()));
+    return Ctx.toCharUnitsFromBits(Ctx.getPreferredTypeAlign(T.getTypePtr()));
   // alignof and _Alignof are defined to return the ABI alignment.
   else if (ExprKind == UETT_AlignOf)
-    return Info.Ctx.getTypeAlignInChars(T.getTypePtr());
+    return Ctx.getTypeAlignInChars(T.getTypePtr());
   else
     llvm_unreachable("GetAlignOfType on a non-alignment ExprKind");
 }
 
-static CharUnits GetAlignOfExpr(EvalInfo &Info, const Expr *E,
-                                UnaryExprOrTypeTrait ExprKind) {
+CharUnits GetAlignOfExpr(const ASTContext &Ctx, const Expr *E,
+                         UnaryExprOrTypeTrait ExprKind) {
   E = E->IgnoreParens();
 
   // The kinds of expressions that we have special-case logic here for
@@ -9657,22 +9656,22 @@ static CharUnits GetAlignOfExpr(EvalInfo &Info, const Expr *E,
   // alignof decl is always accepted, even if it doesn't make sense: we default
   // to 1 in those cases.
   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-    return Info.Ctx.getDeclAlign(DRE->getDecl(),
-                                 /*RefAsPointee*/true);
+    return Ctx.getDeclAlign(DRE->getDecl(),
+                            /*RefAsPointee*/ true);
 
   if (const MemberExpr *ME = dyn_cast<MemberExpr>(E))
-    return Info.Ctx.getDeclAlign(ME->getMemberDecl(),
-                                 /*RefAsPointee*/true);
+    return Ctx.getDeclAlign(ME->getMemberDecl(),
+                            /*RefAsPointee*/ true);
 
-  return GetAlignOfType(Info, E->getType(), ExprKind);
+  return GetAlignOfType(Ctx, E->getType(), ExprKind);
 }
 
 static CharUnits getBaseAlignment(EvalInfo &Info, const LValue &Value) {
   if (const auto *VD = Value.Base.dyn_cast<const ValueDecl *>())
     return Info.Ctx.getDeclAlign(VD);
   if (const auto *E = Value.Base.dyn_cast<const Expr *>())
-    return GetAlignOfExpr(Info, E, UETT_AlignOf);
-  return GetAlignOfType(Info, Value.Base.getTypeInfoType(), UETT_AlignOf);
+    return GetAlignOfExpr(Info.Ctx, E, UETT_AlignOf);
+  return GetAlignOfType(Info.Ctx, Value.Base.getTypeInfoType(), UETT_AlignOf);
 }
 
 /// Evaluate the value of the alignment argument to __builtin_align_{up,down},
@@ -9768,11 +9767,8 @@ bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
 
       if (BaseAlignment < Align) {
         Result.Designator.setInvalid();
-        // FIXME: Add support to Diagnostic for long / long long.
-        CCEDiag(E->getArg(0),
-                diag::note_constexpr_baa_insufficient_alignment) << 0
-          << (unsigned)BaseAlignment.getQuantity()
-          << (unsigned)Align.getQuantity();
+        CCEDiag(E->getArg(0), diag::note_constexpr_baa_insufficient_alignment)
+            << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
         return false;
       }
     }
@@ -9783,11 +9779,11 @@ bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
 
       (OffsetResult.Base
            ? CCEDiag(E->getArg(0),
-                     diag::note_constexpr_baa_insufficient_alignment) << 1
+                     diag::note_constexpr_baa_insufficient_alignment)
+                 << 1
            : CCEDiag(E->getArg(0),
                      diag::note_constexpr_baa_value_insufficient_alignment))
-        << (int)OffsetResult.Offset.getQuantity()
-        << (unsigned)Align.getQuantity();
+          << OffsetResult.Offset.getQuantity() << Align.getQuantity();
       return false;
     }
 
@@ -14478,11 +14474,11 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr(
   case UETT_PreferredAlignOf:
   case UETT_AlignOf: {
     if (E->isArgumentType())
-      return Success(GetAlignOfType(Info, E->getArgumentType(), E->getKind()),
-                     E);
+      return Success(
+          GetAlignOfType(Info.Ctx, E->getArgumentType(), E->getKind()), E);
     else
-      return Success(GetAlignOfExpr(Info, E->getArgumentExpr(), E->getKind()),
-                     E);
+      return Success(
+          GetAlignOfExpr(Info.Ctx, E->getArgumentExpr(), E->getKind()), E);
   }
 
   case UETT_PtrAuthTypeDiscriminator: {
@@ -15314,6 +15310,32 @@ bool FloatExprEvaluator::VisitCallExpr(const CallExpr *E) {
       Result = RHS;
     return true;
   }
+
+  case Builtin::BI__builtin_fmaximum_num:
+  case Builtin::BI__builtin_fmaximum_numf:
+  case Builtin::BI__builtin_fmaximum_numl:
+  case Builtin::BI__builtin_fmaximum_numf16:
+  case Builtin::BI__builtin_fmaximum_numf128: {
+    APFloat RHS(0.);
+    if (!EvaluateFloat(E->getArg(0), Result, Info) ||
+        !EvaluateFloat(E->getArg(1), RHS, Info))
+      return false;
+    Result = maximumnum(Result, RHS);
+    return true;
+  }
+
+  case Builtin::BI__builtin_fminimum_num:
+  case Builtin::BI__builtin_fminimum_numf:
+  case Builtin::BI__builtin_fminimum_numl:
+  case Builtin::BI__builtin_fminimum_numf16:
+  case Builtin::BI__builtin_fminimum_numf128: {
+    APFloat RHS(0.);
+    if (!EvaluateFloat(E->getArg(0), Result, Info) ||
+        !EvaluateFloat(E->getArg(1), RHS, Info))
+      return false;
+    Result = minimumnum(Result, RHS);
+    return true;
+  }
   }
 }
 
diff --git a/clang/lib/AST/InheritViz.cpp b/clang/lib/AST/InheritViz.cpp
index 2ed0ce1c79c5bd37de6097d02178a3487e49ff58..1dafed837a35603f55f12a7572ae6a336bbba39b 100644
--- a/clang/lib/AST/InheritViz.cpp
+++ b/clang/lib/AST/InheritViz.cpp
@@ -63,12 +63,11 @@ void InheritanceHierarchyWriter::WriteNode(QualType Type, bool FromVirtual) {
   QualType CanonType = Context.getCanonicalType(Type);
 
   if (FromVirtual) {
-    if (KnownVirtualBases.find(CanonType) != KnownVirtualBases.end())
+    if (!KnownVirtualBases.insert(CanonType).second)
       return;
 
     // We haven't seen this virtual base before, so display it and
     // its bases.
-    KnownVirtualBases.insert(CanonType);
   }
 
   // Declare the node itself.
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index 0b34ed6189593e8d29b2335efdaaf955f587f943..638252fd811f1de33758c4724e498ac577be0ff4 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -26,7 +26,7 @@ bool OpenACCClauseWithExprs::classof(const OpenACCClause *C) {
   return OpenACCWaitClause::classof(C) || OpenACCNumGangsClause::classof(C) ||
          OpenACCTileClause::classof(C) ||
          OpenACCClauseWithSingleIntExpr::classof(C) ||
-         OpenACCClauseWithVarList::classof(C);
+         OpenACCGangClause::classof(C) || OpenACCClauseWithVarList::classof(C);
 }
 bool OpenACCClauseWithVarList::classof(const OpenACCClause *C) {
   return OpenACCPrivateClause::classof(C) ||
@@ -44,7 +44,8 @@ bool OpenACCClauseWithCondition::classof(const OpenACCClause *C) {
 bool OpenACCClauseWithSingleIntExpr::classof(const OpenACCClause *C) {
   return OpenACCNumWorkersClause::classof(C) ||
          OpenACCVectorLengthClause::classof(C) ||
-         OpenACCCollapseClause::classof(C) || OpenACCAsyncClause::classof(C);
+         OpenACCWorkerClause::classof(C) || OpenACCCollapseClause::classof(C) ||
+         OpenACCAsyncClause::classof(C);
 }
 OpenACCDefaultClause *OpenACCDefaultClause::Create(const ASTContext &C,
                                                    OpenACCDefaultClauseKind K,
@@ -125,6 +126,21 @@ OpenACCNumWorkersClause::OpenACCNumWorkersClause(SourceLocation BeginLoc,
          "Condition expression type not scalar/dependent");
 }
 
+OpenACCGangClause::OpenACCGangClause(SourceLocation BeginLoc,
+                                     SourceLocation LParenLoc,
+                                     ArrayRef<OpenACCGangKind> GangKinds,
+                                     ArrayRef<Expr *> IntExprs,
+                                     SourceLocation EndLoc)
+    : OpenACCClauseWithExprs(OpenACCClauseKind::Gang, BeginLoc, LParenLoc,
+                             EndLoc) {
+  assert(GangKinds.size() == IntExprs.size() && "Mismatch exprs/kind?");
+  std::uninitialized_copy(IntExprs.begin(), IntExprs.end(),
+                          getTrailingObjects<Expr *>());
+  setExprs(MutableArrayRef(getTrailingObjects<Expr *>(), IntExprs.size()));
+  std::uninitialized_copy(GangKinds.begin(), GangKinds.end(),
+                          getTrailingObjects<OpenACCGangKind>());
+}
+
 OpenACCNumWorkersClause *
 OpenACCNumWorkersClause::Create(const ASTContext &C, SourceLocation BeginLoc,
                                 SourceLocation LParenLoc, Expr *IntExpr,
@@ -376,18 +392,36 @@ OpenACCSeqClause *OpenACCSeqClause::Create(const ASTContext &C,
   return new (Mem) OpenACCSeqClause(BeginLoc, EndLoc);
 }
 
-OpenACCGangClause *OpenACCGangClause::Create(const ASTContext &C,
-                                             SourceLocation BeginLoc,
-                                             SourceLocation EndLoc) {
-  void *Mem = C.Allocate(sizeof(OpenACCGangClause));
-  return new (Mem) OpenACCGangClause(BeginLoc, EndLoc);
+OpenACCGangClause *
+OpenACCGangClause::Create(const ASTContext &C, SourceLocation BeginLoc,
+                          SourceLocation LParenLoc,
+                          ArrayRef<OpenACCGangKind> GangKinds,
+                          ArrayRef<Expr *> IntExprs, SourceLocation EndLoc) {
+  void *Mem =
+      C.Allocate(OpenACCGangClause::totalSizeToAlloc<Expr *, OpenACCGangKind>(
+          IntExprs.size(), GangKinds.size()));
+  return new (Mem)
+      OpenACCGangClause(BeginLoc, LParenLoc, GangKinds, IntExprs, EndLoc);
+}
+
+OpenACCWorkerClause::OpenACCWorkerClause(SourceLocation BeginLoc,
+                                         SourceLocation LParenLoc,
+                                         Expr *IntExpr, SourceLocation EndLoc)
+    : OpenACCClauseWithSingleIntExpr(OpenACCClauseKind::Worker, BeginLoc,
+                                     LParenLoc, IntExpr, EndLoc) {
+  assert((!IntExpr || IntExpr->isInstantiationDependent() ||
+          IntExpr->getType()->isIntegerType()) &&
+         "Int expression type not scalar/dependent");
 }
 
 OpenACCWorkerClause *OpenACCWorkerClause::Create(const ASTContext &C,
                                                  SourceLocation BeginLoc,
+                                                 SourceLocation LParenLoc,
+                                                 Expr *IntExpr,
                                                  SourceLocation EndLoc) {
-  void *Mem = C.Allocate(sizeof(OpenACCWorkerClause));
-  return new (Mem) OpenACCWorkerClause(BeginLoc, EndLoc);
+  void *Mem =
+      C.Allocate(sizeof(OpenACCWorkerClause), alignof(OpenACCWorkerClause));
+  return new (Mem) OpenACCWorkerClause(BeginLoc, LParenLoc, IntExpr, EndLoc);
 }
 
 OpenACCVectorClause *OpenACCVectorClause::Create(const ASTContext &C,
@@ -600,3 +634,31 @@ void OpenACCClausePrinter::VisitCollapseClause(const OpenACCCollapseClause &C) {
   printExpr(C.getLoopCount());
   OS << ")";
 }
+
+void OpenACCClausePrinter::VisitGangClause(const OpenACCGangClause &C) {
+  OS << "gang";
+
+  if (C.getNumExprs() > 0) {
+    OS << "(";
+    bool first = true;
+    for (unsigned I = 0; I < C.getNumExprs(); ++I) {
+      if (!first)
+        OS << ", ";
+      first = false;
+
+      OS << C.getExpr(I).first << ": ";
+      printExpr(C.getExpr(I).second);
+    }
+    OS << ")";
+  }
+}
+
+void OpenACCClausePrinter::VisitWorkerClause(const OpenACCWorkerClause &C) {
+  OS << "worker";
+
+  if (C.hasIntExpr()) {
+    OS << "(num: ";
+    printExpr(C.getIntExpr());
+    OS << ")";
+  }
+}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index ce9157f3003130559f926d53c7ca73a8eedc4174..ae36c27342b35085d63968754be392d1c5d6b2df 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2700,6 +2700,12 @@ void OpenACCClauseProfiler::VisitAsyncClause(const OpenACCAsyncClause &Clause) {
     Profiler.VisitStmt(Clause.getIntExpr());
 }
 
+void OpenACCClauseProfiler::VisitWorkerClause(
+    const OpenACCWorkerClause &Clause) {
+  if (Clause.hasIntExpr())
+    Profiler.VisitStmt(Clause.getIntExpr());
+}
+
 void OpenACCClauseProfiler::VisitWaitClause(const OpenACCWaitClause &Clause) {
   if (Clause.hasDevNumExpr())
     Profiler.VisitStmt(Clause.getDevNumExpr());
@@ -2717,6 +2723,12 @@ void OpenACCClauseProfiler::VisitIndependentClause(
 
 void OpenACCClauseProfiler::VisitSeqClause(const OpenACCSeqClause &Clause) {}
 
+void OpenACCClauseProfiler::VisitGangClause(const OpenACCGangClause &Clause) {
+  for (unsigned I = 0; I < Clause.getNumExprs(); ++I) {
+    Profiler.VisitStmt(Clause.getExpr(I).second);
+  }
+}
+
 void OpenACCClauseProfiler::VisitReductionClause(
     const OpenACCReductionClause &Clause) {
   for (auto *E : Clause.getVarList())
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 1f307d087431e388fccb7c3566c8ff897bbc6511..57d420f87ef31d8fd10f2ce08fded6b27fb46169 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -420,11 +420,23 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
     case OpenACCClauseKind::Self:
     case OpenACCClauseKind::Seq:
     case OpenACCClauseKind::Tile:
+    case OpenACCClauseKind::Worker:
     case OpenACCClauseKind::VectorLength:
       // The condition expression will be printed as a part of the 'children',
       // but print 'clause' here so it is clear what is happening from the dump.
       OS << " clause";
       break;
+    case OpenACCClauseKind::Gang: {
+      OS << " clause";
+      // print the list of all GangKinds, so that there is some sort of
+      // relationship to the expressions listed afterwards.
+      auto *GC = cast<OpenACCGangClause>(C);
+
+      for (unsigned I = 0; I < GC->getNumExprs(); ++I) {
+        OS << " " << GC->getExpr(I).first;
+      }
+      break;
+    }
     case OpenACCClauseKind::Collapse:
       OS << " clause";
       if (cast<OpenACCCollapseClause>(C)->hasForce())
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index b3e811b007e96e5487994c4ca0aa7c3b060f6ba7..164b5dcf615b85ad6d8acaac022bf95fdbf4c204 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1462,7 +1462,9 @@ void TypePrinter::AppendScope(DeclContext *DC, raw_ostream &OS,
 
     // Only suppress an inline namespace if the name has the same lookup
     // results in the enclosing namespace.
-    if (Policy.SuppressInlineNamespace && NS->isInline() && NameInScope &&
+    if (Policy.SuppressInlineNamespace !=
+            PrintingPolicy::SuppressInlineNamespaceMode::None &&
+        NS->isInline() && NameInScope &&
         NS->isRedundantInlineQualifierFor(NameInScope))
       return AppendScope(DC->getParent(), OS, NameInScope);
 
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 06309d327896b312ea21df40896ce9d762ec85ca..46dd44e6f2b24f4ffb33c606dffd9fb0b60580c4 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -655,7 +655,9 @@ bool HasNameMatcher::matchesNodeFullSlow(const NamedDecl &Node) const {
 
     PrintingPolicy Policy = Node.getASTContext().getPrintingPolicy();
     Policy.SuppressUnwrittenScope = SkipUnwritten;
-    Policy.SuppressInlineNamespace = SkipUnwritten;
+    Policy.SuppressInlineNamespace =
+        SkipUnwritten ? PrintingPolicy::SuppressInlineNamespaceMode::All
+                      : PrintingPolicy::SuppressInlineNamespaceMode::None;
     Node.printQualifiedName(OS, Policy);
 
     const StringRef FullName = OS.str();
diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp
index faf3878f064d20d70c7f2b7f1671f2690bc26f67..08989b6c2c03f98582993405e8fd48f91748eac8 100644
--- a/clang/lib/Basic/Cuda.cpp
+++ b/clang/lib/Basic/Cuda.cpp
@@ -43,6 +43,7 @@ static const CudaVersionMapEntry CudaNameVersionMap[] = {
     CUDA_ENTRY(12, 3),
     CUDA_ENTRY(12, 4),
     CUDA_ENTRY(12, 5),
+    CUDA_ENTRY(12, 6),
     {"", CudaVersion::NEW, llvm::VersionTuple(std::numeric_limits<int>::max())},
     {"unknown", CudaVersion::UNKNOWN, {}} // End of list tombstone.
 };
@@ -96,6 +97,7 @@ static const OffloadArchToStringMap arch_names[] = {
     SM(89),                          // Ada Lovelace
     SM(90),                          // Hopper
     SM(90a),                         // Hopper
+    SM(100),                         // Blackwell
     GFX(600),  // gfx600
     GFX(601),  // gfx601
     GFX(602),  // gfx602
@@ -221,6 +223,9 @@ CudaVersion MinVersionForOffloadArch(OffloadArch A) {
     return CudaVersion::CUDA_118;
   case OffloadArch::SM_90a:
     return CudaVersion::CUDA_120;
+  case OffloadArch::SM_100:
+    return CudaVersion::NEW; // TODO: use specific CUDA version once it's
+                             // public.
   default:
     llvm_unreachable("invalid enum");
   }
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index a9500a15c16ac4b0d82b6b27d566e5510d202d82..5c8ff377067801a82069504460d930c1dc19109e 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -409,6 +409,9 @@ ReservedLiteralSuffixIdStatus
 IdentifierInfo::isReservedLiteralSuffixId() const {
   StringRef Name = getName();
 
+  // Note: the diag::warn_deprecated_literal_operator_id diagnostic depends on
+  // this being the first check we do, so if this order changes, we have to fix
+  // that as well.
   if (Name[0] != '_')
     return ReservedLiteralSuffixIdStatus::NotStartsWithUnderscore;
 
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index c56b8d9a4485086fdbeafc72a08b0891ce6dd4f7..c87300bf2d60e0407851c395ef294baae1d476a5 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -313,7 +313,9 @@ ARMTargetInfo::ARMTargetInfo(const llvm::Triple &Triple,
     switch (Triple.getEnvironment()) {
     case llvm::Triple::Android:
     case llvm::Triple::GNUEABI:
+    case llvm::Triple::GNUEABIT64:
     case llvm::Triple::GNUEABIHF:
+    case llvm::Triple::GNUEABIHFT64:
     case llvm::Triple::MuslEABI:
     case llvm::Triple::MuslEABIHF:
     case llvm::Triple::OpenHOS:
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index 43b653dc52ce0d78b135c28f100a83f0d09c6bea..88a0dbde52d52baa130b11e82f360068ef44fe00 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -281,6 +281,8 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts,
       case OffloadArch::SM_90:
       case OffloadArch::SM_90a:
         return "900";
+      case OffloadArch::SM_100:
+        return "1000";
       }
       llvm_unreachable("unhandled OffloadArch");
     }();
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index a83d6464e789d6fcae38b025079cfa260b14d391..75f53e96ce28f6c572a283929cb2ebb0443af5bf 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -337,6 +337,10 @@ protected:
       Builder.defineMacro("_GNU_SOURCE");
     if (this->HasFloat128)
       Builder.defineMacro("__FLOAT128__");
+    if (Triple.isTime64ABI()) {
+      Builder.defineMacro("_FILE_OFFSET_BITS", "64");
+      Builder.defineMacro("_TIME_BITS", "64");
+    }
   }
 
 public:
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 672b87d6b43e5b571b20d499829984633b29951f..d939b373b9982e948247e52313aef08658deb49b 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -102,7 +102,7 @@ bool RISCVTargetInfo::validateAsmConstraint(
     return true;
   case 'v':
     // A vector register.
-    if (Name[1] == 'r' || Name[1] == 'm') {
+    if (Name[1] == 'r' || Name[1] == 'd' || Name[1] == 'm') {
       Info.setAllowsRegister();
       Name += 1;
       return true;
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index db50913c753af5598a2f934affb6096f422fb1a8..c67ee3c581963fcca78b707c00b13a3a15ee17d7 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2871,6 +2871,28 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                    Intrinsic::minnum,
                                    Intrinsic::experimental_constrained_minnum));
 
+    case Builtin::BIfmaximum_num:
+    case Builtin::BIfmaximum_numf:
+    case Builtin::BIfmaximum_numl:
+    case Builtin::BI__builtin_fmaximum_num:
+    case Builtin::BI__builtin_fmaximum_numf:
+    case Builtin::BI__builtin_fmaximum_numf16:
+    case Builtin::BI__builtin_fmaximum_numl:
+    case Builtin::BI__builtin_fmaximum_numf128:
+      return RValue::get(
+          emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::maximumnum));
+
+    case Builtin::BIfminimum_num:
+    case Builtin::BIfminimum_numf:
+    case Builtin::BIfminimum_numl:
+    case Builtin::BI__builtin_fminimum_num:
+    case Builtin::BI__builtin_fminimum_numf:
+    case Builtin::BI__builtin_fminimum_numf16:
+    case Builtin::BI__builtin_fminimum_numl:
+    case Builtin::BI__builtin_fminimum_numf128:
+      return RValue::get(
+          emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::minimumnum));
+
     // fmod() is a special-case. It maps to the frem instruction rather than an
     // LLVM intrinsic.
     case Builtin::BIfmod:
@@ -13650,7 +13672,7 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
     Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
 
     // Built the IR for the preserve_field_info intrinsic.
-    llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
+    llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getOrInsertDeclaration(
         &CGM.getModule(), llvm::Intrinsic::bpf_preserve_field_info,
         {FieldAddr->getType()});
     return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
@@ -13672,10 +13694,10 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
 
     llvm::Function *FnDecl;
     if (BuiltinID == BPF::BI__builtin_btf_type_id)
-      FnDecl = llvm::Intrinsic::getDeclaration(
+      FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
           &CGM.getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
     else
-      FnDecl = llvm::Intrinsic::getDeclaration(
+      FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
           &CGM.getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
     CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
@@ -13710,7 +13732,7 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
     Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
     Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
 
-    llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
+    llvm::Function *IntrinsicFn = llvm::Intrinsic::getOrInsertDeclaration(
         &CGM.getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
     CallInst *Fn =
         Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
@@ -18884,7 +18906,7 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
     // for the DirectX intrinsic and the demangled builtin name
     switch (CGM.getTarget().getTriple().getArch()) {
     case llvm::Triple::dxil:
-      return EmitRuntimeCall(Intrinsic::getDeclaration(
+      return EmitRuntimeCall(Intrinsic::getOrInsertDeclaration(
           &CGM.getModule(), Intrinsic::dx_wave_getlaneindex));
     case llvm::Triple::spirv:
       return EmitRuntimeCall(CGM.CreateRuntimeFunction(
@@ -18897,7 +18919,8 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
   }
   case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
     Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
-    return EmitRuntimeCall(Intrinsic::getDeclaration(&CGM.getModule(), ID));
+    return EmitRuntimeCall(
+        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
   }
   case Builtin::BI__builtin_hlsl_elementwise_sign: {
     auto *Arg0 = E->getArg(0);
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 609957b75d6e7e2ae56e9771d43dbe180ef612f7..06015a9e541ea24a03db0f19bb1419bf5c079b2d 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -287,7 +287,8 @@ PrintingPolicy CGDebugInfo::getPrintingPolicy() const {
     PP.SplitTemplateClosers = true;
   }
 
-  PP.SuppressInlineNamespace = false;
+  PP.SuppressInlineNamespace =
+      PrintingPolicy::SuppressInlineNamespaceMode::None;
   PP.PrintCanonicalTypes = true;
   PP.UsePreferredNames = false;
   PP.AlwaysIncludeTypeForTemplateArgument = true;
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 563f728e29d78194cff8661e46299afce0d0ce01..1d0660292cecc1e40279ffe01377048afc27b19c 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1945,7 +1945,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
                                   replaceUndef(CGM, isPattern, constant));
     }
 
-    if (D.getType()->isBitIntType() &&
+    if (constant && D.getType()->isBitIntType() &&
         CGM.getTypes().typeRequiresSplitIntoByteArray(D.getType())) {
       // Constants for long _BitInt types are split into individual bytes.
       // Try to fold these back into an integer constant so it can be stored
@@ -2509,8 +2509,8 @@ void CodeGenFunction::pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
 llvm::Function *CodeGenModule::getLLVMLifetimeStartFn() {
   if (LifetimeStartFn)
     return LifetimeStartFn;
-  LifetimeStartFn = llvm::Intrinsic::getDeclaration(&getModule(),
-    llvm::Intrinsic::lifetime_start, AllocaInt8PtrTy);
+  LifetimeStartFn = llvm::Intrinsic::getOrInsertDeclaration(
+      &getModule(), llvm::Intrinsic::lifetime_start, AllocaInt8PtrTy);
   return LifetimeStartFn;
 }
 
@@ -2518,8 +2518,8 @@ llvm::Function *CodeGenModule::getLLVMLifetimeStartFn() {
 llvm::Function *CodeGenModule::getLLVMLifetimeEndFn() {
   if (LifetimeEndFn)
     return LifetimeEndFn;
-  LifetimeEndFn = llvm::Intrinsic::getDeclaration(&getModule(),
-    llvm::Intrinsic::lifetime_end, AllocaInt8PtrTy);
+  LifetimeEndFn = llvm::Intrinsic::getOrInsertDeclaration(
+      &getModule(), llvm::Intrinsic::lifetime_end, AllocaInt8PtrTy);
   return LifetimeEndFn;
 }
 
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index bb2ed237ee9f35f4c3c5a41c69a52a958e7c513d..44a45413dbc45afdc39c0e9a197cbcbf71ee8ffa 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -1843,7 +1843,7 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
         std::make_pair(ParentAlloca, ParentCGF.EscapedLocals.size()));
     int FrameEscapeIdx = InsertPair.first->second;
     // call ptr @llvm.localrecover(ptr @parentFn, ptr %fp, i32 N)
-    llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration(
+    llvm::Function *FrameRecoverFn = llvm::Intrinsic::getOrInsertDeclaration(
         &CGM.getModule(), llvm::Intrinsic::localrecover);
     RecoverCall = Builder.CreateCall(
         FrameRecoverFn, {ParentCGF.CurFn, ParentFP,
@@ -1942,7 +1942,7 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
       // %1 = call ptr @llvm.localrecover(@"?fin$0@0@main@@",..)
       // %2 = load ptr, ptr %1, align 8
       //   ==> %2 is the frame-pointer of outermost host function
-      llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration(
+      llvm::Function *FrameRecoverFn = llvm::Intrinsic::getOrInsertDeclaration(
           &CGM.getModule(), llvm::Intrinsic::localrecover);
       ParentFP = Builder.CreateCall(
           FrameRecoverFn, {ParentCGF.CurFn, ParentFP,
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index bb120917665b3f88a38a4e9fcb0dec166ffd0ae4..161dd7e4138a7621fbeed3c0444d58d4a21835b7 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1528,7 +1528,12 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
 ///
 LValue CodeGenFunction::EmitLValue(const Expr *E,
                                    KnownNonNull_t IsKnownNonNull) {
-  LValue LV = EmitLValueHelper(E, IsKnownNonNull);
+  // Running with sufficient stack space to avoid deeply nested expressions
+  // cause a stack overflow.
+  LValue LV;
+  CGM.runWithSufficientStackSpace(
+      E->getExprLoc(), [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });
+
   if (IsKnownNonNull && !LV.isKnownNonNull())
     LV.setKnownNonNull();
   return LV;
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 3c680edc6000830c39294148d390d840b09aae4b..e998cd79198a5cbdb7c13d356ed721879c504a10 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -12,6 +12,7 @@
 
 #include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
+#include "CGRecordLayout.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "ConstantEmitter.h"
@@ -64,6 +65,9 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   void withReturnValueSlot(const Expr *E,
                            llvm::function_ref<RValue(ReturnValueSlot)> Fn);
 
+  void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,
+                         const FieldDecl *NextField);
+
 public:
   AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
     : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
@@ -1698,6 +1702,9 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
   // Prepare a 'this' for CXXDefaultInitExprs.
   CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
 
+  const bool ZeroInitPadding =
+      CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();
+
   if (record->isUnion()) {
     // Only initialize one field of a union. The field itself is
     // specified by the initializer list.
@@ -1722,16 +1729,27 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
     if (NumInitElements) {
       // Store the initializer into the field
       EmitInitializationToLValue(InitExprs[0], FieldLoc);
+      if (ZeroInitPadding) {
+        uint64_t TotalSize = CGF.getContext().toBits(
+            Dest.getPreferredSize(CGF.getContext(), DestLV.getType()));
+        uint64_t FieldSize = CGF.getContext().getTypeSize(FieldLoc.getType());
+        DoZeroInitPadding(FieldSize, TotalSize, nullptr);
+      }
     } else {
       // Default-initialize to null.
-      EmitNullInitializationToLValue(FieldLoc);
+      if (ZeroInitPadding)
+        EmitNullInitializationToLValue(DestLV);
+      else
+        EmitNullInitializationToLValue(FieldLoc);
     }
-
     return;
   }
 
   // Here we iterate over the fields; this makes it simpler to both
   // default-initialize fields and skip over unnamed fields.
+  const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(record);
+  uint64_t PaddingStart = 0;
+
   for (const auto *field : record->fields()) {
     // We're done once we hit the flexible array member.
     if (field->getType()->isIncompleteArrayType())
@@ -1748,6 +1766,9 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
         CGF.getTypes().isZeroInitializable(ExprToVisit->getType()))
       break;
 
+    if (ZeroInitPadding)
+      DoZeroInitPadding(PaddingStart,
+                        Layout.getFieldOffset(field->getFieldIndex()), field);
 
     LValue LV = CGF.EmitLValueForFieldInitialization(DestLV, field);
     // We never generate write-barries for initialized fields.
@@ -1774,6 +1795,54 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
       }
     }
   }
+  if (ZeroInitPadding) {
+    uint64_t TotalSize = CGF.getContext().toBits(
+        Dest.getPreferredSize(CGF.getContext(), DestLV.getType()));
+    DoZeroInitPadding(PaddingStart, TotalSize, nullptr);
+  }
+}
+
+void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,
+                                       uint64_t PaddingEnd,
+                                       const FieldDecl *NextField) {
+
+  auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {
+    CharUnits Start = CGF.getContext().toCharUnitsFromBits(StartBit);
+    CharUnits End = CGF.getContext().toCharUnitsFromBits(EndBit);
+    Address Addr = Dest.getAddress().withElementType(CGF.CharTy);
+    if (!Start.isZero())
+      Addr = Builder.CreateConstGEP(Addr, Start.getQuantity());
+    llvm::Constant *SizeVal = Builder.getInt64((End - Start).getQuantity());
+    CGF.Builder.CreateMemSet(Addr, Builder.getInt8(0), SizeVal, false);
+  };
+
+  if (NextField != nullptr && NextField->isBitField()) {
+    // For bitfield, zero init StorageSize before storing the bits. So we don't
+    // need to handle big/little endian.
+    const CGRecordLayout &RL =
+        CGF.getTypes().getCGRecordLayout(NextField->getParent());
+    const CGBitFieldInfo &Info = RL.getBitFieldInfo(NextField);
+    uint64_t StorageStart = CGF.getContext().toBits(Info.StorageOffset);
+    if (StorageStart + Info.StorageSize > PaddingStart) {
+      if (StorageStart > PaddingStart)
+        InitBytes(PaddingStart, StorageStart);
+      Address Addr = Dest.getAddress();
+      if (!Info.StorageOffset.isZero())
+        Addr = Builder.CreateConstGEP(Addr.withElementType(CGF.CharTy),
+                                      Info.StorageOffset.getQuantity());
+      Addr = Addr.withElementType(
+          llvm::Type::getIntNTy(CGF.getLLVMContext(), Info.StorageSize));
+      Builder.CreateStore(Builder.getIntN(Info.StorageSize, 0), Addr);
+      PaddingStart = StorageStart + Info.StorageSize;
+    }
+    return;
+  }
+
+  if (PaddingStart < PaddingEnd)
+    InitBytes(PaddingStart, PaddingEnd);
+  if (NextField != nullptr)
+    PaddingStart =
+        PaddingEnd + CGF.getContext().getTypeSize(NextField->getType());
 }
 
 void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index dd65080a8404467811f2026b4c078fde6e866ba3..655fc3dc954c8199da545ae8aae2c59efecf3bc8 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -42,6 +42,16 @@ using namespace CodeGen;
 namespace {
 class ConstExprEmitter;
 
+llvm::Constant *getPadding(const CodeGenModule &CGM, CharUnits PadSize) {
+  llvm::Type *Ty = CGM.CharTy;
+  if (PadSize > CharUnits::One())
+    Ty = llvm::ArrayType::get(Ty, PadSize.getQuantity());
+  if (CGM.shouldZeroInitPadding()) {
+    return llvm::Constant::getNullValue(Ty);
+  }
+  return llvm::UndefValue::get(Ty);
+}
+
 struct ConstantAggregateBuilderUtils {
   CodeGenModule &CGM;
 
@@ -61,10 +71,7 @@ struct ConstantAggregateBuilderUtils {
   }
 
   llvm::Constant *getPadding(CharUnits PadSize) const {
-    llvm::Type *Ty = CGM.CharTy;
-    if (PadSize > CharUnits::One())
-      Ty = llvm::ArrayType::get(Ty, PadSize.getQuantity());
-    return llvm::UndefValue::get(Ty);
+    return ::getPadding(CGM, PadSize);
   }
 
   llvm::Constant *getZeroes(CharUnits ZeroSize) const {
@@ -591,6 +598,11 @@ private:
   bool Build(const InitListExpr *ILE, bool AllowOverwrite);
   bool Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase,
              const CXXRecordDecl *VTableClass, CharUnits BaseOffset);
+  bool DoZeroInitPadding(const ASTRecordLayout &Layout, unsigned FieldNo,
+                         const FieldDecl &Field, bool AllowOverwrite,
+                         CharUnits &SizeSoFar, bool &ZeroFieldSize);
+  bool DoZeroInitPadding(const ASTRecordLayout &Layout, bool AllowOverwrite,
+                         CharUnits SizeSoFar);
   llvm::Constant *Finalize(QualType Ty);
 };
 
@@ -715,6 +727,10 @@ bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
     if (CXXRD->getNumBases())
       return false;
 
+  const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
+  bool ZeroFieldSize = false;
+  CharUnits SizeSoFar = CharUnits::Zero();
+
   for (FieldDecl *Field : RD->fields()) {
     ++FieldNo;
 
@@ -732,8 +748,13 @@ bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
     const Expr *Init = nullptr;
     if (ElementNo < ILE->getNumInits())
       Init = ILE->getInit(ElementNo++);
-    if (isa_and_nonnull<NoInitExpr>(Init))
+    if (isa_and_nonnull<NoInitExpr>(Init)) {
+      if (ZeroInitPadding &&
+          !DoZeroInitPadding(Layout, FieldNo, *Field, AllowOverwrite, SizeSoFar,
+                             ZeroFieldSize))
+        return false;
       continue;
+    }
 
     // Zero-sized fields are not emitted, but their initializers may still
     // prevent emission of this struct as a constant.
@@ -743,6 +764,11 @@ bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
       continue;
     }
 
+    if (ZeroInitPadding &&
+        !DoZeroInitPadding(Layout, FieldNo, *Field, AllowOverwrite, SizeSoFar,
+                           ZeroFieldSize))
+      return false;
+
     // When emitting a DesignatedInitUpdateExpr, a nested InitListExpr
     // represents additional overwriting of our current constant value, and not
     // a new constant to emit independently.
@@ -768,6 +794,10 @@ bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
     if (!EltInit)
       return false;
 
+    if (ZeroInitPadding && ZeroFieldSize)
+      SizeSoFar += CharUnits::fromQuantity(
+          CGM.getDataLayout().getTypeAllocSize(EltInit->getType()));
+
     if (!Field->isBitField()) {
       // Handle non-bitfield members.
       if (!AppendField(Field, Layout.getFieldOffset(FieldNo), EltInit,
@@ -785,6 +815,9 @@ bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
     }
   }
 
+  if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
+    return false;
+
   return true;
 }
 
@@ -849,6 +882,9 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
 
   unsigned FieldNo = 0;
   uint64_t OffsetBits = CGM.getContext().toBits(Offset);
+  const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
+  bool ZeroFieldSize = false;
+  CharUnits SizeSoFar = CharUnits::Zero();
 
   bool AllowOverwrite = false;
   for (RecordDecl::field_iterator Field = RD->field_begin(),
@@ -870,6 +906,15 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
     if (!EltInit)
       return false;
 
+    if (ZeroInitPadding) {
+      if (!DoZeroInitPadding(Layout, FieldNo, **Field, AllowOverwrite,
+                             SizeSoFar, ZeroFieldSize))
+        return false;
+      if (ZeroFieldSize)
+        SizeSoFar += CharUnits::fromQuantity(
+            CGM.getDataLayout().getTypeAllocSize(EltInit->getType()));
+    }
+
     if (!Field->isBitField()) {
       // Handle non-bitfield members.
       if (!AppendField(*Field, Layout.getFieldOffset(FieldNo) + OffsetBits,
@@ -886,7 +931,49 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
         return false;
     }
   }
+  if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
+    return false;
+
+  return true;
+}
+
+bool ConstStructBuilder::DoZeroInitPadding(
+    const ASTRecordLayout &Layout, unsigned FieldNo, const FieldDecl &Field,
+    bool AllowOverwrite, CharUnits &SizeSoFar, bool &ZeroFieldSize) {
+  uint64_t StartBitOffset = Layout.getFieldOffset(FieldNo);
+  CharUnits StartOffset = CGM.getContext().toCharUnitsFromBits(StartBitOffset);
+  if (SizeSoFar < StartOffset)
+    if (!AppendBytes(SizeSoFar, getPadding(CGM, StartOffset - SizeSoFar),
+                     AllowOverwrite))
+      return false;
+
+  if (!Field.isBitField()) {
+    CharUnits FieldSize = CGM.getContext().getTypeSizeInChars(Field.getType());
+    SizeSoFar = StartOffset + FieldSize;
+    ZeroFieldSize = FieldSize.isZero();
+  } else {
+    const CGRecordLayout &RL =
+        CGM.getTypes().getCGRecordLayout(Field.getParent());
+    const CGBitFieldInfo &Info = RL.getBitFieldInfo(&Field);
+    uint64_t EndBitOffset = StartBitOffset + Info.Size;
+    SizeSoFar = CGM.getContext().toCharUnitsFromBits(EndBitOffset);
+    if (EndBitOffset % CGM.getContext().getCharWidth() != 0) {
+      SizeSoFar++;
+    }
+    ZeroFieldSize = Info.Size == 0;
+  }
+  return true;
+}
 
+bool ConstStructBuilder::DoZeroInitPadding(const ASTRecordLayout &Layout,
+                                           bool AllowOverwrite,
+                                           CharUnits SizeSoFar) {
+  CharUnits TotalSize = Layout.getSize();
+  if (SizeSoFar < TotalSize)
+    if (!AppendBytes(SizeSoFar, getPadding(CGM, TotalSize - SizeSoFar),
+                     AllowOverwrite))
+      return false;
+  SizeSoFar = TotalSize;
   return true;
 }
 
@@ -1127,12 +1214,10 @@ public:
 
       assert(CurSize <= TotalSize && "Union size mismatch!");
       if (unsigned NumPadBytes = TotalSize - CurSize) {
-        llvm::Type *Ty = CGM.CharTy;
-        if (NumPadBytes > 1)
-          Ty = llvm::ArrayType::get(Ty, NumPadBytes);
-
-        Elts.push_back(llvm::UndefValue::get(Ty));
-        Types.push_back(Ty);
+        llvm::Constant *Padding =
+            getPadding(CGM, CharUnits::fromQuantity(NumPadBytes));
+        Elts.push_back(Padding);
+        Types.push_back(Padding->getType());
       }
 
       llvm::StructType *STy = llvm::StructType::get(VMContext, Types, false);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index bf3071c0d7e1bfd140071ab24fcfb15d966a4615..40129da1c01155e739ad50531b8fdcb707b2f46b 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1713,25 +1713,15 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
     if (!Ctor && !Dtor)
       return nullptr;
 
-    llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
-    auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
-                                               /*isVarArg=*/false)
-                           ->getPointerTo();
     // Copying constructor for the threadprivate variable.
     // Must be NULL - reserved by runtime, but currently it requires that this
     // parameter is always NULL. Otherwise it fires assertion.
-    CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
+    CopyCtor = llvm::Constant::getNullValue(CGM.UnqualPtrTy);
     if (Ctor == nullptr) {
-      auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
-                                             /*isVarArg=*/false)
-                         ->getPointerTo();
-      Ctor = llvm::Constant::getNullValue(CtorTy);
+      Ctor = llvm::Constant::getNullValue(CGM.UnqualPtrTy);
     }
     if (Dtor == nullptr) {
-      auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
-                                             /*isVarArg=*/false)
-                         ->getPointerTo();
-      Dtor = llvm::Constant::getNullValue(DtorTy);
+      Dtor = llvm::Constant::getNullValue(CGM.UnqualPtrTy);
     }
     if (!CGF) {
       auto *InitFunctionTy =
@@ -1817,7 +1807,7 @@ Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
               OMPBuilder.getOrCreateRuntimeFunction(
                   CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
               Args),
-          VarLVType->getPointerTo(/*AddrSpace=*/0)),
+          CGF.Builder.getPtrTy(0)),
       VarLVType, CGM.getContext().getTypeAlignInChars(VarType));
 }
 
@@ -2131,10 +2121,7 @@ static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
   llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
 
   llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType());
-  return Address(
-      CGF.Builder.CreateBitCast(
-          Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
-      ElemTy, CGF.getContext().getDeclAlign(Var));
+  return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(Var));
 }
 
 static llvm::Value *emitCopyprivateCopyFunction(
@@ -2166,11 +2153,11 @@ static llvm::Value *emitCopyprivateCopyFunction(
   // Src = (void*[n])(RHSArg);
   Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
                   CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
-                  ArgsElemType->getPointerTo()),
+                  CGF.Builder.getPtrTy(0)),
               ArgsElemType, CGF.getPointerAlign());
   Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
                   CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
-                  ArgsElemType->getPointerTo()),
+                  CGF.Builder.getPtrTy(0)),
               ArgsElemType, CGF.getPointerAlign());
   // *(Type0*)Dst[0] = *(Type0*)Src[0];
   // *(Type1*)Dst[1] = *(Type1*)Src[1];
@@ -3681,9 +3668,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
   QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
   QualType KmpTaskTWithPrivatesPtrQTy =
       C.getPointerType(KmpTaskTWithPrivatesQTy);
-  llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
-  llvm::Type *KmpTaskTWithPrivatesPtrTy =
-      KmpTaskTWithPrivatesTy->getPointerTo();
+  llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(0);
   llvm::Value *KmpTaskTWithPrivatesTySize =
       CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
   QualType SharedsPtrTy = C.getPointerType(SharedsTy);
@@ -4402,7 +4387,7 @@ Address CGOpenMPRuntime::emitDepobjDependClause(
                           Args, ".dep.arr.addr");
   llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy);
   Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-      Addr, KmpDependInfoLlvmTy->getPointerTo());
+      Addr, CGF.Builder.getPtrTy(0));
   DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
   // Write number of elements in the first element of array for depobj.
   LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
@@ -4844,11 +4829,11 @@ llvm::Function *CGOpenMPRuntime::emitReductionFunction(
   // Src = (void*[n])(RHSArg);
   Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
                   CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
-                  ArgsElemType->getPointerTo()),
+                  CGF.Builder.getPtrTy(0)),
               ArgsElemType, CGF.getPointerAlign());
   Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
                   CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
-                  ArgsElemType->getPointerTo()),
+                  CGF.Builder.getPtrTy(0)),
               ArgsElemType, CGF.getPointerAlign());
 
   //  ...
@@ -5280,8 +5265,7 @@ static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
   QualType PrivateType = RCG.getPrivateType(N);
   Address PrivateAddr = CGF.EmitLoadOfPointer(
-      CGF.GetAddrOfLocalVar(&Param).withElementType(
-          CGF.ConvertTypeForMem(PrivateType)->getPointerTo()),
+      CGF.GetAddrOfLocalVar(&Param).withElementType(CGF.Builder.getPtrTy(0)),
       C.getPointerType(PrivateType)->castAs<PointerType>());
   llvm::Value *Size = nullptr;
   // If the size of the reduction item is non-constant, load it from global
@@ -5369,15 +5353,14 @@ static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
       // Pull out the pointer to the variable.
       CGF.EmitLoadOfPointer(
           CGF.GetAddrOfLocalVar(&ParamInOut)
-              .withElementType(
-                  CGF.ConvertTypeForMem(LHSVD->getType())->getPointerTo()),
+              .withElementType(CGF.Builder.getPtrTy(0)),
           C.getPointerType(LHSVD->getType())->castAs<PointerType>()));
   PrivateScope.addPrivate(
       RHSVD,
       // Pull out the pointer to the variable.
       CGF.EmitLoadOfPointer(
           CGF.GetAddrOfLocalVar(&ParamIn).withElementType(
-              CGF.ConvertTypeForMem(RHSVD->getType())->getPointerTo()),
+              CGF.Builder.getPtrTy(0)),
           C.getPointerType(RHSVD->getType())->castAs<PointerType>()));
   PrivateScope.Privatize();
   // Emit the combiner body:
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index c920d93957b16ee41b55007ec13ca43dd7a0c085..35ff75416cb7761fb241119dd3fe00426ee1de79 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -2274,6 +2274,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective(const OMPRequiresDecl *D) {
       case OffloadArch::SM_89:
       case OffloadArch::SM_90:
       case OffloadArch::SM_90a:
+      case OffloadArch::SM_100:
       case OffloadArch::GFX600:
       case OffloadArch::GFX601:
       case OffloadArch::GFX602:
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index e1fd9b72b8d7b2d02d86023bd2b1b0c2df3faafe..f3023c7a20c4050f2a66740b5d747f23a705b48a 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -463,7 +463,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     EscapeArgs.resize(EscapedLocals.size());
     for (auto &Pair : EscapedLocals)
       EscapeArgs[Pair.second] = Pair.first;
-    llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
+    llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(
         &CGM.getModule(), llvm::Intrinsic::localescape);
     CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
   }
@@ -3130,7 +3130,7 @@ void CodeGenFunction::emitAlignmentAssumptionCheck(
     llvm::Instruction *Assumption) {
   assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&
          cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
-             llvm::Intrinsic::getDeclaration(
+             llvm::Intrinsic::getOrInsertDeclaration(
                  Builder.GetInsertBlock()->getParent()->getParent(),
                  llvm::Intrinsic::assume) &&
          "Assumption should be a call to llvm.assume().");
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 5ba098144a74e79f086ebc99fd9e65864a7dae79..b05ab3606a698ba37fdf26a91eabbd3ab388c8f1 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -44,6 +44,7 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/Stack.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Version.h"
 #include "clang/CodeGen/BackendUtil.h"
@@ -175,10 +176,7 @@ createTargetCodeGenInfo(CodeGenModule &CGM) {
     else if (ABIStr == "aapcs16")
       Kind = ARMABIKind::AAPCS16_VFP;
     else if (CodeGenOpts.FloatABI == "hard" ||
-             (CodeGenOpts.FloatABI != "soft" &&
-              (Triple.getEnvironment() == llvm::Triple::GNUEABIHF ||
-               Triple.getEnvironment() == llvm::Triple::MuslEABIHF ||
-               Triple.getEnvironment() == llvm::Triple::EABIHF)))
+             (CodeGenOpts.FloatABI != "soft" && Triple.isHardFloatABI()))
       Kind = ARMABIKind::AAPCS_VFP;
 
     return createARMTargetCodeGenInfo(CGM, Kind);
@@ -1596,6 +1594,19 @@ void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type) {
   getDiags().Report(Context.getFullLoc(D->getLocation()), DiagID) << Msg;
 }
 
+void CodeGenModule::warnStackExhausted(SourceLocation Loc) {
+  // Only warn about this once.
+  if (!WarnedStackExhausted) {
+    getDiags().Report(Loc, diag::warn_stack_exhausted);
+    WarnedStackExhausted = true;
+  }
+}
+
+void CodeGenModule::runWithSufficientStackSpace(SourceLocation Loc,
+                                                llvm::function_ref<void()> Fn) {
+  clang::runWithSufficientStackSpace([&] { warnStackExhausted(Loc); }, Fn);
+}
+
 llvm::ConstantInt *CodeGenModule::getSize(CharUnits size) {
   return llvm::ConstantInt::get(SizeTy, size.getQuantity());
 }
@@ -6218,8 +6229,8 @@ void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) {
 
 llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,
                                             ArrayRef<llvm::Type*> Tys) {
-  return llvm::Intrinsic::getDeclaration(&getModule(), (llvm::Intrinsic::ID)IID,
-                                         Tys);
+  return llvm::Intrinsic::getOrInsertDeclaration(&getModule(),
+                                                 (llvm::Intrinsic::ID)IID, Tys);
 }
 
 static llvm::StringMapEntry<llvm::GlobalVariable *> &
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index c58bb88035ca8ab3ef781e9350424aa8bf89fca1..fa82a81b05dd539c433c66bf369066152db7697f 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -336,6 +336,7 @@ private:
   std::unique_ptr<llvm::IndexedInstrProfReader> PGOReader;
   InstrProfStats PGOStats;
   std::unique_ptr<llvm::SanitizerStatReport> SanStats;
+  bool WarnedStackExhausted = false;
 
   // A set of references that have only been seen via a weakref so far. This is
   // used to remove the weak of the reference if we ever see a direct reference
@@ -1297,6 +1298,16 @@ public:
   /// Print out an error that codegen doesn't support the specified decl yet.
   void ErrorUnsupported(const Decl *D, const char *Type);
 
+  /// Warn that the stack is nearly exhausted.
+  void warnStackExhausted(SourceLocation Loc);
+
+  /// Run some code with "sufficient" stack space. (Currently, at least 256K is
+  /// guaranteed). Produces a warning if we're low on stack space and allocates
+  /// more in that case. Use this in code that may recurse deeply to avoid stack
+  /// overflow.
+  void runWithSufficientStackSpace(SourceLocation Loc,
+                                   llvm::function_ref<void()> Fn);
+
   /// Set the attributes on the LLVM function for the given decl and function
   /// info. This applies attributes necessary for handling the ABI as well as
   /// user specified attributes like section.
@@ -1676,6 +1687,57 @@ public:
     MustTailCallUndefinedGlobals.insert(Global);
   }
 
+  bool shouldZeroInitPadding() const {
+    // In C23 (N3096) $6.7.10:
+    // """
+    // If any object is initialized with an empty iniitializer, then it is
+    // subject to default initialization:
+    //  - if it is an aggregate, every member is initialized (recursively)
+    //  according to these rules, and any padding is initialized to zero bits;
+    //  - if it is a union, the first named member is initialized (recursively)
+    //  according to these rules, and any padding is initialized to zero bits.
+    //
+    // If the aggregate or union contains elements or members that are
+    // aggregates or unions, these rules apply recursively to the subaggregates
+    // or contained unions.
+    //
+    // If there are fewer initializers in a brace-enclosed list than there are
+    // elements or members of an aggregate, or fewer characters in a string
+    // literal used to initialize an array of known size than there are elements
+    // in the array, the remainder of the aggregate is subject to default
+    // initialization.
+    // """
+    //
+    // From my understanding, the standard is ambiguous in the following two
+    // areas:
+    // 1. For a union type with empty initializer, if the first named member is
+    // not the largest member, then the bytes comes after the first named member
+    // but before padding are left unspecified. An example is:
+    //    union U { int a; long long b;};
+    //    union U u = {};  // The first 4 bytes are 0, but 4-8 bytes are left
+    //    unspecified.
+    //
+    // 2. It only mentions padding for empty initializer, but doesn't mention
+    // padding for a non empty initialization list. And if the aggregation or
+    // union contains elements or members that are aggregates or unions, and
+    // some are non empty initializers, while others are empty initiailizers,
+    // the padding initialization is unclear. An example is:
+    //    struct S1 { int a; long long b; };
+    //    struct S2 { char c; struct S1 s1; };
+    //    // The values for paddings between s2.c and s2.s1.a, between s2.s1.a
+    //    and s2.s1.b are unclear.
+    //    struct S2 s2 = { 'c' };
+    //
+    // Here we choose to zero initiailize left bytes of a union type. Because
+    // projects like the Linux kernel are relying on this behavior. If we don't
+    // explicitly zero initialize them, the undef values can be optimized to
+    // return gabage data. We also choose to zero initialize paddings for
+    // aggregates and unions, no matter they are initialized by empty
+    // initializers or non empty initializers. This can provide a consistent
+    // behavior. So projects like the Linux kernel can rely on it.
+    return !getLangOpts().CPlusPlus;
+  }
+
 private:
   bool shouldDropDLLAttribute(const Decl *D, const llvm::GlobalValue *GV) const;
 
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index c38198f7db6eb3ad26c1f6c39ea9f75a56338e49..82eb7234c98c7af0db3a1eca66960bb5d0a5c7af 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -60,7 +60,8 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
   // example, we should probably enable PrintCanonicalTypes and
   // FullyQualifiedNames.
   PrintingPolicy Policy = RD->getASTContext().getPrintingPolicy();
-  Policy.SuppressInlineNamespace = false;
+  Policy.SuppressInlineNamespace =
+      PrintingPolicy::SuppressInlineNamespaceMode::None;
 
   // Name the codegen type after the typedef name
   // if there is no tag type name available
diff --git a/clang/lib/CodeGen/Targets/ARM.cpp b/clang/lib/CodeGen/Targets/ARM.cpp
index f7d7471d386b2116bd97bfc1f5a95dfc42e9dc35..49ac1a76e767aa0602e1b8eb61a6c94451223a41 100644
--- a/clang/lib/CodeGen/Targets/ARM.cpp
+++ b/clang/lib/CodeGen/Targets/ARM.cpp
@@ -35,7 +35,9 @@ public:
     case llvm::Triple::EABI:
     case llvm::Triple::EABIHF:
     case llvm::Triple::GNUEABI:
+    case llvm::Triple::GNUEABIT64:
     case llvm::Triple::GNUEABIHF:
+    case llvm::Triple::GNUEABIHFT64:
     case llvm::Triple::MuslEABI:
     case llvm::Triple::MuslEABIHF:
       return true;
@@ -48,6 +50,7 @@ public:
     switch (getTarget().getTriple().getEnvironment()) {
     case llvm::Triple::EABIHF:
     case llvm::Triple::GNUEABIHF:
+    case llvm::Triple::GNUEABIHFT64:
     case llvm::Triple::MuslEABIHF:
       return true;
     default:
diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp
index 56129622f48dbde7e92a541bca3ae6135464b2e6..23c96fa5cf98cb3926f80c0de80f5aca4c04c7d9 100644
--- a/clang/lib/CodeGen/Targets/SystemZ.cpp
+++ b/clang/lib/CodeGen/Targets/SystemZ.cpp
@@ -110,8 +110,8 @@ public:
     if (Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isFP128Ty()) {
       llvm::Module &M = CGM.getModule();
       auto &Ctx = M.getContext();
-      llvm::Function *TDCFunc =
-          llvm::Intrinsic::getDeclaration(&M, llvm::Intrinsic::s390_tdc, Ty);
+      llvm::Function *TDCFunc = llvm::Intrinsic::getOrInsertDeclaration(
+          &M, llvm::Intrinsic::s390_tdc, Ty);
       unsigned TDCBits = 0;
       switch (BuiltinID) {
       case Builtin::BI__builtin_isnan:
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index a53f09e3033954c211c96cc7099820096ed1f5b2..883edf841128cf5bfdf4ed2fe49ac6eddb22c405 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -610,7 +610,8 @@ static llvm::Triple computeTargetTriple(const Driver &D,
     if (A->getOption().matches(options::OPT_m64) ||
         A->getOption().matches(options::OPT_maix64)) {
       AT = Target.get64BitArchVariant().getArch();
-      if (Target.getEnvironment() == llvm::Triple::GNUX32)
+      if (Target.getEnvironment() == llvm::Triple::GNUX32 ||
+          Target.getEnvironment() == llvm::Triple::GNUT64)
         Target.setEnvironment(llvm::Triple::GNU);
       else if (Target.getEnvironment() == llvm::Triple::MuslX32)
         Target.setEnvironment(llvm::Triple::Musl);
@@ -673,6 +674,7 @@ static llvm::Triple computeTargetTriple(const Driver &D,
       } else if (ABIName == "n32") {
         Target = Target.get64BitArchVariant();
         if (Target.getEnvironment() == llvm::Triple::GNU ||
+            Target.getEnvironment() == llvm::Triple::GNUT64 ||
             Target.getEnvironment() == llvm::Triple::GNUABI64)
           Target.setEnvironment(llvm::Triple::GNUABIN32);
         else if (Target.getEnvironment() == llvm::Triple::Musl ||
@@ -681,6 +683,7 @@ static llvm::Triple computeTargetTriple(const Driver &D,
       } else if (ABIName == "64") {
         Target = Target.get64BitArchVariant();
         if (Target.getEnvironment() == llvm::Triple::GNU ||
+            Target.getEnvironment() == llvm::Triple::GNUT64 ||
             Target.getEnvironment() == llvm::Triple::GNUABIN32)
           Target.setEnvironment(llvm::Triple::GNUABI64);
         else if (Target.getEnvironment() == llvm::Triple::Musl ||
diff --git a/clang/lib/Driver/ToolChains/AVR.cpp b/clang/lib/Driver/ToolChains/AVR.cpp
index bb5c0e6db9978eb4aa05f4676efefd11641c9507..08e906ac9e80676c451019f6ae17b901144a8756 100644
--- a/clang/lib/Driver/ToolChains/AVR.cpp
+++ b/clang/lib/Driver/ToolChains/AVR.cpp
@@ -400,6 +400,14 @@ void AVRToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
 void AVRToolChain::addClangTargetOptions(
     const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
     Action::OffloadKind DeviceOffloadKind) const {
+  // Reject C/C++ compilation for avr1 devices since they have no SRAM.
+  const Driver &D = getDriver();
+  std::string CPU = getCPUName(D, DriverArgs, getTriple());
+  std::optional<StringRef> FamilyName = GetMCUFamilyName(CPU);
+  if (CPU == "avr1" || (FamilyName && *FamilyName == "avr1"))
+    D.Diag(diag::err_drv_opt_unsupported_input_type)
+        << "-mmcu=" + CPU << "c/c++";
+
   // By default, use `.ctors` (not `.init_array`), as required by libgcc, which
   // runs constructors/destructors on AVR.
   if (!DriverArgs.hasFlag(options::OPT_fuse_init_array,
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index a6041b809b80b6fc80e7c286b60dbb08cf3da11a..0489911ecd9deea5061701fff2ac7f2c2aa8fa25 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -327,6 +327,11 @@ void arm::setFloatABIInTriple(const Driver &D, const ArgList &Args,
     Triple.setEnvironment(isHardFloat ? llvm::Triple::GNUEABIHF
                                       : llvm::Triple::GNUEABI);
     break;
+  case llvm::Triple::GNUEABIT64:
+  case llvm::Triple::GNUEABIHFT64:
+    Triple.setEnvironment(isHardFloat ? llvm::Triple::GNUEABIHFT64
+                                      : llvm::Triple::GNUEABIT64);
+    break;
   case llvm::Triple::EABI:
   case llvm::Triple::EABIHF:
     Triple.setEnvironment(isHardFloat ? llvm::Triple::EABIHF
@@ -414,10 +419,12 @@ arm::FloatABI arm::getDefaultFloatABI(const llvm::Triple &Triple) {
       return FloatABI::Soft;
     switch (Triple.getEnvironment()) {
     case llvm::Triple::GNUEABIHF:
+    case llvm::Triple::GNUEABIHFT64:
     case llvm::Triple::MuslEABIHF:
     case llvm::Triple::EABIHF:
       return FloatABI::Hard;
     case llvm::Triple::GNUEABI:
+    case llvm::Triple::GNUEABIT64:
     case llvm::Triple::MuslEABI:
     case llvm::Triple::EABI:
       // EABI is always AAPCS, and if it was not marked 'hard', it's softfp
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 34d24b02e1423fa3339fc9b7509803489b802ec6..02b3453a4f6c7361d933814d645b54c99071a4cb 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -94,24 +94,6 @@ static void CheckCodeGenerationOptions(const Driver &D, const ArgList &Args) {
                                                       << "-static";
 }
 
-// Add backslashes to escape spaces and other backslashes.
-// This is used for the space-separated argument list specified with
-// the -dwarf-debug-flags option.
-static void EscapeSpacesAndBackslashes(const char *Arg,
-                                       SmallVectorImpl<char> &Res) {
-  for (; *Arg; ++Arg) {
-    switch (*Arg) {
-    default:
-      break;
-    case ' ':
-    case '\\':
-      Res.push_back('\\');
-      break;
-    }
-    Res.push_back(*Arg);
-  }
-}
-
 /// Apply \a Work on the current tool chain \a RegularToolChain and any other
 /// offloading tool chain that is associated with the current action \a JA.
 static void
@@ -7761,31 +7743,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Also record command line arguments into the debug info if
   // -grecord-gcc-switches options is set on.
   // By default, -gno-record-gcc-switches is set on and no recording.
-  auto GRecordSwitches =
-      Args.hasFlag(options::OPT_grecord_command_line,
-                   options::OPT_gno_record_command_line, false);
-  auto FRecordSwitches =
-      Args.hasFlag(options::OPT_frecord_command_line,
-                   options::OPT_fno_record_command_line, false);
-  if (FRecordSwitches && !Triple.isOSBinFormatELF() &&
-      !Triple.isOSBinFormatXCOFF() && !Triple.isOSBinFormatMachO())
-    D.Diag(diag::err_drv_unsupported_opt_for_target)
-        << Args.getLastArg(options::OPT_frecord_command_line)->getAsString(Args)
-        << TripleStr;
-  if (TC.UseDwarfDebugFlags() || GRecordSwitches || FRecordSwitches) {
-    ArgStringList OriginalArgs;
-    for (const auto &Arg : Args)
-      Arg->render(Args, OriginalArgs);
-
-    SmallString<256> Flags;
-    EscapeSpacesAndBackslashes(Exec, Flags);
-    for (const char *OriginalArg : OriginalArgs) {
-      SmallString<128> EscapedArg;
-      EscapeSpacesAndBackslashes(OriginalArg, EscapedArg);
-      Flags += " ";
-      Flags += EscapedArg;
-    }
-    auto FlagsArgString = Args.MakeArgString(Flags);
+  auto GRecordSwitches = false;
+  auto FRecordSwitches = false;
+  if (shouldRecordCommandLine(TC, Args, FRecordSwitches, GRecordSwitches)) {
+    auto FlagsArgString = renderEscapedCommandLine(TC, Args);
     if (TC.UseDwarfDebugFlags() || GRecordSwitches) {
       CmdArgs.push_back("-dwarf-debug-flags");
       CmdArgs.push_back(FlagsArgString);
@@ -8785,10 +8746,10 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
 
     SmallString<256> Flags;
     const char *Exec = getToolChain().getDriver().getClangProgramPath();
-    EscapeSpacesAndBackslashes(Exec, Flags);
+    escapeSpacesAndBackslashes(Exec, Flags);
     for (const char *OriginalArg : OriginalArgs) {
       SmallString<128> EscapedArg;
-      EscapeSpacesAndBackslashes(OriginalArg, EscapedArg);
+      escapeSpacesAndBackslashes(OriginalArg, EscapedArg);
       Flags += " ";
       Flags += EscapedArg;
     }
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index cb5577e06e3e01fea6d64d8343ffeac02301fd69..7841f8b7125236e13437aa3d334652b496de0c51 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -2994,3 +2994,62 @@ void tools::handleColorDiagnosticsArgs(const Driver &D, const ArgList &Args,
   if (D.getDiags().getDiagnosticOptions().ShowColors)
     CmdArgs.push_back("-fcolor-diagnostics");
 }
+
+void tools::escapeSpacesAndBackslashes(const char *Arg,
+                                       llvm::SmallVectorImpl<char> &Res) {
+  for (; *Arg; ++Arg) {
+    switch (*Arg) {
+    default:
+      break;
+    case ' ':
+    case '\\':
+      Res.push_back('\\');
+      break;
+    }
+    Res.push_back(*Arg);
+  }
+}
+
+const char *tools::renderEscapedCommandLine(const ToolChain &TC,
+                                            const llvm::opt::ArgList &Args) {
+  const Driver &D = TC.getDriver();
+  const char *Exec = D.getClangProgramPath();
+
+  llvm::opt::ArgStringList OriginalArgs;
+  for (const auto &Arg : Args)
+    Arg->render(Args, OriginalArgs);
+
+  llvm::SmallString<256> Flags;
+  escapeSpacesAndBackslashes(Exec, Flags);
+  for (const char *OriginalArg : OriginalArgs) {
+    llvm::SmallString<128> EscapedArg;
+    escapeSpacesAndBackslashes(OriginalArg, EscapedArg);
+    Flags += " ";
+    Flags += EscapedArg;
+  }
+
+  return Args.MakeArgString(Flags);
+}
+
+bool tools::shouldRecordCommandLine(const ToolChain &TC,
+                                    const llvm::opt::ArgList &Args,
+                                    bool &FRecordCommandLine,
+                                    bool &GRecordCommandLine) {
+  const Driver &D = TC.getDriver();
+  const llvm::Triple &Triple = TC.getEffectiveTriple();
+  const std::string &TripleStr = Triple.getTriple();
+
+  FRecordCommandLine =
+      Args.hasFlag(options::OPT_frecord_command_line,
+                   options::OPT_fno_record_command_line, false);
+  GRecordCommandLine =
+      Args.hasFlag(options::OPT_grecord_command_line,
+                   options::OPT_gno_record_command_line, false);
+  if (FRecordCommandLine && !Triple.isOSBinFormatELF() &&
+      !Triple.isOSBinFormatXCOFF() && !Triple.isOSBinFormatMachO())
+    D.Diag(diag::err_drv_unsupported_opt_for_target)
+        << Args.getLastArg(options::OPT_frecord_command_line)->getAsString(Args)
+        << TripleStr;
+
+  return FRecordCommandLine || TC.UseDwarfDebugFlags() || GRecordCommandLine;
+}
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
index eff21b210b4244306a6236391383767001bec612..9cafac2538862ae82609b8b54027cc85e92ac951 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -237,6 +237,31 @@ void addMCModel(const Driver &D, const llvm::opt::ArgList &Args,
 void handleColorDiagnosticsArgs(const Driver &D, const llvm::opt::ArgList &Args,
                                 llvm::opt::ArgStringList &CmdArgs);
 
+/// Add backslashes to escape spaces and other backslashes.
+/// This is used for the space-separated argument list specified with
+/// the -dwarf-debug-flags option.
+void escapeSpacesAndBackslashes(const char *Arg,
+                                llvm::SmallVectorImpl<char> &Res);
+
+/// Join the args in the given ArgList, escape spaces and backslashes and
+/// return the joined string. This is used when saving the command line as a
+/// result of using either the -frecord-command-line or -grecord-command-line
+/// options. The lifetime of the returned c-string will match that of the Args
+/// argument.
+const char *renderEscapedCommandLine(const ToolChain &TC,
+                                     const llvm::opt::ArgList &Args);
+
+/// Check if the command line should be recorded in the object file. This is
+/// done if either -frecord-command-line or -grecord-command-line options have
+/// been passed. This also does some error checking since -frecord-command-line
+/// is currently only supported on ELF platforms. The last two boolean
+/// arguments are out parameters and will be set depending on the command
+/// line options that were passed.
+bool shouldRecordCommandLine(const ToolChain &TC,
+                             const llvm::opt::ArgList &Args,
+                             bool &FRecordCommandLine,
+                             bool &GRecordCommandLine);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 7a70cf1c5694fd6eb5493540d4f12e45f98ee4fc..b368c473756731beb5e79df43cf8528c22faf7d6 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -87,6 +87,8 @@ CudaVersion getCudaVersion(uint32_t raw_version) {
     return CudaVersion::CUDA_124;
   if (raw_version < 12060)
     return CudaVersion::CUDA_125;
+  if (raw_version < 12070)
+    return CudaVersion::CUDA_126;
   return CudaVersion::NEW;
 }
 
@@ -669,6 +671,7 @@ void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   case CudaVersion::CUDA_##CUDA_VER:                                           \
     PtxFeature = "+ptx" #PTX_VER;                                              \
     break;
+    CASE_CUDA_VERSION(126, 85);
     CASE_CUDA_VERSION(125, 85);
     CASE_CUDA_VERSION(124, 84);
     CASE_CUDA_VERSION(123, 83);
@@ -691,6 +694,10 @@ void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,
     CASE_CUDA_VERSION(91, 61);
     CASE_CUDA_VERSION(90, 60);
 #undef CASE_CUDA_VERSION
+  // TODO: Use specific CUDA version once it's public.
+  case clang::CudaVersion::NEW:
+    PtxFeature = "+ptx86";
+    break;
   default:
     PtxFeature = "+ptx42";
   }
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 19b43594b00815919c64d98296b7a67c500c39e3..e2f8f6e0cca1c6d4cdf5c4a85f11cabe14d7eb91 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -882,6 +882,20 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
   addDashXForInput(Args, Input, CmdArgs);
 
+  bool FRecordCmdLine = false;
+  bool GRecordCmdLine = false;
+  if (shouldRecordCommandLine(TC, Args, FRecordCmdLine, GRecordCmdLine)) {
+    const char *CmdLine = renderEscapedCommandLine(TC, Args);
+    if (FRecordCmdLine) {
+      CmdArgs.push_back("-record-command-line");
+      CmdArgs.push_back(CmdLine);
+    }
+    if (TC.UseDwarfDebugFlags() || GRecordCmdLine) {
+      CmdArgs.push_back("-dwarf-debug-flags");
+      CmdArgs.push_back(CmdLine);
+    }
+  }
+
   CmdArgs.push_back(Input.getFilename());
 
   const char *Exec = Args.MakeArgString(D.GetProgramPath("flang", TC));
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index eea61cc42634ae7523f7e42e293fce7451e8f2b3..2410fb86b39a72c145e6c3e4799334ca3be3f0ae 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -671,7 +671,7 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  Args.AddAllArgs(CmdArgs, options::OPT_T);
+  Args.addAllArgs(CmdArgs, {options::OPT_T, options::OPT_t});
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
   C.addCommand(std::make_unique<Command>(JA, *this,
@@ -2703,6 +2703,7 @@ void Generic_GCC::GCCInstallationDetector::AddDefaultGCCPrefixes(
   case llvm::Triple::thumb:
     LibDirs.append(begin(ARMLibDirs), end(ARMLibDirs));
     if (TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHF ||
+        TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHFT64 ||
         TargetTriple.getEnvironment() == llvm::Triple::MuslEABIHF ||
         TargetTriple.getEnvironment() == llvm::Triple::EABIHF) {
       TripleAliases.append(begin(ARMHFTriples), end(ARMHFTriples));
@@ -2714,6 +2715,7 @@ void Generic_GCC::GCCInstallationDetector::AddDefaultGCCPrefixes(
   case llvm::Triple::thumbeb:
     LibDirs.append(begin(ARMebLibDirs), end(ARMebLibDirs));
     if (TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHF ||
+        TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHFT64 ||
         TargetTriple.getEnvironment() == llvm::Triple::MuslEABIHF ||
         TargetTriple.getEnvironment() == llvm::Triple::EABIHF) {
       TripleAliases.append(begin(ARMebHFTriples), end(ARMebHFTriples));
diff --git a/clang/lib/Driver/ToolChains/HIPUtility.cpp b/clang/lib/Driver/ToolChains/HIPUtility.cpp
index b3adfe65402ff350796d2c9a04bf369444061a6a..9fe4f1e0e20965df027c3bc430734dff4da872c4 100644
--- a/clang/lib/Driver/ToolChains/HIPUtility.cpp
+++ b/clang/lib/Driver/ToolChains/HIPUtility.cpp
@@ -340,21 +340,21 @@ void HIP::constructHIPFatbinCommand(Compilation &C, const JobAction &JA,
 void HIP::constructGenerateObjFileFromHIPFatBinary(
     Compilation &C, const InputInfo &Output, const InputInfoList &Inputs,
     const ArgList &Args, const JobAction &JA, const Tool &T) {
-  const ToolChain &TC = T.getToolChain();
+  const Driver &D = C.getDriver();
   std::string Name = std::string(llvm::sys::path::stem(Output.getFilename()));
 
   // Create Temp Object File Generator,
   // Offload Bundled file and Bundled Object file.
   // Keep them if save-temps is enabled.
-  const char *McinFile;
+  const char *ObjinFile;
   const char *BundleFile;
-  if (C.getDriver().isSaveTempsEnabled()) {
-    McinFile = C.getArgs().MakeArgString(Name + ".mcin");
+  if (D.isSaveTempsEnabled()) {
+    ObjinFile = C.getArgs().MakeArgString(Name + ".mcin");
     BundleFile = C.getArgs().MakeArgString(Name + ".hipfb");
   } else {
-    auto TmpNameMcin = C.getDriver().GetTemporaryPath(Name, "mcin");
-    McinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin));
-    auto TmpNameFb = C.getDriver().GetTemporaryPath(Name, "hipfb");
+    auto TmpNameMcin = D.GetTemporaryPath(Name, "mcin");
+    ObjinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin));
+    auto TmpNameFb = D.GetTemporaryPath(Name, "hipfb");
     BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameFb));
   }
   HIP::constructHIPFatbinCommand(C, JA, BundleFile, Inputs, Args, T);
@@ -454,19 +454,20 @@ void HIP::constructGenerateObjFileFromHIPFatBinary(
 
   // Open script file and write the contents.
   std::error_code EC;
-  llvm::raw_fd_ostream Objf(McinFile, EC, llvm::sys::fs::OF_None);
+  llvm::raw_fd_ostream Objf(ObjinFile, EC, llvm::sys::fs::OF_None);
 
   if (EC) {
-    C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message();
+    D.Diag(clang::diag::err_unable_to_make_temp) << EC.message();
     return;
   }
 
   Objf << ObjBuffer;
 
-  ArgStringList McArgs{"-triple", Args.MakeArgString(HostTriple.normalize()),
+  ArgStringList McArgs{"-target", Args.MakeArgString(HostTriple.normalize()),
                        "-o",      Output.getFilename(),
-                       McinFile,  "--filetype=obj"};
-  const char *Mc = Args.MakeArgString(TC.GetProgramPath("llvm-mc"));
-  C.addCommand(std::make_unique<Command>(JA, T, ResponseFileSupport::None(), Mc,
-                                         McArgs, Inputs, Output));
+                       "-x",      "assembler",
+                       ObjinFile, "-c"};
+  C.addCommand(std::make_unique<Command>(JA, T, ResponseFileSupport::None(),
+                                         D.getClangProgramPath(), McArgs,
+                                         Inputs, Output, D.getPrependArg()));
 }
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index a1646fbb0c358b90b989f3b1313f800af47283f8..fa03236c0ce9253e8ba1e6bdd3c26bbde7715976 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -525,6 +525,7 @@ std::string Linux::getDynamicLinker(const ArgList &Args) const {
   case llvm::Triple::thumbeb: {
     const bool HF =
         Triple.getEnvironment() == llvm::Triple::GNUEABIHF ||
+        Triple.getEnvironment() == llvm::Triple::GNUEABIHFT64 ||
         tools::arm::getARMFloatABI(*this, Args) == tools::arm::FloatABI::Hard;
 
     LibDir = "lib";
diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp
index 9aec11e69fde1dbabd0592331a7077dd3036c92d..44a6894d30fb29be96f58db391e38689cf39fb4e 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -163,6 +163,8 @@ void wasm::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     AddRunTimeLibs(ToolChain, ToolChain.getDriver(), CmdArgs, Args);
   }
 
+  ToolChain.addProfileRTLibs(Args, CmdArgs);
+
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp
index 75304908dc65066b2017327f71544f99350819bf..bde77578769906a86ff341a13b17eb256111d481 100644
--- a/clang/lib/Format/BreakableToken.cpp
+++ b/clang/lib/Format/BreakableToken.cpp
@@ -420,7 +420,7 @@ BreakableComment::getSplit(unsigned LineIndex, unsigned TailOffset,
                            unsigned ColumnLimit, unsigned ContentStartColumn,
                            const llvm::Regex &CommentPragmasRegex) const {
   // Don't break lines matching the comment pragmas regex.
-  if (CommentPragmasRegex.match(Content[LineIndex]))
+  if (!AlwaysReflow || CommentPragmasRegex.match(Content[LineIndex]))
     return Split(StringRef::npos, 0);
   return getCommentSplit(Content[LineIndex].substr(TailOffset),
                          ContentStartColumn, ColumnLimit, Style.TabWidth,
@@ -608,7 +608,7 @@ BreakableToken::Split BreakableBlockComment::getSplit(
     unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
     unsigned ContentStartColumn, const llvm::Regex &CommentPragmasRegex) const {
   // Don't break lines matching the comment pragmas regex.
-  if (CommentPragmasRegex.match(Content[LineIndex]))
+  if (!AlwaysReflow || CommentPragmasRegex.match(Content[LineIndex]))
     return Split(StringRef::npos, 0);
   return getCommentSplit(Content[LineIndex].substr(TailOffset),
                          ContentStartColumn, ColumnLimit, Style.TabWidth,
@@ -855,7 +855,8 @@ bool BreakableBlockComment::mayReflow(
   StringRef IndentContent = Content[LineIndex];
   if (Lines[LineIndex].ltrim(Blanks).starts_with("*"))
     IndentContent = Lines[LineIndex].ltrim(Blanks).substr(1);
-  return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
+  return LineIndex > 0 && AlwaysReflow &&
+         !CommentPragmasRegex.match(IndentContent) &&
          mayReflowContent(Content[LineIndex]) && !Tok.Finalized &&
          !switchesFormatting(tokenAt(LineIndex));
 }
@@ -1160,7 +1161,8 @@ bool BreakableLineCommentSection::mayReflow(
   // // text that protrudes
   // //    into text with different indent
   // We do reflow in that case in block comments.
-  return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
+  return LineIndex > 0 && AlwaysReflow &&
+         !CommentPragmasRegex.match(IndentContent) &&
          mayReflowContent(Content[LineIndex]) && !Tok.Finalized &&
          !switchesFormatting(tokenAt(LineIndex)) &&
          OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1];
diff --git a/clang/lib/Format/BreakableToken.h b/clang/lib/Format/BreakableToken.h
index 8b9360a3335ef41965d5448793a358417d85f4df..45c00b35fd01ede5161e8d04d4ea291bf744e27d 100644
--- a/clang/lib/Format/BreakableToken.h
+++ b/clang/lib/Format/BreakableToken.h
@@ -384,6 +384,8 @@ protected:
   // The intended start column of the first line of text from this section.
   unsigned StartColumn;
 
+  const bool AlwaysReflow = Style.ReflowComments == FormatStyle::RCS_Always;
+
   // The prefix to use in front a line that has been reflown up.
   // For example, when reflowing the second line after the first here:
   // // comment 1
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 4df8dc89be459fc7c913c3638791da609489829e..fd53969e4b3b330fd5ad4f5bc471840cbfc2596c 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -2471,7 +2471,7 @@ ContinuationIndenter::createBreakableToken(const FormatToken &Current,
           State.Line->InPPDirective, Encoding, Style);
     }
   } else if (Current.is(TT_BlockComment)) {
-    if (!Style.ReflowComments ||
+    if (Style.ReflowComments == FormatStyle::RCS_Never ||
         // If a comment token switches formatting, like
         // /* clang-format on */, we don't want to break it further,
         // but we may still want to adjust its indentation.
@@ -2492,7 +2492,7 @@ ContinuationIndenter::createBreakableToken(const FormatToken &Current,
       }
       return true;
     }();
-    if (!Style.ReflowComments ||
+    if (Style.ReflowComments == FormatStyle::RCS_Never ||
         CommentPragmasRegex.match(Current.TokenText.substr(2)) ||
         switchesFormatting(Current) || !RegularComments) {
       return nullptr;
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index f97127f09d2096ba53c82c422fc1a8ef7587df63..148270795c562f81e4f869f9ddfa512dc5b69a6f 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -527,6 +527,17 @@ template <> struct MappingTraits<FormatStyle::RawStringFormat> {
   }
 };
 
+template <> struct ScalarEnumerationTraits<FormatStyle::ReflowCommentsStyle> {
+  static void enumeration(IO &IO, FormatStyle::ReflowCommentsStyle &Value) {
+    IO.enumCase(Value, "Never", FormatStyle::RCS_Never);
+    IO.enumCase(Value, "IndentOnly", FormatStyle::RCS_IndentOnly);
+    IO.enumCase(Value, "Always", FormatStyle::RCS_Always);
+    // For backward compatibility:
+    IO.enumCase(Value, "false", FormatStyle::RCS_Never);
+    IO.enumCase(Value, "true", FormatStyle::RCS_Always);
+  }
+};
+
 template <>
 struct ScalarEnumerationTraits<FormatStyle::ReferenceAlignmentStyle> {
   static void enumeration(IO &IO, FormatStyle::ReferenceAlignmentStyle &Value) {
@@ -1569,7 +1580,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.PPIndentWidth = -1;
   LLVMStyle.QualifierAlignment = FormatStyle::QAS_Leave;
   LLVMStyle.ReferenceAlignment = FormatStyle::RAS_Pointer;
-  LLVMStyle.ReflowComments = true;
+  LLVMStyle.ReflowComments = FormatStyle::RCS_Always;
   LLVMStyle.RemoveBracesLLVM = false;
   LLVMStyle.RemoveParentheses = FormatStyle::RPS_Leave;
   LLVMStyle.RemoveSemicolon = false;
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 364d7e9855e8cf5c8b86ea3ab545ad8508fceaab..fcefaa7bb298eabd7358749d63a7c1ce772d5e13 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -1175,26 +1175,19 @@ private:
 
     ScopedContextCreator ContextCreator(*this, tok::l_brace, 1);
     Contexts.back().ColonIsDictLiteral = true;
-
-    const auto *Prev = OpeningBrace.getPreviousNonComment();
-
-    if (OpeningBrace.is(BK_BracedInit)) {
+    if (OpeningBrace.is(BK_BracedInit))
       Contexts.back().IsExpression = true;
-      if (Prev) {
-        for (auto *Tok = Prev->Previous; Tok && Tok->isPointerOrReference();
-             Tok = Tok->Previous) {
-          Tok->setFinalizedType(TT_PointerOrReference);
-        }
-      }
-    }
-
-    if (Style.isJavaScript() && Prev && Prev->is(TT_JsTypeColon))
+    if (Style.isJavaScript() && OpeningBrace.Previous &&
+        OpeningBrace.Previous->is(TT_JsTypeColon)) {
       Contexts.back().IsExpression = false;
-
+    }
+    if (Style.isVerilog() &&
+        (!OpeningBrace.getPreviousNonComment() ||
+         OpeningBrace.getPreviousNonComment()->isNot(Keywords.kw_apostrophe))) {
+      Contexts.back().VerilogMayBeConcatenation = true;
+    }
     if (Style.isTableGen())
       Contexts.back().ColonIsDictLiteral = false;
-    else if (Style.isVerilog() && !(Prev && Prev->is(Keywords.kw_apostrophe)))
-      Contexts.back().VerilogMayBeConcatenation = true;
 
     unsigned CommaCount = 0;
     while (CurrentToken) {
@@ -1551,7 +1544,8 @@ private:
           // Case D.
           if (Keywords.isVerilogIdentifier(*Prev) && PrevPrev->is(tok::comma)) {
             const FormatToken *PrevParen = PrevPrev->getPreviousNonComment();
-            if (PrevParen->is(tok::r_paren) && PrevParen->MatchingParen &&
+            if (PrevParen && PrevParen->is(tok::r_paren) &&
+                PrevParen->MatchingParen &&
                 PrevParen->MatchingParen->is(TT_VerilogInstancePortLParen)) {
               return true;
             }
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 9c4a065fd93b96300ce7fedb529113b8211771ae..c9625c39e527b4ac9670169a9c94164359b77ffc 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -4623,9 +4623,9 @@ bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
 // section on \p Line.
 static bool
 continuesLineCommentSection(const FormatToken &FormatTok,
-                            const UnwrappedLine &Line,
+                            const UnwrappedLine &Line, const FormatStyle &Style,
                             const llvm::Regex &CommentPragmasRegex) {
-  if (Line.Tokens.empty())
+  if (Line.Tokens.empty() || Style.ReflowComments != FormatStyle::RCS_Always)
     return false;
 
   StringRef IndentContent = FormatTok.TokenText;
@@ -4738,7 +4738,7 @@ void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
     // FIXME: Consider putting separate line comment sections as children to the
     // unwrapped line instead.
     Tok->ContinuesLineCommentSection =
-        continuesLineCommentSection(*Tok, *Line, CommentPragmasRegex);
+        continuesLineCommentSection(*Tok, *Line, Style, CommentPragmasRegex);
     if (isOnNewLine(*Tok) && JustComments && !Tok->ContinuesLineCommentSection)
       addUnwrappedLine();
     pushToken(Tok);
@@ -4811,8 +4811,8 @@ void UnwrappedLineParser::distributeComments(
     if (HasTrailAlignedWithNextToken && i == StartOfTrailAlignedWithNextToken) {
       FormatTok->ContinuesLineCommentSection = false;
     } else {
-      FormatTok->ContinuesLineCommentSection =
-          continuesLineCommentSection(*FormatTok, *Line, CommentPragmasRegex);
+      FormatTok->ContinuesLineCommentSection = continuesLineCommentSection(
+          *FormatTok, *Line, Style, CommentPragmasRegex);
     }
     if (!FormatTok->ContinuesLineCommentSection &&
         (isOnNewLine(*FormatTok) || FormatTok->IsFirst)) {
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index e4b462b9b0fd81b0c04b35406935b5aaa134eaec..64f90c493c1055ddef06d4924c2d02cabee2ad3d 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -457,8 +457,6 @@ private:
       return "BuildingDeductionGuides";
     case CodeSynthesisContext::TypeAliasTemplateInstantiation:
       return "TypeAliasTemplateInstantiation";
-    case CodeSynthesisContext::PartialOrderingTTP:
-      return "PartialOrderingTTP";
     }
     return "";
   }
diff --git a/clang/lib/Headers/intrin0.h b/clang/lib/Headers/intrin0.h
index 866c8896617d22aea66ef301f68a623318bddc6e..6b01f3808652aa8077c4f0448d41681f250686c3 100644
--- a/clang/lib/Headers/intrin0.h
+++ b/clang/lib/Headers/intrin0.h
@@ -44,7 +44,7 @@ unsigned char _InterlockedCompareExchange128_rel(__int64 volatile *_Destination,
                                                  __int64 *_ComparandResult);
 #endif
 
-#ifdef __x86_64__ && !defined(__arm64ec__)
+#if defined(__x86_64__) && !defined(__arm64ec__)
 unsigned __int64 _umul128(unsigned __int64, unsigned __int64,
                           unsigned __int64 *);
 unsigned __int64 __shiftleft128(unsigned __int64 _LowPart,
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index f0b4593e0cc22e1d167257154497876c35498a9c..ecc5166d7b814c9b0be0b681d8c2dba0c6dd7188 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -902,6 +902,10 @@ void Preprocessor::Lex(Token &Result) {
     case tok::r_brace:
       StdCXXImportSeqState.handleCloseBrace();
       break;
+#define PRAGMA_ANNOTATION(X) case tok::annot_##X:
+// For `#pragma ...` mimic ';'.
+#include "clang/Basic/TokenKinds.def"
+#undef PRAGMA_ANNOTATION
     // This token is injected to represent the translation of '#include "a.h"'
     // into "import a.h;". Mimic the notional ';'.
     case tok::annot_module_include:
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index b27e50b147f4a8e2343a3dbdbc7c33dec5385349..51d4dc38c17f670727bf01ae02da2d65416075c7 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -797,23 +797,26 @@ bool Parser::ParseOpenACCSizeExprList(
 /// [num:]int-expr
 /// dim:int-expr
 /// static:size-expr
-bool Parser::ParseOpenACCGangArg(SourceLocation GangLoc) {
+Parser::OpenACCGangArgRes Parser::ParseOpenACCGangArg(SourceLocation GangLoc) {
 
   if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::Static, getCurToken()) &&
       NextToken().is(tok::colon)) {
     // 'static' just takes a size-expr, which is an int-expr or an asterisk.
     ConsumeToken();
     ConsumeToken();
-    return ParseOpenACCSizeExpr(OpenACCClauseKind::Gang).isInvalid();
+    ExprResult Res = ParseOpenACCSizeExpr(OpenACCClauseKind::Gang);
+    return {OpenACCGangKind::Static, Res};
   }
 
   if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::Dim, getCurToken()) &&
       NextToken().is(tok::colon)) {
     ConsumeToken();
     ConsumeToken();
-    return ParseOpenACCIntExpr(OpenACCDirectiveKind::Invalid,
-                               OpenACCClauseKind::Gang, GangLoc)
-        .first.isInvalid();
+    // Parse this as a const-expression, and we'll check its integer-ness/value
+    // in CheckGangExpr.
+    ExprResult Res =
+        getActions().CorrectDelayedTyposInExpr(ParseConstantExpression());
+    return {OpenACCGangKind::Dim, Res};
   }
 
   if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::Num, getCurToken()) &&
@@ -822,27 +825,40 @@ bool Parser::ParseOpenACCGangArg(SourceLocation GangLoc) {
     ConsumeToken();
     // Fallthrough to the 'int-expr' handling for when 'num' is omitted.
   }
+
   // This is just the 'num' case where 'num' is optional.
-  return ParseOpenACCIntExpr(OpenACCDirectiveKind::Invalid,
-                             OpenACCClauseKind::Gang, GangLoc)
-      .first.isInvalid();
+  ExprResult Res = ParseOpenACCIntExpr(OpenACCDirectiveKind::Invalid,
+                                       OpenACCClauseKind::Gang, GangLoc)
+                       .first;
+  return {OpenACCGangKind::Num, Res};
 }
 
-bool Parser::ParseOpenACCGangArgList(SourceLocation GangLoc) {
-  if (ParseOpenACCGangArg(GangLoc)) {
+bool Parser::ParseOpenACCGangArgList(
+    SourceLocation GangLoc, llvm::SmallVectorImpl<OpenACCGangKind> &GKs,
+    llvm::SmallVectorImpl<Expr *> &IntExprs) {
+
+  Parser::OpenACCGangArgRes Res = ParseOpenACCGangArg(GangLoc);
+  if (!Res.second.isUsable()) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
               Parser::StopBeforeMatch);
-    return false;
+    return true;
   }
 
+  GKs.push_back(Res.first);
+  IntExprs.push_back(Res.second.get());
+
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
 
-    if (ParseOpenACCGangArg(GangLoc)) {
+    Res = ParseOpenACCGangArg(GangLoc);
+    if (!Res.second.isUsable()) {
       SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
                 Parser::StopBeforeMatch);
-      return false;
+      return true;
     }
+
+    GKs.push_back(Res.first);
+    IntExprs.push_back(Res.second.get());
   }
   return false;
 }
@@ -1114,6 +1130,7 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
           Parens.skipToEnd();
           return OpenACCCanContinue();
         }
+        ParsedClause.setIntExprDetails(IntExpr.get());
         break;
       }
       case OpenACCClauseKind::Async: {
@@ -1129,12 +1146,16 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
         }
         break;
       }
-      case OpenACCClauseKind::Gang:
-        if (ParseOpenACCGangArgList(ClauseLoc)) {
+      case OpenACCClauseKind::Gang: {
+        llvm::SmallVector<OpenACCGangKind> GKs;
+        llvm::SmallVector<Expr *> IntExprs;
+        if (ParseOpenACCGangArgList(ClauseLoc, GKs, IntExprs)) {
           Parens.skipToEnd();
           return OpenACCCanContinue();
         }
+        ParsedClause.setGangDetails(std::move(GKs), std::move(IntExprs));
         break;
+      }
       case OpenACCClauseKind::Wait: {
         OpenACCWaitParseInfo Info =
             ParseOpenACCWaitArgument(ClauseLoc,
diff --git a/clang/lib/Sema/CheckExprLifetime.cpp b/clang/lib/Sema/CheckExprLifetime.cpp
index 009b8d000e6b0e3e5489d71ab399231e3061d994..9b3894767d86291a4fd38dbf9df42267bc3ba344 100644
--- a/clang/lib/Sema/CheckExprLifetime.cpp
+++ b/clang/lib/Sema/CheckExprLifetime.cpp
@@ -404,7 +404,7 @@ shouldTrackFirstArgumentForConstructor(const CXXConstructExpr *Ctor) {
   if (LHSRecordDecl->hasAttr<PointerAttr>())
     return true;
 
-  if (Ctor->getConstructor()->getNumParams() != 1 ||
+  if (Ctor->getConstructor()->param_empty() ||
       !isContainerOfPointer(LHSRecordDecl))
     return false;
 
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 9f91ee9a39f2f99e76d38fd6a442e46d77903f31..f0d1634af529f0b86055926cc41fa9357a9e31c5 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -2385,10 +2385,11 @@ FunctionScopeInfo *Sema::getEnclosingFunction() const {
   return nullptr;
 }
 
-LambdaScopeInfo *Sema::getEnclosingLambda() const {
+CapturingScopeInfo *Sema::getEnclosingLambdaOrBlock() const {
   for (auto *Scope : llvm::reverse(FunctionScopes)) {
-    if (auto *LSI = dyn_cast<sema::LambdaScopeInfo>(Scope)) {
-      if (LSI->Lambda && !LSI->Lambda->Encloses(CurContext) &&
+    if (auto *CSI = dyn_cast<CapturingScopeInfo>(Scope)) {
+      auto *LSI = dyn_cast<LambdaScopeInfo>(CSI);
+      if (LSI && LSI->Lambda && !LSI->Lambda->Encloses(CurContext) &&
           LSI->AfterParameterList) {
         // We have switched contexts due to template instantiation.
         // FIXME: We should swap out the FunctionScopes during code synthesis
@@ -2396,7 +2397,7 @@ LambdaScopeInfo *Sema::getEnclosingLambda() const {
         assert(!CodeSynthesisContexts.empty());
         return nullptr;
       }
-      return LSI;
+      return CSI;
     }
   }
   return nullptr;
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index cf2a5a622a3a4d38d9653f91b607fa4b694d5759..68a8dfaf1f6183cb819dab36ff3a5a208262bb25 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -750,12 +750,10 @@ bool Sema::UnifySection(StringRef SectionName, int SectionFlags,
   if (auto A = Decl->getAttr<SectionAttr>())
     if (A->isImplicit())
       PragmaLocation = A->getLocation();
-  auto SectionIt = Context.SectionInfos.find(SectionName);
-  if (SectionIt == Context.SectionInfos.end()) {
-    Context.SectionInfos[SectionName] =
-        ASTContext::SectionInfo(Decl, PragmaLocation, SectionFlags);
+  auto [SectionIt, Inserted] = Context.SectionInfos.try_emplace(
+      SectionName, Decl, PragmaLocation, SectionFlags);
+  if (Inserted)
     return false;
-  }
   // A pre-declared section takes precedence w/o diagnostic.
   const auto &Section = SectionIt->second;
   if (Section.SectionFlags == SectionFlags ||
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 998a148a7d24a14d9c1b93e9fc863ed6be41116e..e36ee06221371668e66946d6d0e4db1fa25f389b 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -585,8 +585,8 @@ static bool CheckConstraintSatisfaction(
 
   ArrayRef<TemplateArgument> TemplateArgs =
       TemplateArgsLists.getNumSubstitutedLevels() > 0
-          ? TemplateArgsLists.getOutermost()
-          : ArrayRef<TemplateArgument> {};
+          ? TemplateArgsLists.getInnermost()
+          : ArrayRef<TemplateArgument>{};
   Sema::InstantiatingTemplate Inst(S, TemplateIDRange.getBegin(),
       Sema::InstantiatingTemplate::ConstraintsCheck{},
       const_cast<NamedDecl *>(Template), TemplateArgs, TemplateIDRange);
@@ -834,7 +834,6 @@ Sema::SetupConstraintCheckingTemplateArgumentsAndScope(
       getTemplateInstantiationArgs(FD, FD->getLexicalDeclContext(),
                                    /*Final=*/false, /*Innermost=*/std::nullopt,
                                    /*RelativeToPrimary=*/true,
-                                   /*Pattern=*/nullptr,
                                    /*ForConstraintInstantiation=*/true);
   if (SetupConstraintScope(FD, TemplateArgs, MLTAL, Scope))
     return std::nullopt;
@@ -910,15 +909,13 @@ bool Sema::CheckFunctionConstraints(const FunctionDecl *FD,
 // Figure out the to-translation-unit depth for this function declaration for
 // the purpose of seeing if they differ by constraints. This isn't the same as
 // getTemplateDepth, because it includes already instantiated parents.
-static unsigned
-CalculateTemplateDepthForConstraints(Sema &S, const NamedDecl *ND,
-                                     bool SkipForSpecialization = false) {
+static unsigned CalculateTemplateDepthForConstraints(Sema &S,
+                                                     const NamedDecl *ND) {
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
       ND, ND->getLexicalDeclContext(), /*Final=*/false,
       /*Innermost=*/std::nullopt,
       /*RelativeToPrimary=*/true,
-      /*Pattern=*/nullptr,
-      /*ForConstraintInstantiation=*/true, SkipForSpecialization);
+      /*ForConstraintInstantiation=*/true);
   return MLTAL.getNumLevels();
 }
 
@@ -957,8 +954,7 @@ static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
       DeclInfo.getDecl(), DeclInfo.getLexicalDeclContext(), /*Final=*/false,
       /*Innermost=*/std::nullopt,
       /*RelativeToPrimary=*/true,
-      /*Pattern=*/nullptr, /*ForConstraintInstantiation=*/true,
-      /*SkipForSpecialization*/ false);
+      /*ForConstraintInstantiation=*/true);
 
   if (MLTAL.getNumSubstitutedLevels() == 0)
     return ConstrExpr;
@@ -1068,16 +1064,16 @@ bool Sema::AreConstraintExpressionsEqual(const NamedDecl *Old,
 bool Sema::FriendConstraintsDependOnEnclosingTemplate(const FunctionDecl *FD) {
   assert(FD->getFriendObjectKind() && "Must be a friend!");
 
+  FunctionTemplateDecl *FTD = FD->getDescribedFunctionTemplate();
   // The logic for non-templates is handled in ASTContext::isSameEntity, so we
   // don't have to bother checking 'DependsOnEnclosingTemplate' for a
   // non-function-template.
-  assert(FD->getDescribedFunctionTemplate() &&
-         "Non-function templates don't need to be checked");
+  assert(FTD && "Non-function templates don't need to be checked");
 
   SmallVector<const Expr *, 3> ACs;
-  FD->getDescribedFunctionTemplate()->getAssociatedConstraints(ACs);
+  FTD->getAssociatedConstraints(ACs);
 
-  unsigned OldTemplateDepth = CalculateTemplateDepthForConstraints(*this, FD);
+  unsigned OldTemplateDepth = FTD->getTemplateParameters()->getDepth();
   for (const Expr *Constraint : ACs)
     if (ConstraintExpressionDependsOnEnclosingTemplate(FD, OldTemplateDepth,
                                                        Constraint))
@@ -1524,7 +1520,6 @@ static bool substituteParameterMappings(Sema &S, NormalizedConstraint &N,
       CSE->getNamedConcept(), CSE->getNamedConcept()->getLexicalDeclContext(),
       /*Final=*/false, CSE->getTemplateArguments(),
       /*RelativeToPrimary=*/true,
-      /*Pattern=*/nullptr,
       /*ForConstraintInstantiation=*/true);
 
   return substituteParameterMappings(S, N, CSE->getNamedConcept(), MLTAL,
@@ -1805,8 +1800,8 @@ bool Sema::IsAtLeastAsConstrained(NamedDecl *D1,
     return false;
   }
 
-  unsigned Depth1 = CalculateTemplateDepthForConstraints(*this, D1, true);
-  unsigned Depth2 = CalculateTemplateDepthForConstraints(*this, D2, true);
+  unsigned Depth1 = CalculateTemplateDepthForConstraints(*this, D1);
+  unsigned Depth2 = CalculateTemplateDepthForConstraints(*this, D2);
 
   for (size_t I = 0; I != AC1.size() && I != AC2.size(); ++I) {
     if (Depth2 > Depth1) {
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index b4494de060538b352cc6506e6c9dd9bca0c9cbbb..5642c821f4ad7cb986f45eae8ee0d47501bdbe17 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -357,10 +357,13 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
       return nullptr;
   }
 
-  // FIXME: LookupNestedNameSpecifierName isn't the right kind of
-  // lookup for class-names.
-  LookupNameKind Kind = isClassName ? LookupNestedNameSpecifierName :
-                                      LookupOrdinaryName;
+  // In the case where we know that the identifier is a class name, we know that
+  // it is a type declaration (struct, class, union or enum) so we can use tag
+  // name lookup.
+  //
+  // C++ [class.derived]p2 (wrt lookup in a base-specifier): The lookup for
+  // the component name of the type-name or simple-template-id is type-only.
+  LookupNameKind Kind = isClassName ? LookupTagName : LookupOrdinaryName;
   LookupResult Result(*this, &II, NameLoc, Kind);
   if (LookupCtx) {
     // Perform "qualified" name lookup into the declaration context we
@@ -4510,10 +4513,10 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
   adjustDeclContextForDeclaratorDecl(New, Old);
 
   // Ensure the template parameters are compatible.
-  if (NewTemplate &&
-      !TemplateParameterListsAreEqual(NewTemplate->getTemplateParameters(),
-                                      OldTemplate->getTemplateParameters(),
-                                      /*Complain=*/true, TPL_TemplateMatch))
+  if (NewTemplate && !TemplateParameterListsAreEqual(
+                         NewTemplate, NewTemplate->getTemplateParameters(),
+                         OldTemplate, OldTemplate->getTemplateParameters(),
+                         /*Complain=*/true, TPL_TemplateMatch))
     return New->setInvalidDecl();
 
   // C++ [class.mem]p1:
@@ -7663,7 +7666,7 @@ NamedDecl *Sema::ActOnVariableDeclarator(
               : SourceLocation();
       DeclResult Res = ActOnVarTemplateSpecialization(
           S, D, TInfo, Previous, TemplateKWLoc, TemplateParams, SC,
-          IsPartialSpecialization);
+          IsPartialSpecialization, IsMemberSpecialization);
       if (Res.isInvalid())
         return nullptr;
       NewVD = cast<VarDecl>(Res.get());
@@ -7682,6 +7685,10 @@ NamedDecl *Sema::ActOnVariableDeclarator(
           VarTemplateDecl::Create(Context, DC, D.getIdentifierLoc(), Name,
                                   TemplateParams, NewVD);
       NewVD->setDescribedVarTemplate(NewTemplate);
+      // If we are providing an explicit specialization of a static variable
+      // template, make a note of that.
+      if (IsMemberSpecialization)
+        NewTemplate->setMemberSpecialization();
     }
 
     // If this decl has an auto type in need of deduction, make a note of the
@@ -8059,12 +8066,6 @@ NamedDecl *Sema::ActOnVariableDeclarator(
                   ? TPC_ClassTemplateMember
                   : TPC_VarTemplate))
         NewVD->setInvalidDecl();
-
-      // If we are providing an explicit specialization of a static variable
-      // template, make a note of that.
-      if (PrevVarTemplate &&
-          PrevVarTemplate->getInstantiatedFromMemberTemplate())
-        PrevVarTemplate->setMemberSpecialization();
     }
   }
 
@@ -9885,6 +9886,8 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
                                                         NewFD);
         FunctionTemplate->setLexicalDeclContext(CurContext);
         NewFD->setDescribedFunctionTemplate(FunctionTemplate);
+        if (isMemberSpecialization)
+          FunctionTemplate->setMemberSpecialization();
 
         // For source fidelity, store the other template param lists.
         if (TemplateParamLists.size() > 1) {
@@ -12042,10 +12045,7 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
 
       // If this is an explicit specialization of a member that is a function
       // template, mark it as a member specialization.
-      if (IsMemberSpecialization &&
-          NewTemplateDecl->getInstantiatedFromMemberTemplate()) {
-        NewTemplateDecl->setMemberSpecialization();
-        assert(OldTemplateDecl->isMemberSpecialization());
+      if (IsMemberSpecialization) {
         // Explicit specializations of a member template do not inherit deleted
         // status from the parent member template that they are specializing.
         if (OldFD->isDeleted()) {
@@ -15179,8 +15179,8 @@ ParmVarDecl *Sema::CheckParameter(DeclContext *DC, SourceLocation StartLoc,
   // we know that references to that pack must also be expanded within the
   // lambda scope.
   if (New->isParameterPack())
-    if (auto *LSI = getEnclosingLambda())
-      LSI->LocalPacks.push_back(New);
+    if (auto *CSI = getEnclosingLambdaOrBlock())
+      CSI->LocalPacks.push_back(New);
 
   if (New->getType().hasNonTrivialToPrimitiveDestructCUnion() ||
       New->getType().hasNonTrivialToPrimitiveCopyCUnion())
@@ -17107,8 +17107,8 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
         DeclResult Result = CheckClassTemplate(
             S, TagSpec, TUK, KWLoc, SS, Name, NameLoc, Attrs, TemplateParams,
             AS, ModulePrivateLoc,
-            /*FriendLoc*/ SourceLocation(), TemplateParameterLists.size() - 1,
-            TemplateParameterLists.data(), SkipBody);
+            /*FriendLoc*/ SourceLocation(), TemplateParameterLists.drop_back(),
+            isMemberSpecialization, SkipBody);
         return Result.get();
       } else {
         // The "template<>" header is extraneous.
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 811b3ed2859a2689c4ad4e55f4e6f0c79ecf7f23..c7f2fe08bc1d3e9fdac56101ff4136db6a8341b2 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -17426,8 +17426,8 @@ DeclResult Sema::ActOnTemplatedFriendTag(
       return CheckClassTemplate(S, TagSpec, TagUseKind::Friend, TagLoc, SS,
                                 Name, NameLoc, Attr, TemplateParams, AS_public,
                                 /*ModulePrivateLoc=*/SourceLocation(),
-                                FriendLoc, TempParamLists.size() - 1,
-                                TempParamLists.data())
+                                FriendLoc, TempParamLists.drop_back(),
+                                IsMemberSpecialization)
           .get();
     } else {
       // The "template<>" header is extraneous.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index a9e370a9ffbdf9e915af242329ecea7a27196ad5..06a03ee11cc29022117b518254bb751f4010c1b5 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4096,7 +4096,15 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
     Res = new (Context) ImaginaryLiteral(Res,
                                         Context.getComplexType(Res->getType()));
 
-    Diag(Tok.getLocation(), diag::ext_imaginary_constant);
+    // In C++, this is a GNU extension. In C, it's a C2y extension.
+    unsigned DiagId;
+    if (getLangOpts().CPlusPlus)
+      DiagId = diag::ext_gnu_imaginary_constant;
+    else if (getLangOpts().C2y)
+      DiagId = diag::warn_c23_compat_imaginary_constant;
+    else
+      DiagId = diag::ext_c2y_imaginary_constant;
+    Diag(Tok.getLocation(), DiagId);
   }
   return Res;
 }
@@ -16138,17 +16146,7 @@ void Sema::ActOnBlockArguments(SourceLocation CaretLoc, Declarator &ParamInfo,
 
   TypeSourceInfo *Sig = GetTypeForDeclarator(ParamInfo);
   QualType T = Sig->getType();
-
-  // FIXME: We should allow unexpanded parameter packs here, but that would,
-  // in turn, make the block expression contain unexpanded parameter packs.
-  if (DiagnoseUnexpandedParameterPack(CaretLoc, Sig, UPPC_Block)) {
-    // Drop the parameters.
-    FunctionProtoType::ExtProtoInfo EPI;
-    EPI.HasTrailingReturn = false;
-    EPI.TypeQuals.addConst();
-    T = Context.getFunctionType(Context.DependentTy, std::nullopt, EPI);
-    Sig = Context.getTrivialTypeSourceInfo(T);
-  }
+  DiagnoseUnexpandedParameterPack(CaretLoc, Sig, UPPC_Block);
 
   // GetTypeForDeclarator always produces a function type for a block
   // literal signature.  Furthermore, it is always a FunctionProtoType
@@ -16420,7 +16418,8 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
   AnalysisBasedWarnings::Policy WP = AnalysisWarnings.getDefaultPolicy();
   PoppedFunctionScopePtr ScopeRAII = PopFunctionScopeInfo(&WP, BD, BlockTy);
 
-  BlockExpr *Result = new (Context) BlockExpr(BD, BlockTy);
+  BlockExpr *Result = new (Context)
+      BlockExpr(BD, BlockTy, BSI->ContainsUnexpandedParameterPack);
 
   // If the block isn't obviously global, i.e. it captures anything at
   // all, then we need to do a few things in the surrounding context:
@@ -16443,6 +16442,8 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
   if (getCurFunction())
     getCurFunction()->addBlock(BD);
 
+  // This can happen if the block's return type is deduced, but
+  // the return expression is invalid.
   if (BD->isInvalidDecl())
     return CreateRecoveryExpr(Result->getBeginLoc(), Result->getEndLoc(),
                               {Result}, Result->getType());
@@ -18915,7 +18916,17 @@ bool Sema::tryCaptureVariable(
   // We need to sync up the Declaration Context with the
   // FunctionScopeIndexToStopAt
   if (FunctionScopeIndexToStopAt) {
+    assert(!FunctionScopes.empty() && "No function scopes to stop at?");
     unsigned FSIndex = FunctionScopes.size() - 1;
+    // When we're parsing the lambda parameter list, the current DeclContext is
+    // NOT the lambda but its parent. So move away the current LSI before
+    // aligning DC and FunctionScopeIndexToStopAt.
+    if (auto *LSI = dyn_cast<LambdaScopeInfo>(FunctionScopes[FSIndex]);
+        FSIndex && LSI && !LSI->AfterParameterList)
+      --FSIndex;
+    assert(MaxFunctionScopesIndex <= FSIndex &&
+           "FunctionScopeIndexToStopAt should be no greater than FSIndex into "
+           "FunctionScopes.");
     while (FSIndex != MaxFunctionScopesIndex) {
       DC = getLambdaAwareParentOfDeclContext(DC);
       --FSIndex;
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 8e9bcb10a80b46fe086bbd7a234dbf781a5738ac..40f24ea0ab2eaada8b0d0512f9cc47242b187df8 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -503,17 +503,23 @@ bool Sema::checkLiteralOperatorId(const CXXScopeSpec &SS,
     const IdentifierInfo *II = Name.Identifier;
     ReservedIdentifierStatus Status = II->isReserved(PP.getLangOpts());
     SourceLocation Loc = Name.getEndLoc();
-    if (!PP.getSourceManager().isInSystemHeader(Loc)) {
-      if (auto Hint = FixItHint::CreateReplacement(
-              Name.getSourceRange(),
-              (StringRef("operator\"\"") + II->getName()).str());
-          isReservedInAllContexts(Status)) {
-        Diag(Loc, diag::warn_reserved_extern_symbol)
-            << II << static_cast<int>(Status) << Hint;
-      } else {
-        Diag(Loc, diag::warn_deprecated_literal_operator_id) << II << Hint;
-      }
-    }
+
+    auto Hint = FixItHint::CreateReplacement(
+        Name.getSourceRange(),
+        (StringRef("operator\"\"") + II->getName()).str());
+
+    // Only emit this diagnostic if we start with an underscore, else the
+    // diagnostic for C++11 requiring a space between the quotes and the
+    // identifier conflicts with this and gets confusing. The diagnostic stating
+    // this is a reserved name should force the underscore, which gets this
+    // back.
+    if (II->isReservedLiteralSuffixId() !=
+        ReservedLiteralSuffixIdStatus::NotStartsWithUnderscore)
+      Diag(Loc, diag::warn_deprecated_literal_operator_id) << II << Hint;
+
+    if (isReservedInAllContexts(Status))
+      Diag(Loc, diag::warn_reserved_extern_symbol)
+          << II << static_cast<int>(Status) << Hint;
   }
 
   if (!SS.isValid())
@@ -2151,7 +2157,8 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
 
   // Per C++0x [expr.new]p5, the type being constructed may be a
   // typedef of an array type.
-  if (!ArraySize) {
+  // Dependent case will be handled separately.
+  if (!ArraySize && !AllocType->isDependentType()) {
     if (const ConstantArrayType *Array
                               = Context.getAsConstantArrayType(AllocType)) {
       ArraySize = IntegerLiteral::Create(Context, Array->getSize(),
@@ -8680,7 +8687,7 @@ static void CheckIfAnyEnclosingLambdasMustCaptureAnyPotentialCaptures(
   while (isa_and_nonnull<CapturedDecl>(DC))
     DC = DC->getParent();
   assert(
-      CurrentLSI->CallOperator == DC &&
+      (CurrentLSI->CallOperator == DC || !CurrentLSI->AfterParameterList) &&
       "The current call operator must be synchronized with Sema's CurContext");
 #endif // NDEBUG
 
diff --git a/clang/lib/Sema/SemaFunctionEffects.cpp b/clang/lib/Sema/SemaFunctionEffects.cpp
index 0ac5de29f66aa75e9d2ebadd9cc13dc7510bbb13..70e5d78661a835639aba5d527a3474ed3275d39c 100644
--- a/clang/lib/Sema/SemaFunctionEffects.cpp
+++ b/clang/lib/Sema/SemaFunctionEffects.cpp
@@ -1133,6 +1133,13 @@ private:
       return true;
     }
 
+    bool VisitObjCAtFinallyStmt(ObjCAtFinallyStmt *Finally) {
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeCatch,
+                                ViolationID::ThrowsOrCatchesExceptions,
+                                Finally->getAtFinallyLoc());
+      return true;
+    }
+
     bool VisitObjCMessageExpr(ObjCMessageExpr *Msg) {
       diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeObjCMessageSend,
                                 ViolationID::AccessesObjCMethodOrProperty,
@@ -1140,6 +1147,26 @@ private:
       return true;
     }
 
+    bool VisitObjCAutoreleasePoolStmt(ObjCAutoreleasePoolStmt *ARP) {
+      // Under the hood, @autorelease (potentially?) allocates memory and
+      // invokes ObjC methods. We don't currently have memory allocation as
+      // a "language construct" but we do have ObjC messaging, so diagnose that.
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeObjCMessageSend,
+                                ViolationID::AccessesObjCMethodOrProperty,
+                                ARP->getBeginLoc());
+      return true;
+    }
+
+    bool VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *Sync) {
+      // Under the hood, this calls objc_sync_enter and objc_sync_exit, wrapped
+      // in a @try/@finally block. Diagnose this generically as "ObjC
+      // messaging".
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeObjCMessageSend,
+                                ViolationID::AccessesObjCMethodOrProperty,
+                                Sync->getBeginLoc());
+      return true;
+    }
+
     bool VisitSEHExceptStmt(SEHExceptStmt *Exc) {
       diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeCatch,
                                 ViolationID::ThrowsOrCatchesExceptions,
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 5d6a586fe5a2cfd1f7e7c53ae84058fa0d253172..f560865681fa5a6850f1943688b169465b6f919c 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -9953,7 +9953,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     auto SynthesizeAggrGuide = [&](InitListExpr *ListInit) {
       auto *Pattern = Template;
       while (Pattern->getInstantiatedFromMemberTemplate()) {
-        if (Pattern->isMemberSpecialization())
+        if (Pattern->hasMemberSpecialization())
           break;
         Pattern = Pattern->getInstantiatedFromMemberTemplate();
       }
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index aeb20299b714a3c4e7cfb01ded73b8fe3c5094b5..7b9d5f4ff7eb341545fead5136f9714cf04fd828 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -2353,7 +2353,10 @@ ExprResult Sema::BuildBlockForLambdaConversion(SourceLocation CurrentLocation,
   Block->setBody(new (Context) CompoundStmt(ConvLocation));
 
   // Create the block literal expression.
-  Expr *BuildBlock = new (Context) BlockExpr(Block, Conv->getConversionType());
+  // TODO: Do we ever get here if we have unexpanded packs in the lambda???
+  Expr *BuildBlock =
+      new (Context) BlockExpr(Block, Conv->getConversionType(),
+                              /*ContainsUnexpandedParameterPack=*/false);
   ExprCleanupObjects.push_back(Block);
   Cleanup.setExprNeedsCleanups(true);
 
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 60fa195221c9384a89e4211774b76bce79e5fce5..f3f62474d06441585d8928d67142a427239b759f 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3666,9 +3666,7 @@ Sema::LookupLiteralOperator(Scope *S, LookupResult &R,
           TemplateArgumentLoc Arg(TemplateArgument(StringLit), StringLit);
           if (CheckTemplateArgument(
                   Params->getParam(0), Arg, FD, R.getNameLoc(), R.getNameLoc(),
-                  0, SugaredChecked, CanonicalChecked, CTAK_Specified,
-                  /*PartialOrdering=*/false,
-                  /*MatchedPackOnParmToNonPackOnArg=*/nullptr) ||
+                  0, SugaredChecked, CanonicalChecked, CTAK_Specified) ||
               Trap.hasErrorOccurred())
             IsTemplate = false;
         }
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 66f8029a2754b9339556be449c1de0ac1374c0f6..1b24331cbd87ca11d79873d95994b1dfad73387a 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -366,6 +366,31 @@ bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
     }
   }
 
+  case OpenACCClauseKind::Gang: {
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Loop:
+    case OpenACCDirectiveKind::ParallelLoop:
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::KernelsLoop:
+    case OpenACCDirectiveKind::Routine:
+      return true;
+    default:
+      return false;
+    }
+  case OpenACCClauseKind::Worker: {
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Loop:
+    case OpenACCDirectiveKind::ParallelLoop:
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::KernelsLoop:
+    case OpenACCDirectiveKind::Routine:
+      return true;
+    default:
+      return false;
+    }
+  }
+  }
+
   default:
     // Do nothing so we can go to the 'unimplemented' diagnostic instead.
     return true;
@@ -459,6 +484,23 @@ class SemaOpenACCClauseVisitor {
     return nullptr;
   }
 
+  // OpenACC 3.3 2.9:
+  // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
+  // appears.
+  bool DiagIfSeqClause(SemaOpenACC::OpenACCParsedClause &Clause) {
+    const auto *Itr =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSeqClause>);
+
+    if (Itr != ExistingClauses.end()) {
+      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
+          << Clause.getClauseKind() << (*Itr)->getClauseKind();
+      SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+
+      return true;
+    }
+    return false;
+  }
+
 public:
   SemaOpenACCClauseVisitor(SemaOpenACC &S,
                            ArrayRef<const OpenACCClause *> ExistingClauses)
@@ -470,26 +512,13 @@ public:
 
   OpenACCClause *Visit(SemaOpenACC::OpenACCParsedClause &Clause) {
     switch (Clause.getClauseKind()) {
-  case OpenACCClauseKind::Gang:
-  case OpenACCClauseKind::Worker:
-  case OpenACCClauseKind::Vector: {
-    // TODO OpenACC: These are only implemented enough for the 'seq' diagnostic,
-    // otherwise treats itself as unimplemented.  When we implement these, we
-    // can remove them from here.
-
-    // OpenACC 3.3 2.9:
-    // A 'gang', 'worker', or 'vector' clause may not appear if a 'seq' clause
-    // appears.
-    const auto *Itr =
-        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSeqClause>);
-
-    if (Itr != ExistingClauses.end()) {
-      SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_cannot_combine)
-          << Clause.getClauseKind() << (*Itr)->getClauseKind();
-      SemaRef.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    case OpenACCClauseKind::Vector: {
+      // TODO OpenACC: These are only implemented enough for the 'seq'
+      // diagnostic, otherwise treats itself as unimplemented.  When we
+      // implement these, we can remove them from here.
+      DiagIfSeqClause(Clause);
+      return isNotImplemented();
     }
-    return isNotImplemented();
-  }
 
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
   case OpenACCClauseKind::CLAUSE_NAME:                                         \
@@ -1006,6 +1035,169 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitIndependentClause(
                                           Clause.getEndLoc());
 }
 
+OpenACCClause *SemaOpenACCClauseVisitor::VisitWorkerClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  if (DiagIfSeqClause(Clause))
+    return nullptr;
+
+  // Restrictions only properly implemented on 'loop' constructs, and it is
+  // the only construct that can do anything with this, so skip/treat as
+  // unimplemented for the combined constructs.
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  Expr *IntExpr =
+      Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr;
+
+  if (IntExpr) {
+    switch (SemaRef.getActiveComputeConstructInfo().Kind) {
+    case OpenACCDirectiveKind::Invalid:
+      SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_int_arg_invalid)
+          << OpenACCClauseKind::Worker << "num" << /*orphan=*/0;
+      IntExpr = nullptr;
+      break;
+    case OpenACCDirectiveKind::Parallel:
+      SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_int_arg_invalid)
+          << OpenACCClauseKind::Worker << "num" << /*parallel=*/1;
+      IntExpr = nullptr;
+      break;
+    case OpenACCDirectiveKind::Serial:
+      SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_int_arg_invalid)
+          << OpenACCClauseKind::Worker << "num" << /*serial=*/3;
+      IntExpr = nullptr;
+      break;
+    case OpenACCDirectiveKind::Kernels: {
+      const auto *Itr =
+          llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
+                        llvm::IsaPred<OpenACCNumWorkersClause>);
+      if (Itr != SemaRef.getActiveComputeConstructInfo().Clauses.end()) {
+        SemaRef.Diag(IntExpr->getBeginLoc(), diag::err_acc_num_arg_conflict)
+            << OpenACCClauseKind::Worker << /*num_workers=*/1;
+        SemaRef.Diag((*Itr)->getBeginLoc(),
+                     diag::note_acc_previous_clause_here);
+
+        IntExpr = nullptr;
+      }
+      break;
+    }
+    default:
+      llvm_unreachable("Non compute construct in active compute construct");
+    }
+  }
+
+  // OpenACC 3.3 2.9.3: The region of a loop with a 'worker' clause may not
+  // contain a loop with a gang or worker clause unless within a nested compute
+  // region.
+  if (SemaRef.LoopWorkerClauseLoc.isValid()) {
+    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
+    // one of these until we get to the end of the construct.
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+        << OpenACCClauseKind::Worker << OpenACCClauseKind::Worker
+        << /*skip kernels construct info*/ 0;
+    SemaRef.Diag(SemaRef.LoopWorkerClauseLoc,
+                 diag::note_acc_previous_clause_here);
+    return nullptr;
+  }
+
+  return OpenACCWorkerClause::Create(Ctx, Clause.getBeginLoc(),
+                                     Clause.getLParenLoc(), IntExpr,
+                                     Clause.getEndLoc());
+}
+
+OpenACCClause *SemaOpenACCClauseVisitor::VisitGangClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  if (DiagIfSeqClause(Clause))
+    return nullptr;
+
+  // Restrictions only properly implemented on 'loop' constructs, and it is
+  // the only construct that can do anything with this, so skip/treat as
+  // unimplemented for the combined constructs.
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  llvm::SmallVector<OpenACCGangKind> GangKinds;
+  llvm::SmallVector<Expr *> IntExprs;
+
+  // Store the existing locations, so we can do duplicate checking.  Index is
+  // the int-value of the OpenACCGangKind enum.
+  SourceLocation ExistingElemLoc[3];
+
+  for (unsigned I = 0; I < Clause.getIntExprs().size(); ++I) {
+    OpenACCGangKind GK = Clause.getGangKinds()[I];
+    ExprResult ER = SemaRef.CheckGangExpr(GK, Clause.getIntExprs()[I]);
+
+    if (!ER.isUsable())
+      continue;
+
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+    // construct, the gang clause behaves as follows. ... An argument with no
+    // keyword or with num keyword is only allowed when num_gangs does not
+    // appear on the kernels construct.
+    if (SemaRef.getActiveComputeConstructInfo().Kind ==
+            OpenACCDirectiveKind::Kernels &&
+        GK == OpenACCGangKind::Num) {
+      const auto *Itr =
+          llvm::find_if(SemaRef.getActiveComputeConstructInfo().Clauses,
+                        llvm::IsaPred<OpenACCNumGangsClause>);
+
+      if (Itr != SemaRef.getActiveComputeConstructInfo().Clauses.end()) {
+        SemaRef.Diag(ER.get()->getBeginLoc(), diag::err_acc_num_arg_conflict)
+            << OpenACCClauseKind::Gang << /*num_gangs=*/0;
+        SemaRef.Diag((*Itr)->getBeginLoc(),
+                     diag::note_acc_previous_clause_here);
+        continue;
+      }
+    }
+
+    // OpenACC 3.3 2.9: 'gang-arg-list' may have at most one num, one dim, and
+    // one static argument.
+    if (ExistingElemLoc[static_cast<unsigned>(GK)].isValid()) {
+      SemaRef.Diag(ER.get()->getBeginLoc(), diag::err_acc_gang_multiple_elt)
+          << static_cast<unsigned>(GK);
+      SemaRef.Diag(ExistingElemLoc[static_cast<unsigned>(GK)],
+                   diag::note_acc_previous_expr_here);
+      continue;
+    }
+
+    ExistingElemLoc[static_cast<unsigned>(GK)] = ER.get()->getBeginLoc();
+    GangKinds.push_back(GK);
+    IntExprs.push_back(ER.get());
+  }
+
+  // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+  // construct, the gang clause behaves as follows. ... The region of a loop
+  // with a gang clause may not contain another loop with a gang clause unless
+  // within a nested compute region.
+  if (SemaRef.LoopGangClauseOnKernelLoc.isValid()) {
+    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
+    // one of these until we get to the end of the construct.
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+        << OpenACCClauseKind::Gang << OpenACCClauseKind::Gang
+        << /*kernels construct info*/ 1;
+    SemaRef.Diag(SemaRef.LoopGangClauseOnKernelLoc,
+                 diag::note_acc_previous_clause_here);
+    return nullptr;
+  }
+
+  // OpenACC 3.3 2.9.3: The region of a loop with a 'worker' clause may not
+  // contain a loop with a gang or worker clause unless within a nested compute
+  // region.
+  if (SemaRef.LoopWorkerClauseLoc.isValid()) {
+    // This handles the 'inner loop' diagnostic, but we cannot set that we're on
+    // one of these until we get to the end of the construct.
+    SemaRef.Diag(Clause.getBeginLoc(), diag::err_acc_clause_in_clause_region)
+        << OpenACCClauseKind::Gang << OpenACCClauseKind::Worker
+        << /*kernels construct info*/ 1;
+    SemaRef.Diag(SemaRef.LoopWorkerClauseLoc,
+                 diag::note_acc_previous_clause_here);
+    return nullptr;
+  }
+
+  return OpenACCGangClause::Create(Ctx, Clause.getBeginLoc(),
+                                   Clause.getLParenLoc(), GangKinds, IntExprs,
+                                   Clause.getEndLoc());
+}
+
 OpenACCClause *SemaOpenACCClauseVisitor::VisitSeqClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
   // Restrictions only properly implemented on 'loop' constructs, and it is
@@ -1118,17 +1310,52 @@ SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
     SemaOpenACC &S, OpenACCDirectiveKind DK,
     ArrayRef<const OpenACCClause *> UnInstClauses,
     ArrayRef<OpenACCClause *> Clauses)
-    : SemaRef(S), WasInsideComputeConstruct(S.InsideComputeConstruct),
-      DirKind(DK), LoopRAII(SemaRef, /*PreserveDepth=*/false) {
+    : SemaRef(S), OldActiveComputeConstructInfo(S.ActiveComputeConstructInfo),
+      DirKind(DK), OldLoopGangClauseOnKernelLoc(S.LoopGangClauseOnKernelLoc),
+      OldLoopWorkerClauseLoc(S.LoopWorkerClauseLoc),
+      LoopRAII(SemaRef, /*PreserveDepth=*/false) {
   // Compute constructs end up taking their 'loop'.
   if (DirKind == OpenACCDirectiveKind::Parallel ||
       DirKind == OpenACCDirectiveKind::Serial ||
       DirKind == OpenACCDirectiveKind::Kernels) {
-    SemaRef.InsideComputeConstruct = true;
+    SemaRef.ActiveComputeConstructInfo.Kind = DirKind;
+    SemaRef.ActiveComputeConstructInfo.Clauses = Clauses;
     SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs);
+
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+    // construct, the gang clause behaves as follows. ... The region of a loop
+    // with a gang clause may not contain another loop with a gang clause unless
+    // within a nested compute region.
+    //
+    // Implement the 'unless within a nested compute region' part.
+    SemaRef.LoopGangClauseOnKernelLoc = {};
+    SemaRef.LoopWorkerClauseLoc = {};
   } else if (DirKind == OpenACCDirectiveKind::Loop) {
     SetCollapseInfoBeforeAssociatedStmt(UnInstClauses, Clauses);
     SetTileInfoBeforeAssociatedStmt(UnInstClauses, Clauses);
+
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+    // construct, the gang clause behaves as follows. ... The region of a loop
+    // with a gang clause may not contain another loop with a gang clause unless
+    // within a nested compute region.
+    //
+    // We don't bother doing this when this is a template instantiation, as
+    // there is no reason to do these checks: the existance of a
+    // gang/kernels/etc cannot be dependent.
+    if (SemaRef.getActiveComputeConstructInfo().Kind ==
+            OpenACCDirectiveKind::Kernels &&
+        UnInstClauses.empty()) {
+      // This handles the 'outer loop' part of this.
+      auto *Itr = llvm::find_if(Clauses, llvm::IsaPred<OpenACCGangClause>);
+      if (Itr != Clauses.end())
+        SemaRef.LoopGangClauseOnKernelLoc = (*Itr)->getBeginLoc();
+    }
+
+    if (UnInstClauses.empty()) {
+      auto *Itr = llvm::find_if(Clauses, llvm::IsaPred<OpenACCWorkerClause>);
+      if (Itr != Clauses.end())
+        SemaRef.LoopWorkerClauseLoc = (*Itr)->getBeginLoc();
+    }
   }
 }
 
@@ -1199,7 +1426,10 @@ void SemaOpenACC::AssociatedStmtRAII::SetTileInfoBeforeAssociatedStmt(
 }
 
 SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() {
-  SemaRef.InsideComputeConstruct = WasInsideComputeConstruct;
+  SemaRef.ActiveComputeConstructInfo = OldActiveComputeConstructInfo;
+  SemaRef.LoopGangClauseOnKernelLoc = OldLoopGangClauseOnKernelLoc;
+  SemaRef.LoopWorkerClauseLoc = OldLoopWorkerClauseLoc;
+
   if (DirKind == OpenACCDirectiveKind::Parallel ||
       DirKind == OpenACCDirectiveKind::Serial ||
       DirKind == OpenACCDirectiveKind::Kernels) {
@@ -1761,6 +1991,109 @@ ExprResult SemaOpenACC::CheckCollapseLoopCount(Expr *LoopCount) {
       ConstantExpr::Create(getASTContext(), LoopCount, APValue{*ICE})};
 }
 
+namespace {
+ExprResult CheckGangStaticExpr(SemaOpenACC &S, Expr *E) {
+  if (isa<OpenACCAsteriskSizeExpr>(E))
+    return E;
+  return S.ActOnIntExpr(OpenACCDirectiveKind::Invalid, OpenACCClauseKind::Gang,
+                        E->getBeginLoc(), E);
+}
+} // namespace
+
+ExprResult SemaOpenACC::CheckGangExpr(OpenACCGangKind GK, Expr *E) {
+  // Gang Expr legality depends on the associated compute construct.
+  switch (ActiveComputeConstructInfo.Kind) {
+  case OpenACCDirectiveKind::Invalid:
+  case OpenACCDirectiveKind::Parallel: {
+    switch (GK) {
+      // OpenACC 3.3 2.9.2: When the parent compute construct is a parallel
+      // construct, or an orphaned loop construct, the gang clause behaves as
+      // follows. ... The dim argument must be a constant positive integer value
+      // 1, 2, or 3.
+    case OpenACCGangKind::Dim: {
+      if (!E)
+        return ExprError();
+      ExprResult Res =
+          ActOnIntExpr(OpenACCDirectiveKind::Invalid, OpenACCClauseKind::Gang,
+                       E->getBeginLoc(), E);
+
+      if (!Res.isUsable())
+        return Res;
+
+      if (Res.get()->isInstantiationDependent())
+        return Res;
+
+      std::optional<llvm::APSInt> ICE =
+          Res.get()->getIntegerConstantExpr(getASTContext());
+
+      if (!ICE || *ICE <= 0 || ICE > 3) {
+        Diag(Res.get()->getBeginLoc(), diag::err_acc_gang_dim_value)
+            << ICE.has_value() << ICE.value_or(llvm::APSInt{}).getExtValue();
+        return ExprError();
+      }
+
+      return ExprResult{
+          ConstantExpr::Create(getASTContext(), Res.get(), APValue{*ICE})};
+    }
+      // OpenACC 3.3 2.9.2: When the parent compute construct is a parallel
+      // construct, or an orphaned loop construct, the gang clause behaves as
+      // follows. ... The num argument is not allowed.
+    case OpenACCGangKind::Num:
+      Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
+          << OpenACCClauseKind::Gang << GK
+          << (/*orphan/parallel=*/ActiveComputeConstructInfo.Kind ==
+                      OpenACCDirectiveKind::Parallel
+                  ? 1
+                  : 0);
+      return ExprError();
+    case OpenACCGangKind::Static:
+      return CheckGangStaticExpr(*this, E);
+    }
+  } break;
+  case OpenACCDirectiveKind::Kernels: {
+    switch (GK) {
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+    // construct, the gang clause behaves as follows. ... The dim argument is
+    // not allowed.
+    case OpenACCGangKind::Dim:
+      Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
+          << OpenACCClauseKind::Gang << GK << /*kernels=*/2;
+      return ExprError();
+    // OpenACC 3.3 2.9.2: When the parent compute construct is a kernels
+    // construct, the gang clause behaves as follows. ... An argument with no
+    // keyword or with num keyword is only allowed when num_gangs does not
+    // appear on the kernels construct. ... The region of a loop with the gang
+    // clause may not contain another loop with a gang clause unless within a
+    // nested compute region.
+    case OpenACCGangKind::Num:
+      // This isn't allowed if there is a 'num_gangs' on the kernel construct,
+      // and makes loop-with-gang-clause ill-formed inside of this 'loop', but
+      // nothing can be enforced here.
+      return ExprResult{E};
+    case OpenACCGangKind::Static:
+      return CheckGangStaticExpr(*this, E);
+    }
+  } break;
+  case OpenACCDirectiveKind::Serial: {
+    switch (GK) {
+    // 'dim' and 'num' don't really make sense on serial, and GCC rejects them
+    // too, so we disallow them too.
+    case OpenACCGangKind::Dim:
+    case OpenACCGangKind::Num:
+      Diag(E->getBeginLoc(), diag::err_acc_int_arg_invalid)
+          << OpenACCClauseKind::Gang << GK << /*Kernels=*/3;
+      return ExprError();
+    case OpenACCGangKind::Static:
+      return CheckGangStaticExpr(*this, E);
+    }
+  }
+  default:
+    llvm_unreachable("Non compute construct in active compute construct?");
+  }
+
+  llvm_unreachable("Compute construct directive not handled?");
+}
+
 ExprResult SemaOpenACC::CheckTileSizeExpr(Expr *SizeExpr) {
   if (!SizeExpr)
     return ExprError();
@@ -2031,7 +2364,7 @@ StmtResult SemaOpenACC::ActOnEndStmtDirective(OpenACCDirectiveKind K,
     // If we are in the scope of a compute construct, add this to the list of
     // loop constructs that need assigning to the next closing compute
     // construct.
-    if (InsideComputeConstruct)
+    if (isInComputeConstruct())
       ParentlessLoopConstructs.push_back(LoopConstruct);
 
     return LoopConstruct;
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index f545e9341e1ae6c5c45f235ef7178bfe59b0afdf..1205e85b4e6f531a01af00a3ea4ba89f263f6e9b 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1422,8 +1422,12 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
   // the implicit object parameter are of the same type.
 
   auto NormalizeQualifiers = [&](const CXXMethodDecl *M, Qualifiers Q) {
-    if (M->isExplicitObjectMemberFunction())
+    if (M->isExplicitObjectMemberFunction()) {
+      auto ThisType = M->getFunctionObjectParameterReferenceType();
+      if (ThisType.isConstQualified())
+        Q.removeConst();
       return Q;
+    }
 
     // We do not allow overloading based off of '__restrict'.
     Q.removeRestrict();
@@ -1439,14 +1443,23 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
     return Q;
   };
 
-  auto CompareType = [&](QualType Base, QualType D) {
-    auto BS = Base.getNonReferenceType().getCanonicalType().split();
+  auto AreQualifiersEqual = [&](SplitQualType BS, SplitQualType DS) {
     BS.Quals = NormalizeQualifiers(OldMethod, BS.Quals);
+    DS.Quals = NormalizeQualifiers(NewMethod, DS.Quals);
+
+    if (OldMethod->isExplicitObjectMemberFunction()) {
+      BS.Quals.removeVolatile();
+      DS.Quals.removeVolatile();
+    }
+
+    return BS.Quals == DS.Quals;
+  };
 
+  auto CompareType = [&](QualType Base, QualType D) {
+    auto BS = Base.getNonReferenceType().getCanonicalType().split();
     auto DS = D.getNonReferenceType().getCanonicalType().split();
-    DS.Quals = NormalizeQualifiers(NewMethod, DS.Quals);
 
-    if (BS.Quals != DS.Quals)
+    if (!AreQualifiersEqual(BS, DS))
       return false;
 
     if (OldMethod->isImplicitObjectMemberFunction() &&
@@ -6864,8 +6877,7 @@ void Sema::AddOverloadCandidate(
     OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
     bool PartialOverloading, bool AllowExplicit, bool AllowExplicitConversions,
     ADLCallKind IsADLCandidate, ConversionSequenceList EarlyConversions,
-    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction,
-    bool HasMatchedPackOnParmToNonPackOnArg) {
+    OverloadCandidateParamOrder PO, bool AggregateCandidateDeduction) {
   const FunctionProtoType *Proto
     = dyn_cast<FunctionProtoType>(Function->getType()->getAs<FunctionType>());
   assert(Proto && "Functions without a prototype cannot be overloaded");
@@ -6884,8 +6896,7 @@ void Sema::AddOverloadCandidate(
       AddMethodCandidate(Method, FoundDecl, Method->getParent(), QualType(),
                          Expr::Classification::makeSimpleLValue(), Args,
                          CandidateSet, SuppressUserConversions,
-                         PartialOverloading, EarlyConversions, PO,
-                         HasMatchedPackOnParmToNonPackOnArg);
+                         PartialOverloading, EarlyConversions, PO);
       return;
     }
     // We treat a constructor like a non-member function, since its object
@@ -6928,8 +6939,6 @@ void Sema::AddOverloadCandidate(
       CandidateSet.getRewriteInfo().getRewriteKind(Function, PO);
   Candidate.IsADLCandidate = IsADLCandidate;
   Candidate.ExplicitCallArguments = Args.size();
-  Candidate.HasMatchedPackOnParmToNonPackOnArg =
-      HasMatchedPackOnParmToNonPackOnArg;
 
   // Explicit functions are not actually candidates at all if we're not
   // allowing them in this context, but keep them around so we can point
@@ -7457,13 +7466,16 @@ void Sema::AddMethodCandidate(DeclAccessPair FoundDecl, QualType ObjectType,
   }
 }
 
-void Sema::AddMethodCandidate(
-    CXXMethodDecl *Method, DeclAccessPair FoundDecl,
-    CXXRecordDecl *ActingContext, QualType ObjectType,
-    Expr::Classification ObjectClassification, ArrayRef<Expr *> Args,
-    OverloadCandidateSet &CandidateSet, bool SuppressUserConversions,
-    bool PartialOverloading, ConversionSequenceList EarlyConversions,
-    OverloadCandidateParamOrder PO, bool HasMatchedPackOnParmToNonPackOnArg) {
+void
+Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
+                         CXXRecordDecl *ActingContext, QualType ObjectType,
+                         Expr::Classification ObjectClassification,
+                         ArrayRef<Expr *> Args,
+                         OverloadCandidateSet &CandidateSet,
+                         bool SuppressUserConversions,
+                         bool PartialOverloading,
+                         ConversionSequenceList EarlyConversions,
+                         OverloadCandidateParamOrder PO) {
   const FunctionProtoType *Proto
     = dyn_cast<FunctionProtoType>(Method->getType()->getAs<FunctionType>());
   assert(Proto && "Methods without a prototype cannot be overloaded");
@@ -7494,8 +7506,6 @@ void Sema::AddMethodCandidate(
   Candidate.TookAddressOfOverload =
       CandidateSet.getKind() == OverloadCandidateSet::CSK_AddressOfOverloadSet;
   Candidate.ExplicitCallArguments = Args.size();
-  Candidate.HasMatchedPackOnParmToNonPackOnArg =
-      HasMatchedPackOnParmToNonPackOnArg;
 
   bool IgnoreExplicitObject =
       (Method->isExplicitObjectMemberFunction() &&
@@ -7666,8 +7676,8 @@ void Sema::AddMethodTemplateCandidate(
   ConversionSequenceList Conversions;
   if (TemplateDeductionResult Result = DeduceTemplateArguments(
           MethodTmpl, ExplicitTemplateArgs, Args, Specialization, Info,
-          PartialOverloading, /*AggregateDeductionCandidate=*/false,
-          /*PartialOrdering=*/false, ObjectType, ObjectClassification,
+          PartialOverloading, /*AggregateDeductionCandidate=*/false, ObjectType,
+          ObjectClassification,
           [&](ArrayRef<QualType> ParamTypes) {
             return CheckNonDependentConversions(
                 MethodTmpl, ParamTypes, Args, CandidateSet, Conversions,
@@ -7705,8 +7715,7 @@ void Sema::AddMethodTemplateCandidate(
   AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
                      ActingContext, ObjectType, ObjectClassification, Args,
                      CandidateSet, SuppressUserConversions, PartialOverloading,
-                     Conversions, PO,
-                     Info.hasMatchedPackOnParmToNonPackOnArg());
+                     Conversions, PO);
 }
 
 /// Determine whether a given function template has a simple explicit specifier
@@ -7752,7 +7761,6 @@ void Sema::AddTemplateOverloadCandidate(
   if (TemplateDeductionResult Result = DeduceTemplateArguments(
           FunctionTemplate, ExplicitTemplateArgs, Args, Specialization, Info,
           PartialOverloading, AggregateCandidateDeduction,
-          /*PartialOrdering=*/false,
           /*ObjectType=*/QualType(),
           /*ObjectClassification=*/Expr::Classification(),
           [&](ArrayRef<QualType> ParamTypes) {
@@ -7793,8 +7801,7 @@ void Sema::AddTemplateOverloadCandidate(
       Specialization, FoundDecl, Args, CandidateSet, SuppressUserConversions,
       PartialOverloading, AllowExplicit,
       /*AllowExplicitConversions=*/false, IsADLCandidate, Conversions, PO,
-      Info.AggregateDeductionCandidateHasMismatchedArity,
-      Info.hasMatchedPackOnParmToNonPackOnArg());
+      Info.AggregateDeductionCandidateHasMismatchedArity);
 }
 
 bool Sema::CheckNonDependentConversions(
@@ -7916,8 +7923,7 @@ void Sema::AddConversionCandidate(
     CXXConversionDecl *Conversion, DeclAccessPair FoundDecl,
     CXXRecordDecl *ActingContext, Expr *From, QualType ToType,
     OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit,
-    bool AllowExplicit, bool AllowResultConversion,
-    bool HasMatchedPackOnParmToNonPackOnArg) {
+    bool AllowExplicit, bool AllowResultConversion) {
   assert(!Conversion->getDescribedFunctionTemplate() &&
          "Conversion function templates use AddTemplateConversionCandidate");
   QualType ConvType = Conversion->getConversionType().getNonReferenceType();
@@ -7962,8 +7968,6 @@ void Sema::AddConversionCandidate(
   Candidate.FinalConversion.setAllToTypes(ToType);
   Candidate.Viable = true;
   Candidate.ExplicitCallArguments = 1;
-  Candidate.HasMatchedPackOnParmToNonPackOnArg =
-      HasMatchedPackOnParmToNonPackOnArg;
 
   // Explicit functions are not actually candidates at all if we're not
   // allowing them in this context, but keep them around so we can point
@@ -8165,8 +8169,7 @@ void Sema::AddTemplateConversionCandidate(
   assert(Specialization && "Missing function template specialization?");
   AddConversionCandidate(Specialization, FoundDecl, ActingDC, From, ToType,
                          CandidateSet, AllowObjCConversionOnExplicit,
-                         AllowExplicit, AllowResultConversion,
-                         Info.hasMatchedPackOnParmToNonPackOnArg());
+                         AllowExplicit, AllowResultConversion);
 }
 
 void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
@@ -10519,10 +10522,6 @@ bool clang::isBetterOverloadCandidate(
           isa<CXXConstructorDecl>(Cand2.Function))
     return isa<CXXConstructorDecl>(Cand1.Function);
 
-  if (Cand1.HasMatchedPackOnParmToNonPackOnArg !=
-      Cand2.HasMatchedPackOnParmToNonPackOnArg)
-    return Cand2.HasMatchedPackOnParmToNonPackOnArg;
-
   //    -- F1 is a non-template function and F2 is a function template
   //       specialization, or, if not that,
   bool Cand1IsSpecialization = Cand1.Function &&
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index d81c6de3428dc7b684ed07fd632644d9568ef238..f801455596fe6f408feb7e33932545cdf61c4a0d 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -682,6 +682,10 @@ static Attr *ProcessStmtAttribute(Sema &S, Stmt *St, const ParsedAttr &A,
   case ParsedAttr::AT_Annotate:
     return S.CreateAnnotationAttr(A);
   default:
+    if (Attr *AT = nullptr; A.getInfo().handleStmtAttribute(S, St, A, AT) !=
+                            ParsedAttrInfo::NotHandled) {
+      return AT;
+    }
     // N.B., ClangAttrEmitter.cpp emits a diagnostic helper that ensures a
     // declaration attribute is not written on a statement, but this code is
     // needed for attributes in Attr.td that do not list any subjects.
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 62d0d0914fa3062a6ae07b8230365987347516ac..294eb8e3353cde75d39b0a9c3e32a1c9357b9d0a 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -1010,8 +1010,8 @@ NamedDecl *Sema::ActOnTypeParameter(Scope *S, bool Typename,
   Param->setAccess(AS_public);
 
   if (Param->isParameterPack())
-    if (auto *LSI = getEnclosingLambda())
-      LSI->LocalPacks.push_back(Param);
+    if (auto *CSI = getEnclosingLambdaOrBlock())
+      CSI->LocalPacks.push_back(Param);
 
   if (ParamName) {
     maybeDiagnoseTemplateParameterShadow(*this, S, ParamNameLoc, ParamName);
@@ -1542,8 +1542,8 @@ NamedDecl *Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
     Param->setInvalidDecl();
 
   if (Param->isParameterPack())
-    if (auto *LSI = getEnclosingLambda())
-      LSI->LocalPacks.push_back(Param);
+    if (auto *CSI = getEnclosingLambdaOrBlock())
+      CSI->LocalPacks.push_back(Param);
 
   if (ParamName) {
     maybeDiagnoseTemplateParameterShadow(*this, S, D.getIdentifierLoc(),
@@ -1593,7 +1593,7 @@ NamedDecl *Sema::ActOnTemplateTemplateParameter(
   Param->setAccess(AS_public);
 
   if (Param->isParameterPack())
-    if (auto *LSI = getEnclosingLambda())
+    if (auto *LSI = getEnclosingLambdaOrBlock())
       LSI->LocalPacks.push_back(Param);
 
   // If the template template parameter has a name, then link the identifier
@@ -1795,8 +1795,9 @@ DeclResult Sema::CheckClassTemplate(
     CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc,
     const ParsedAttributesView &Attr, TemplateParameterList *TemplateParams,
     AccessSpecifier AS, SourceLocation ModulePrivateLoc,
-    SourceLocation FriendLoc, unsigned NumOuterTemplateParamLists,
-    TemplateParameterList **OuterTemplateParamLists, SkipBodyInfo *SkipBody) {
+    SourceLocation FriendLoc,
+    ArrayRef<TemplateParameterList *> OuterTemplateParamLists,
+    bool IsMemberSpecialization, SkipBodyInfo *SkipBody) {
   assert(TemplateParams && TemplateParams->size() > 0 &&
          "No template parameters");
   assert(TUK != TagUseKind::Reference &&
@@ -1984,19 +1985,6 @@ DeclResult Sema::CheckClassTemplate(
   }
 
   if (PrevClassTemplate) {
-    // Ensure that the template parameter lists are compatible. Skip this check
-    // for a friend in a dependent context: the template parameter list itself
-    // could be dependent.
-    if (!(TUK == TagUseKind::Friend && CurContext->isDependentContext()) &&
-        !TemplateParameterListsAreEqual(
-            TemplateCompareNewDeclInfo(SemanticContext ? SemanticContext
-                                                       : CurContext,
-                                       CurContext, KWLoc),
-            TemplateParams, PrevClassTemplate,
-            PrevClassTemplate->getTemplateParameters(), /*Complain=*/true,
-            TPL_TemplateMatch))
-      return true;
-
     // C++ [temp.class]p4:
     //   In a redeclaration, partial specialization, explicit
     //   specialization or explicit instantiation of a class template,
@@ -2011,30 +1999,6 @@ DeclResult Sema::CheckClassTemplate(
       Diag(PrevRecordDecl->getLocation(), diag::note_previous_use);
       Kind = PrevRecordDecl->getTagKind();
     }
-
-    // Check for redefinition of this class template.
-    if (TUK == TagUseKind::Definition) {
-      if (TagDecl *Def = PrevRecordDecl->getDefinition()) {
-        // If we have a prior definition that is not visible, treat this as
-        // simply making that previous definition visible.
-        NamedDecl *Hidden = nullptr;
-        if (SkipBody && !hasVisibleDefinition(Def, &Hidden)) {
-          SkipBody->ShouldSkip = true;
-          SkipBody->Previous = Def;
-          auto *Tmpl = cast<CXXRecordDecl>(Hidden)->getDescribedClassTemplate();
-          assert(Tmpl && "original definition of a class template is not a "
-                         "class template?");
-          makeMergedDefinitionVisible(Hidden);
-          makeMergedDefinitionVisible(Tmpl);
-        } else {
-          Diag(NameLoc, diag::err_redefinition) << Name;
-          Diag(Def->getLocation(), diag::note_previous_definition);
-          // FIXME: Would it make sense to try to "forget" the previous
-          // definition, as part of error recovery?
-          return true;
-        }
-      }
-    }
   } else if (PrevDecl) {
     // C++ [temp]p5:
     //   A class template shall not have the same name as any other
@@ -2046,23 +2010,6 @@ DeclResult Sema::CheckClassTemplate(
     return true;
   }
 
-  // Check the template parameter list of this declaration, possibly
-  // merging in the template parameter list from the previous class
-  // template declaration. Skip this check for a friend in a dependent
-  // context, because the template parameter list might be dependent.
-  if (!(TUK == TagUseKind::Friend && CurContext->isDependentContext()) &&
-      CheckTemplateParameterList(
-          TemplateParams,
-          PrevClassTemplate ? GetTemplateParameterList(PrevClassTemplate)
-                            : nullptr,
-          (SS.isSet() && SemanticContext && SemanticContext->isRecord() &&
-           SemanticContext->isDependentContext())
-              ? TPC_ClassTemplateMember
-          : TUK == TagUseKind::Friend ? TPC_FriendClassTemplate
-                                      : TPC_ClassTemplate,
-          SkipBody))
-    Invalid = true;
-
   if (SS.isSet()) {
     // If the name of the template was qualified, we must be defining the
     // template out-of-line.
@@ -2089,10 +2036,8 @@ DeclResult Sema::CheckClassTemplate(
                             PrevClassTemplate->getTemplatedDecl() : nullptr,
                           /*DelayTypeCreation=*/true);
   SetNestedNameSpecifier(*this, NewClass, SS);
-  if (NumOuterTemplateParamLists > 0)
-    NewClass->setTemplateParameterListsInfo(
-        Context,
-        llvm::ArrayRef(OuterTemplateParamLists, NumOuterTemplateParamLists));
+  if (!OuterTemplateParamLists.empty())
+    NewClass->setTemplateParameterListsInfo(Context, OuterTemplateParamLists);
 
   // Add alignment attributes if necessary; these attributes are checked when
   // the ASTContext lays out the structure.
@@ -2105,7 +2050,10 @@ DeclResult Sema::CheckClassTemplate(
     = ClassTemplateDecl::Create(Context, SemanticContext, NameLoc,
                                 DeclarationName(Name), TemplateParams,
                                 NewClass);
-
+  // If we are providing an explicit specialization of a member that is a
+  // class template, make a note of that.
+  if (IsMemberSpecialization)
+    NewTemplate->setMemberSpecialization();
   if (ShouldAddRedecl)
     NewTemplate->setPreviousDecl(PrevClassTemplate);
 
@@ -2120,12 +2068,6 @@ DeclResult Sema::CheckClassTemplate(
   assert(T->isDependentType() && "Class template type is not dependent?");
   (void)T;
 
-  // If we are providing an explicit specialization of a member that is a
-  // class template, make a note of that.
-  if (PrevClassTemplate &&
-      PrevClassTemplate->getInstantiatedFromMemberTemplate())
-    PrevClassTemplate->setMemberSpecialization();
-
   // Set the access specifier.
   if (!Invalid && TUK != TagUseKind::Friend &&
       NewTemplate->getDeclContext()->isRecord())
@@ -2135,8 +2077,62 @@ DeclResult Sema::CheckClassTemplate(
   NewClass->setLexicalDeclContext(CurContext);
   NewTemplate->setLexicalDeclContext(CurContext);
 
-  if (TUK == TagUseKind::Definition && (!SkipBody || !SkipBody->ShouldSkip))
-    NewClass->startDefinition();
+  // Ensure that the template parameter lists are compatible. Skip this check
+  // for a friend in a dependent context: the template parameter list itself
+  // could be dependent.
+  if (ShouldAddRedecl && PrevClassTemplate &&
+      !TemplateParameterListsAreEqual(
+          NewTemplate, TemplateParams, PrevClassTemplate,
+          PrevClassTemplate->getTemplateParameters(),
+          /*Complain=*/true, TPL_TemplateMatch))
+    return true;
+
+  // Check the template parameter list of this declaration, possibly
+  // merging in the template parameter list from the previous class
+  // template declaration. Skip this check for a friend in a dependent
+  // context, because the template parameter list might be dependent.
+  if (ShouldAddRedecl &&
+      CheckTemplateParameterList(
+          TemplateParams,
+          PrevClassTemplate ? PrevClassTemplate->getTemplateParameters()
+                            : nullptr,
+          (SS.isSet() && SemanticContext && SemanticContext->isRecord() &&
+           SemanticContext->isDependentContext())
+              ? TPC_ClassTemplateMember
+          : TUK == TagUseKind::Friend ? TPC_FriendClassTemplate
+                                      : TPC_ClassTemplate,
+          SkipBody))
+    Invalid = true;
+
+  if (TUK == TagUseKind::Definition) {
+    if (PrevClassTemplate) {
+      // Check for redefinition of this class template.
+      if (TagDecl *Def =
+              PrevClassTemplate->getTemplatedDecl()->getDefinition()) {
+        // If we have a prior definition that is not visible, treat this as
+        // simply making that previous definition visible.
+        NamedDecl *Hidden = nullptr;
+        if (SkipBody && !hasVisibleDefinition(Def, &Hidden)) {
+          SkipBody->ShouldSkip = true;
+          SkipBody->Previous = Def;
+          auto *Tmpl = cast<CXXRecordDecl>(Hidden)->getDescribedClassTemplate();
+          assert(Tmpl && "original definition of a class template is not a "
+                         "class template?");
+          makeMergedDefinitionVisible(Hidden);
+          makeMergedDefinitionVisible(Tmpl);
+        } else {
+          Diag(NameLoc, diag::err_redefinition) << Name;
+          Diag(Def->getLocation(), diag::note_previous_definition);
+          // FIXME: Would it make sense to try to "forget" the previous
+          // definition, as part of error recovery?
+          return true;
+        }
+      }
+    }
+
+    if (!SkipBody || !SkipBody->ShouldSkip)
+      NewClass->startDefinition();
+  }
 
   ProcessDeclAttributeList(S, NewClass, Attr);
   ProcessAPINotes(NewClass);
@@ -4133,7 +4129,8 @@ void Sema::CheckDeductionGuideTemplate(FunctionTemplateDecl *TD) {
 DeclResult Sema::ActOnVarTemplateSpecialization(
     Scope *S, Declarator &D, TypeSourceInfo *DI, LookupResult &Previous,
     SourceLocation TemplateKWLoc, TemplateParameterList *TemplateParams,
-    StorageClass SC, bool IsPartialSpecialization) {
+    StorageClass SC, bool IsPartialSpecialization,
+    bool IsMemberSpecialization) {
   // D must be variable template id.
   assert(D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId &&
          "Variable template specialization is declared with a template id.");
@@ -4251,17 +4248,16 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
             Context, VarTemplate->getDeclContext(), TemplateKWLoc,
             TemplateNameLoc, TemplateParams, VarTemplate, DI->getType(), DI, SC,
             CanonicalConverted);
+    // If we are providing an explicit specialization of a member variable
+    // template specialization, make a note of that.
+    if (IsMemberSpecialization)
+      Partial->setMemberSpecialization();
     Partial->setTemplateArgsAsWritten(TemplateArgs);
 
     if (!PrevPartial)
       VarTemplate->AddPartialSpecialization(Partial, InsertPos);
     Specialization = Partial;
 
-    // If we are providing an explicit specialization of a member variable
-    // template specialization, make a note of that.
-    if (PrevPartial && PrevPartial->getInstantiatedFromMember())
-      PrevPartial->setMemberSpecialization();
-
     CheckTemplatePartialSpecialization(Partial);
   } else {
     // Create a new class template specialization declaration node for
@@ -5179,8 +5175,7 @@ bool Sema::CheckTemplateArgument(
     unsigned ArgumentPackIndex,
     SmallVectorImpl<TemplateArgument> &SugaredConverted,
     SmallVectorImpl<TemplateArgument> &CanonicalConverted,
-    CheckTemplateArgumentKind CTAK, bool PartialOrdering,
-    bool *MatchedPackOnParmToNonPackOnArg) {
+    CheckTemplateArgumentKind CTAK) {
   // Check template type parameters.
   if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
     return CheckTemplateTypeArgument(TTP, Arg, SugaredConverted,
@@ -5395,8 +5390,8 @@ bool Sema::CheckTemplateArgument(
 
   case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion:
-    if (CheckTemplateTemplateArgument(TempParm, Params, Arg, PartialOrdering,
-                                      MatchedPackOnParmToNonPackOnArg))
+    if (CheckTemplateTemplateArgument(TempParm, Params, Arg,
+                                      /*IsDeduced=*/CTAK != CTAK_Specified))
       return true;
 
     SugaredConverted.push_back(Arg.getArgument());
@@ -5470,7 +5465,7 @@ bool Sema::CheckTemplateArgumentList(
     SmallVectorImpl<TemplateArgument> &SugaredConverted,
     SmallVectorImpl<TemplateArgument> &CanonicalConverted,
     bool UpdateArgsWithConversions, bool *ConstraintsNotSatisfied,
-    bool PartialOrderingTTP, bool *MatchedPackOnParmToNonPackOnArg) {
+    bool PartialOrderingTTP) {
 
   if (ConstraintsNotSatisfied)
     *ConstraintsNotSatisfied = false;
@@ -5503,7 +5498,8 @@ bool Sema::CheckTemplateArgumentList(
         DefaultArgs && ParamIdx >= DefaultArgs.StartPos) {
       // All written arguments should have been consumed by this point.
       assert(ArgIdx == NumArgs && "bad default argument deduction");
-      if (ParamIdx == DefaultArgs.StartPos) {
+      // FIXME: Don't ignore parameter packs.
+      if (ParamIdx == DefaultArgs.StartPos && !(*Param)->isParameterPack()) {
         assert(Param + DefaultArgs.Args.size() <= ParamEnd);
         // Default arguments from a DeducedTemplateName are already converted.
         for (const TemplateArgument &DefArg : DefaultArgs.Args) {
@@ -5549,8 +5545,7 @@ bool Sema::CheckTemplateArgumentList(
       if (CheckTemplateArgument(*Param, NewArgs[ArgIdx], Template, TemplateLoc,
                                 RAngleLoc, SugaredArgumentPack.size(),
                                 SugaredConverted, CanonicalConverted,
-                                CTAK_Specified, /*PartialOrdering=*/false,
-                                MatchedPackOnParmToNonPackOnArg))
+                                CTAK_Specified))
         return true;
 
       CanonicalConverted.back().setIsDefaulted(
@@ -5708,8 +5703,7 @@ bool Sema::CheckTemplateArgumentList(
     // Check the default template argument.
     if (CheckTemplateArgument(*Param, Arg, Template, TemplateLoc, RAngleLoc, 0,
                               SugaredConverted, CanonicalConverted,
-                              CTAK_Specified, /*PartialOrdering=*/false,
-                              /*MatchedPackOnParmToNonPackOnArg=*/nullptr))
+                              CTAK_Specified))
       return true;
 
     SugaredConverted.back().setIsDefaulted(true);
@@ -5730,9 +5724,8 @@ bool Sema::CheckTemplateArgumentList(
   // pack expansions; they might be empty. This can happen even if
   // PartialTemplateArgs is false (the list of arguments is complete but
   // still dependent).
-  if (PartialOrderingTTP ||
-      (CurrentInstantiationScope &&
-       CurrentInstantiationScope->getPartiallySubstitutedPack())) {
+  if (ArgIdx < NumArgs && CurrentInstantiationScope &&
+      CurrentInstantiationScope->getPartiallySubstitutedPack()) {
     while (ArgIdx < NumArgs &&
            NewArgs[ArgIdx].getArgument().isPackExpansion()) {
       const TemplateArgument &Arg = NewArgs[ArgIdx++].getArgument();
@@ -5779,9 +5772,7 @@ bool Sema::CheckTemplateArgumentList(
 
     MultiLevelTemplateArgumentList MLTAL = getTemplateInstantiationArgs(
         Template, NewContext, /*Final=*/false, CanonicalConverted,
-        /*RelativeToPrimary=*/true,
-        /*Pattern=*/nullptr,
-        /*ForConceptInstantiation=*/true);
+        /*RelativeToPrimary=*/true, /*ForConceptInstantiation=*/true);
     if (EnsureTemplateArgumentListConstraints(
             Template, MLTAL,
             SourceRange(TemplateLoc, TemplateArgs.getRAngleLoc()))) {
@@ -7292,10 +7283,10 @@ static void DiagnoseTemplateParameterListArityMismatch(
     Sema &S, TemplateParameterList *New, TemplateParameterList *Old,
     Sema::TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc);
 
-bool Sema::CheckTemplateTemplateArgument(
-    TemplateTemplateParmDecl *Param, TemplateParameterList *Params,
-    TemplateArgumentLoc &Arg, bool PartialOrdering,
-    bool *MatchedPackOnParmToNonPackOnArg) {
+bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
+                                         TemplateParameterList *Params,
+                                         TemplateArgumentLoc &Arg,
+                                         bool IsDeduced) {
   TemplateName Name = Arg.getArgument().getAsTemplateOrTemplatePattern();
   auto [Template, DefaultArgs] = Name.getTemplateDeclAndDefaultArgs();
   if (!Template) {
@@ -7330,47 +7321,64 @@ bool Sema::CheckTemplateTemplateArgument(
       << Template;
   }
 
-  if (!getLangOpts().RelaxedTemplateTemplateArgs)
-    return !TemplateParameterListsAreEqual(
-        Template->getTemplateParameters(), Params, /*Complain=*/true,
-        TPL_TemplateTemplateArgumentMatch, Arg.getLocation());
-
   // C++1z [temp.arg.template]p3: (DR 150)
   //   A template-argument matches a template template-parameter P when P
   //   is at least as specialized as the template-argument A.
-  if (!isTemplateTemplateParameterAtLeastAsSpecializedAs(
-          Params, Param, Template, DefaultArgs, Arg.getLocation(),
-          PartialOrdering, MatchedPackOnParmToNonPackOnArg))
-    return true;
-  // P2113
-  // C++20[temp.func.order]p2
-  //   [...] If both deductions succeed, the partial ordering selects the
-  // more constrained template (if one exists) as determined below.
-  SmallVector<const Expr *, 3> ParamsAC, TemplateAC;
-  Params->getAssociatedConstraints(ParamsAC);
-  // C++20[temp.arg.template]p3
-  //   [...] In this comparison, if P is unconstrained, the constraints on A
-  //   are not considered.
-  if (ParamsAC.empty())
-    return false;
+  if (getLangOpts().RelaxedTemplateTemplateArgs) {
+    // Quick check for the common case:
+    //   If P contains a parameter pack, then A [...] matches P if each of A's
+    //   template parameters matches the corresponding template parameter in
+    //   the template-parameter-list of P.
+    if (TemplateParameterListsAreEqual(
+            Template->getTemplateParameters(), Params, false,
+            TPL_TemplateTemplateArgumentMatch, Arg.getLocation()) &&
+        // If the argument has no associated constraints, then the parameter is
+        // definitely at least as specialized as the argument.
+        // Otherwise - we need a more thorough check.
+        !Template->hasAssociatedConstraints())
+      return false;
 
-  Template->getAssociatedConstraints(TemplateAC);
+    if (isTemplateTemplateParameterAtLeastAsSpecializedAs(
+            Params, Template, DefaultArgs, Arg.getLocation(), IsDeduced)) {
+      // P2113
+      // C++20[temp.func.order]p2
+      //   [...] If both deductions succeed, the partial ordering selects the
+      // more constrained template (if one exists) as determined below.
+      SmallVector<const Expr *, 3> ParamsAC, TemplateAC;
+      Params->getAssociatedConstraints(ParamsAC);
+      // C++2a[temp.arg.template]p3
+      //   [...] In this comparison, if P is unconstrained, the constraints on A
+      //   are not considered.
+      if (ParamsAC.empty())
+        return false;
 
-  bool IsParamAtLeastAsConstrained;
-  if (IsAtLeastAsConstrained(Param, ParamsAC, Template, TemplateAC,
-                             IsParamAtLeastAsConstrained))
-    return true;
-  if (!IsParamAtLeastAsConstrained) {
-    Diag(Arg.getLocation(),
-         diag::err_template_template_parameter_not_at_least_as_constrained)
-        << Template << Param << Arg.getSourceRange();
-    Diag(Param->getLocation(), diag::note_entity_declared_at) << Param;
-    Diag(Template->getLocation(), diag::note_entity_declared_at) << Template;
-    MaybeEmitAmbiguousAtomicConstraintsDiagnostic(Param, ParamsAC, Template,
-                                                  TemplateAC);
-    return true;
+      Template->getAssociatedConstraints(TemplateAC);
+
+      bool IsParamAtLeastAsConstrained;
+      if (IsAtLeastAsConstrained(Param, ParamsAC, Template, TemplateAC,
+                                 IsParamAtLeastAsConstrained))
+        return true;
+      if (!IsParamAtLeastAsConstrained) {
+        Diag(Arg.getLocation(),
+             diag::err_template_template_parameter_not_at_least_as_constrained)
+            << Template << Param << Arg.getSourceRange();
+        Diag(Param->getLocation(), diag::note_entity_declared_at) << Param;
+        Diag(Template->getLocation(), diag::note_entity_declared_at)
+            << Template;
+        MaybeEmitAmbiguousAtomicConstraintsDiagnostic(Param, ParamsAC, Template,
+                                                      TemplateAC);
+        return true;
+      }
+      return false;
+    }
+    // FIXME: Produce better diagnostics for deduction failures.
   }
-  return false;
+
+  return !TemplateParameterListsAreEqual(Template->getTemplateParameters(),
+                                         Params,
+                                         true,
+                                         TPL_TemplateTemplateArgumentMatch,
+                                         Arg.getLocation());
 }
 
 static Sema::SemaDiagnosticBuilder noteLocation(Sema &S, const NamedDecl &Decl,
@@ -8453,15 +8461,12 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
       Diag(TemplateNameLoc, diag::err_partial_spec_args_match_primary_template)
           << /*class template*/ 0 << (TUK == TagUseKind::Definition)
           << FixItHint::CreateRemoval(SourceRange(LAngleLoc, RAngleLoc));
-      return CheckClassTemplate(S, TagSpec, TUK, KWLoc, SS,
-                                ClassTemplate->getIdentifier(),
-                                TemplateNameLoc,
-                                Attr,
-                                TemplateParams,
-                                AS_none, /*ModulePrivateLoc=*/SourceLocation(),
-                                /*FriendLoc*/SourceLocation(),
-                                TemplateParameterLists.size() - 1,
-                                TemplateParameterLists.data());
+      return CheckClassTemplate(
+          S, TagSpec, TUK, KWLoc, SS, ClassTemplate->getIdentifier(),
+          TemplateNameLoc, Attr, TemplateParams, AS_none,
+          /*ModulePrivateLoc=*/SourceLocation(),
+          /*FriendLoc*/ SourceLocation(), TemplateParameterLists.drop_back(),
+          isMemberSpecialization);
     }
 
     // Create a new class template partial specialization declaration node.
@@ -8471,6 +8476,11 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
         ClassTemplatePartialSpecializationDecl::Create(
             Context, Kind, DC, KWLoc, TemplateNameLoc, TemplateParams,
             ClassTemplate, CanonicalConverted, CanonType, PrevPartial);
+
+    // If we are providing an explicit specialization of a member class
+    // template specialization, make a note of that.
+    if (isMemberSpecialization)
+      Partial->setMemberSpecialization();
     Partial->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Partial, SS);
     if (TemplateParameterLists.size() > 1 && SS.isSet()) {
@@ -8482,11 +8492,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
       ClassTemplate->AddPartialSpecialization(Partial, InsertPos);
     Specialization = Partial;
 
-    // If we are providing an explicit specialization of a member class
-    // template specialization, make a note of that.
-    if (PrevPartial && PrevPartial->getInstantiatedFromMember())
-      PrevPartial->setMemberSpecialization();
-
     CheckTemplatePartialSpecialization(Partial);
   } else {
     // Create a new class template specialization declaration node for
@@ -9086,8 +9091,8 @@ bool Sema::CheckFunctionTemplateSpecialization(
       TemplateDeductionInfo Info(FailedCandidates.getLocation());
       FunctionDecl *Specialization = nullptr;
       if (TemplateDeductionResult TDK = DeduceTemplateArguments(
-              cast<FunctionTemplateDecl>(FunTmpl->getFirstDecl()),
-              ExplicitTemplateArgs ? &Args : nullptr, FT, Specialization, Info);
+              FunTmpl, ExplicitTemplateArgs ? &Args : nullptr, FT,
+              Specialization, Info);
           TDK != TemplateDeductionResult::Success) {
         // Template argument deduction failed; record why it failed, so
         // that we can provide nifty diagnostics.
@@ -9758,14 +9763,11 @@ DeclResult Sema::ActOnExplicitInstantiation(
 
   // Check that the template argument list is well-formed for this
   // template.
-  bool PrimaryHasMatchedPackOnParmToNonPackOnArg = false;
   SmallVector<TemplateArgument, 4> SugaredConverted, CanonicalConverted;
-  if (CheckTemplateArgumentList(
-          ClassTemplate, TemplateNameLoc, TemplateArgs,
-          /*DefaultArgs=*/{}, false, SugaredConverted, CanonicalConverted,
-          /*UpdateArgsWithConversions=*/true,
-          /*ConstraintsNotSatisfied=*/nullptr, /*PartialOrderingTTP=*/false,
-          &PrimaryHasMatchedPackOnParmToNonPackOnArg))
+  if (CheckTemplateArgumentList(ClassTemplate, TemplateNameLoc, TemplateArgs,
+                                /*DefaultArgs=*/{}, false, SugaredConverted,
+                                CanonicalConverted,
+                                /*UpdateArgsWithConversions=*/true))
     return true;
 
   // Find the class template specialization declaration that
@@ -9886,9 +9888,7 @@ DeclResult Sema::ActOnExplicitInstantiation(
     = cast_or_null<ClassTemplateSpecializationDecl>(
                                               Specialization->getDefinition());
   if (!Def)
-    InstantiateClassTemplateSpecialization(
-        TemplateNameLoc, Specialization, TSK,
-        /*Complain=*/true, PrimaryHasMatchedPackOnParmToNonPackOnArg);
+    InstantiateClassTemplateSpecialization(TemplateNameLoc, Specialization, TSK);
   else if (TSK == TSK_ExplicitInstantiationDefinition) {
     MarkVTableUsed(TemplateNameLoc, Specialization, true);
     Specialization->setPointOfInstantiation(Def->getPointOfInstantiation());
@@ -11290,8 +11290,8 @@ private:
 
   template<typename TemplDecl>
   void checkTemplate(TemplDecl *TD) {
-    if (TD->isMemberSpecialization()) {
-      if (!CheckMemberSpecialization(TD))
+    if (TD->getMostRecentDecl()->isMemberSpecialization()) {
+      if (!CheckMemberSpecialization(TD->getMostRecentDecl()))
         diagnose(TD->getMostRecentDecl(), false);
     }
   }
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index e49d315f7186bca02d88bae209d86015407d88dc..7cfb8d687c796a92fe117352d597c0c53515b774 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -145,9 +145,7 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
     PartialOrderingKind POK, bool DeducedFromArrayBound,
     bool *HasDeducedAnyParam);
 
-/// What directions packs are allowed to match non-packs.
-enum class PackFold { ParameterToArgument, ArgumentToParameter, Both };
-
+enum class PackFold { ParameterToArgument, ArgumentToParameter };
 static TemplateDeductionResult
 DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
                         ArrayRef<TemplateArgument> Ps,
@@ -1713,21 +1711,7 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
     DeducedTemplateArgument Result =
         checkDeducedTemplateArguments(S.Context, Deduced[Index], NewDeduced);
     if (Result.isNull()) {
-      // We can also get inconsistencies when matching NTTP type.
-      switch (NamedDecl *Param = TemplateParams->getParam(Index);
-              Param->getKind()) {
-      case Decl::TemplateTypeParm:
-        Info.Param = cast<TemplateTypeParmDecl>(Param);
-        break;
-      case Decl::NonTypeTemplateParm:
-        Info.Param = cast<NonTypeTemplateParmDecl>(Param);
-        break;
-      case Decl::TemplateTemplateParm:
-        Info.Param = cast<TemplateTemplateParmDecl>(Param);
-        break;
-      default:
-        llvm_unreachable("unexpected kind");
-      }
+      Info.Param = cast<TemplateTypeParmDecl>(TemplateParams->getParam(Index));
       Info.FirstArg = Deduced[Index];
       Info.SecondArg = NewDeduced;
       return TemplateDeductionResult::Inconsistent;
@@ -2565,31 +2549,8 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     if (const NonTypeTemplateParmDecl *NTTP =
             getDeducedParameterFromExpr(Info, P.getAsExpr())) {
       switch (A.getKind()) {
-      case TemplateArgument::Expression: {
-        const Expr *E = A.getAsExpr();
-        // When checking NTTP, if either the parameter or the argument is
-        // dependent, as there would be otherwise nothing to deduce, we force
-        // the argument to the parameter type using this dependent implicit
-        // cast, in order to maintain invariants. Now we can deduce the
-        // resulting type from the original type, and deduce the original type
-        // against the parameter we are checking.
-        if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E);
-            ICE && ICE->getCastKind() == clang::CK_Dependent) {
-          E = ICE->getSubExpr();
-          if (auto Result = DeduceTemplateArgumentsByTypeMatch(
-                  S, TemplateParams, ICE->getType(), E->getType(), Info,
-                  Deduced, TDF_SkipNonDependent,
-                  PartialOrdering ? PartialOrderingKind::NonCall
-                                  : PartialOrderingKind::None,
-                  /*DeducedFromArrayBound=*/false, HasDeducedAnyParam);
-              Result != TemplateDeductionResult::Success)
-            return Result;
-        }
-        return DeduceNonTypeTemplateArgument(
-            S, TemplateParams, NTTP, DeducedTemplateArgument(A), E->getType(),
-            Info, PartialOrdering, Deduced, HasDeducedAnyParam);
-      }
       case TemplateArgument::Integral:
+      case TemplateArgument::Expression:
       case TemplateArgument::StructuralValue:
         return DeduceNonTypeTemplateArgument(
             S, TemplateParams, NTTP, DeducedTemplateArgument(A),
@@ -2678,75 +2639,50 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
                         SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         bool NumberOfArgumentsMustMatch, bool PartialOrdering,
                         PackFold PackFold, bool *HasDeducedAnyParam) {
-  bool FoldPackParameter = PackFold == PackFold::ParameterToArgument ||
-                           PackFold == PackFold::Both,
-       FoldPackArgument = PackFold == PackFold::ArgumentToParameter ||
-                          PackFold == PackFold::Both;
-
+  if (PackFold == PackFold::ArgumentToParameter)
+    std::swap(Ps, As);
   // C++0x [temp.deduct.type]p9:
   //   If the template argument list of P contains a pack expansion that is not
   //   the last template argument, the entire template argument list is a
   //   non-deduced context.
-  if (FoldPackParameter && hasPackExpansionBeforeEnd(Ps))
-    return TemplateDeductionResult::Success;
-
-  if (FoldPackArgument && hasPackExpansionBeforeEnd(As))
+  if (hasPackExpansionBeforeEnd(Ps))
     return TemplateDeductionResult::Success;
 
   // C++0x [temp.deduct.type]p9:
   //   If P has a form that contains <T> or <i>, then each argument Pi of the
   //   respective template argument list P is compared with the corresponding
   //   argument Ai of the corresponding template argument list of A.
-  for (unsigned ArgIdx = 0, ParamIdx = 0; /**/; /**/) {
-    if (!hasTemplateArgumentForDeduction(Ps, ParamIdx))
-      return !FoldPackParameter && hasTemplateArgumentForDeduction(As, ArgIdx)
-                 ? TemplateDeductionResult::MiscellaneousDeductionFailure
-                 : TemplateDeductionResult::Success;
-
-    if (!Ps[ParamIdx].isPackExpansion()) {
+  unsigned ArgIdx = 0, ParamIdx = 0;
+  for (; hasTemplateArgumentForDeduction(Ps, ParamIdx); ++ParamIdx) {
+    const TemplateArgument &P = Ps[ParamIdx];
+    if (!P.isPackExpansion()) {
       // The simple case: deduce template arguments by matching Pi and Ai.
 
       // Check whether we have enough arguments.
       if (!hasTemplateArgumentForDeduction(As, ArgIdx))
-        return !FoldPackArgument && NumberOfArgumentsMustMatch
+        return NumberOfArgumentsMustMatch
                    ? TemplateDeductionResult::MiscellaneousDeductionFailure
                    : TemplateDeductionResult::Success;
 
-      if (As[ArgIdx].isPackExpansion()) {
-        // C++1z [temp.deduct.type]p9:
-        //   During partial ordering, if Ai was originally a pack expansion
-        //   [and] Pi is not a pack expansion, template argument deduction
-        //   fails.
-        if (!FoldPackArgument)
-          return TemplateDeductionResult::MiscellaneousDeductionFailure;
-
-        TemplateArgument Pattern = As[ArgIdx].getPackExpansionPattern();
-        for (;;) {
-          // Deduce template parameters from the pattern.
-          if (auto Result = DeduceTemplateArguments(
-                  S, TemplateParams, Ps[ParamIdx], Pattern, Info,
-                  PartialOrdering, Deduced, HasDeducedAnyParam);
-              Result != TemplateDeductionResult::Success)
-            return Result;
+      // C++1z [temp.deduct.type]p9:
+      //   During partial ordering, if Ai was originally a pack expansion [and]
+      //   Pi is not a pack expansion, template argument deduction fails.
+      if (As[ArgIdx].isPackExpansion())
+        return TemplateDeductionResult::MiscellaneousDeductionFailure;
 
-          ++ParamIdx;
-          if (!hasTemplateArgumentForDeduction(Ps, ParamIdx))
-            return TemplateDeductionResult::Success;
-          if (Ps[ParamIdx].isPackExpansion())
-            break;
-        }
-      } else {
-        // Perform deduction for this Pi/Ai pair.
-        if (auto Result = DeduceTemplateArguments(
-                S, TemplateParams, Ps[ParamIdx], As[ArgIdx], Info,
-                PartialOrdering, Deduced, HasDeducedAnyParam);
-            Result != TemplateDeductionResult::Success)
-          return Result;
+      // Perform deduction for this Pi/Ai pair.
+      TemplateArgument Pi = P, Ai = As[ArgIdx];
+      if (PackFold == PackFold::ArgumentToParameter)
+        std::swap(Pi, Ai);
+      if (auto Result = DeduceTemplateArguments(S, TemplateParams, Pi, Ai, Info,
+                                                PartialOrdering, Deduced,
+                                                HasDeducedAnyParam);
+          Result != TemplateDeductionResult::Success)
+        return Result;
 
-        ++ArgIdx;
-        ++ParamIdx;
-        continue;
-      }
+      // Move to the next argument.
+      ++ArgIdx;
+      continue;
     }
 
     // The parameter is a pack expansion.
@@ -2756,7 +2692,7 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     //   each remaining argument in the template argument list of A. Each
     //   comparison deduces template arguments for subsequent positions in the
     //   template parameter packs expanded by Pi.
-    TemplateArgument Pattern = Ps[ParamIdx].getPackExpansionPattern();
+    TemplateArgument Pattern = P.getPackExpansionPattern();
 
     // Prepare to deduce the packs within the pattern.
     PackDeductionScope PackScope(S, TemplateParams, Deduced, Info, Pattern);
@@ -2767,16 +2703,13 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     for (; hasTemplateArgumentForDeduction(As, ArgIdx) &&
            PackScope.hasNextElement();
          ++ArgIdx) {
-      if (!As[ArgIdx].isPackExpansion()) {
-        if (!FoldPackParameter)
-          return TemplateDeductionResult::MiscellaneousDeductionFailure;
-        if (FoldPackArgument)
-          Info.setMatchedPackOnParmToNonPackOnArg();
-      }
+      TemplateArgument Pi = Pattern, Ai = As[ArgIdx];
+      if (PackFold == PackFold::ArgumentToParameter)
+        std::swap(Pi, Ai);
       // Deduce template arguments from the pattern.
-      if (auto Result = DeduceTemplateArguments(
-              S, TemplateParams, Pattern, As[ArgIdx], Info, PartialOrdering,
-              Deduced, HasDeducedAnyParam);
+      if (auto Result = DeduceTemplateArguments(S, TemplateParams, Pi, Ai, Info,
+                                                PartialOrdering, Deduced,
+                                                HasDeducedAnyParam);
           Result != TemplateDeductionResult::Success)
         return Result;
 
@@ -2785,8 +2718,12 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
 
     // Build argument packs for each of the parameter packs expanded by this
     // pack expansion.
-    return PackScope.finish();
+    if (auto Result = PackScope.finish();
+        Result != TemplateDeductionResult::Success)
+      return Result;
   }
+
+  return TemplateDeductionResult::Success;
 }
 
 TemplateDeductionResult Sema::DeduceTemplateArguments(
@@ -2955,7 +2892,7 @@ Sema::getIdentityTemplateArgumentLoc(NamedDecl *TemplateParm,
 /// fully-converted template arguments.
 static bool ConvertDeducedTemplateArgument(
     Sema &S, NamedDecl *Param, DeducedTemplateArgument Arg, NamedDecl *Template,
-    TemplateDeductionInfo &Info, bool IsDeduced, bool PartialOrdering,
+    TemplateDeductionInfo &Info, bool IsDeduced,
     SmallVectorImpl<TemplateArgument> &SugaredOutput,
     SmallVectorImpl<TemplateArgument> &CanonicalOutput) {
   auto ConvertArg = [&](DeducedTemplateArgument Arg,
@@ -2966,20 +2903,15 @@ static bool ConvertDeducedTemplateArgument(
     TemplateArgumentLoc ArgLoc = S.getTrivialTemplateArgumentLoc(
         Arg, QualType(), Info.getLocation(), Param);
 
-    bool MatchedPackOnParmToNonPackOnArg = false;
     // Check the template argument, converting it as necessary.
-    auto Res = S.CheckTemplateArgument(
+    return S.CheckTemplateArgument(
         Param, ArgLoc, Template, Template->getLocation(),
         Template->getSourceRange().getEnd(), ArgumentPackIndex, SugaredOutput,
         CanonicalOutput,
         IsDeduced
             ? (Arg.wasDeducedFromArrayBound() ? Sema::CTAK_DeducedFromArrayBound
                                               : Sema::CTAK_Deduced)
-            : Sema::CTAK_Specified,
-        PartialOrdering, &MatchedPackOnParmToNonPackOnArg);
-    if (MatchedPackOnParmToNonPackOnArg)
-      Info.setMatchedPackOnParmToNonPackOnArg();
-    return Res;
+            : Sema::CTAK_Specified);
   };
 
   if (Arg.getKind() == TemplateArgument::Pack) {
@@ -3062,9 +2994,9 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
     SmallVectorImpl<DeducedTemplateArgument> &Deduced,
     TemplateDeductionInfo &Info,
     SmallVectorImpl<TemplateArgument> &SugaredBuilder,
-    SmallVectorImpl<TemplateArgument> &CanonicalBuilder, bool PartialOrdering,
-    LocalInstantiationScope *CurrentInstantiationScope,
-    unsigned NumAlreadyConverted, bool *IsIncomplete) {
+    SmallVectorImpl<TemplateArgument> &CanonicalBuilder,
+    LocalInstantiationScope *CurrentInstantiationScope = nullptr,
+    unsigned NumAlreadyConverted = 0, bool *IsIncomplete = nullptr) {
   TemplateParameterList *TemplateParams = Template->getTemplateParameters();
 
   for (unsigned I = 0, N = TemplateParams->size(); I != N; ++I) {
@@ -3107,8 +3039,8 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
       // We may have deduced this argument, so it still needs to be
       // checked and converted.
       if (ConvertDeducedTemplateArgument(S, Param, Deduced[I], Template, Info,
-                                         IsDeduced, PartialOrdering,
-                                         SugaredBuilder, CanonicalBuilder)) {
+                                         IsDeduced, SugaredBuilder,
+                                         CanonicalBuilder)) {
         Info.Param = makeTemplateParameter(Param);
         // FIXME: These template arguments are temporary. Free them!
         Info.reset(
@@ -3174,9 +3106,7 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
     // Check whether we can actually use the default argument.
     if (S.CheckTemplateArgument(
             Param, DefArg, TD, TD->getLocation(), TD->getSourceRange().getEnd(),
-            /*ArgumentPackIndex=*/0, SugaredBuilder, CanonicalBuilder,
-            Sema::CTAK_Specified, /*PartialOrdering=*/false,
-            /*MatchedPackOnParmToNonPackOnArg=*/nullptr)) {
+            0, SugaredBuilder, CanonicalBuilder, Sema::CTAK_Specified)) {
       Info.Param = makeTemplateParameter(
                          const_cast<NamedDecl *>(TemplateParams->getParam(I)));
       // FIXME: These template arguments are temporary. Free them!
@@ -3208,20 +3138,6 @@ template<>
 struct IsPartialSpecialization<VarTemplatePartialSpecializationDecl> {
   static constexpr bool value = true;
 };
-template <typename TemplateDeclT>
-static bool DeducedArgsNeedReplacement(TemplateDeclT *Template) {
-  return false;
-}
-template <>
-bool DeducedArgsNeedReplacement<VarTemplatePartialSpecializationDecl>(
-    VarTemplatePartialSpecializationDecl *Spec) {
-  return !Spec->isClassScopeExplicitSpecialization();
-}
-template <>
-bool DeducedArgsNeedReplacement<ClassTemplatePartialSpecializationDecl>(
-    ClassTemplatePartialSpecializationDecl *Spec) {
-  return !Spec->isClassScopeExplicitSpecialization();
-}
 
 template <typename TemplateDeclT>
 static TemplateDeductionResult
@@ -3232,23 +3148,10 @@ CheckDeducedArgumentConstraints(Sema &S, TemplateDeclT *Template,
   llvm::SmallVector<const Expr *, 3> AssociatedConstraints;
   Template->getAssociatedConstraints(AssociatedConstraints);
 
-  std::optional<ArrayRef<TemplateArgument>> Innermost;
-  // If we don't need to replace the deduced template arguments,
-  // we can add them immediately as the inner-most argument list.
-  if (!DeducedArgsNeedReplacement(Template))
-    Innermost = CanonicalDeducedArgs;
-
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
-      Template, Template->getDeclContext(), /*Final=*/false, Innermost,
-      /*RelativeToPrimary=*/true, /*Pattern=*/
-      nullptr, /*ForConstraintInstantiation=*/true);
-
-  // getTemplateInstantiationArgs picks up the non-deduced version of the
-  // template args when this is a variable template partial specialization and
-  // not class-scope explicit specialization, so replace with Deduced Args
-  // instead of adding to inner-most.
-  if (!Innermost)
-    MLTAL.replaceInnermostTemplateArguments(Template, CanonicalDeducedArgs);
+      Template, Template->getDeclContext(), /*Final=*/false,
+      /*Innermost=*/CanonicalDeducedArgs, /*RelativeToPrimary=*/true,
+      /*ForConstraintInstantiation=*/true);
 
   if (S.CheckConstraintSatisfaction(Template, AssociatedConstraints, MLTAL,
                                     Info.getLocation(),
@@ -3284,9 +3187,7 @@ FinishTemplateArgumentDeduction(
   SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
   if (auto Result = ConvertDeducedTemplateArguments(
           S, Partial, IsPartialOrdering, Deduced, Info, SugaredBuilder,
-          CanonicalBuilder, IsPartialOrdering,
-          /*CurrentInstantiationScope=*/nullptr, /*NumAlreadyConverted=*/0,
-          /*IsIncomplete=*/nullptr);
+          CanonicalBuilder);
       Result != TemplateDeductionResult::Success)
     return Result;
 
@@ -3327,20 +3228,16 @@ FinishTemplateArgumentDeduction(
     return TemplateDeductionResult::SubstitutionFailure;
   }
 
-  bool MatchedPackOnParmToNonPackOnArg = false;
   bool ConstraintsNotSatisfied;
   SmallVector<TemplateArgument, 4> SugaredConvertedInstArgs,
       CanonicalConvertedInstArgs;
   if (S.CheckTemplateArgumentList(
           Template, Partial->getLocation(), InstArgs, /*DefaultArgs=*/{}, false,
           SugaredConvertedInstArgs, CanonicalConvertedInstArgs,
-          /*UpdateArgsWithConversions=*/true, &ConstraintsNotSatisfied,
-          /*PartialOrderingTTP=*/false, &MatchedPackOnParmToNonPackOnArg))
+          /*UpdateArgsWithConversions=*/true, &ConstraintsNotSatisfied))
     return ConstraintsNotSatisfied
                ? TemplateDeductionResult::ConstraintsNotSatisfied
                : TemplateDeductionResult::SubstitutionFailure;
-  if (MatchedPackOnParmToNonPackOnArg)
-    Info.setMatchedPackOnParmToNonPackOnArg();
 
   TemplateParameterList *TemplateParams = Template->getTemplateParameters();
   for (unsigned I = 0, E = TemplateParams->size(); I != E; ++I) {
@@ -3378,6 +3275,7 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   // Unevaluated SFINAE context.
   EnterExpressionEvaluationContext Unevaluated(
       S, Sema::ExpressionEvaluationContext::Unevaluated);
+  Sema::SFINAETrap Trap(S);
 
   Sema::ContextRAII SavedContext(S, getAsDeclContextOrEnclosing(Template));
 
@@ -3386,45 +3284,29 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   //   explicitly specified, template argument deduction fails.
   SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
   if (auto Result = ConvertDeducedTemplateArguments(
-          S, Template, /*IsDeduced=*/PartialOrdering, Deduced, Info,
-          SugaredBuilder, CanonicalBuilder, PartialOrdering,
+          S, Template, /*IsDeduced*/ PartialOrdering, Deduced, Info,
+          SugaredBuilder, CanonicalBuilder,
           /*CurrentInstantiationScope=*/nullptr,
-          /*NumAlreadyConverted=*/0U, /*IsIncomplete=*/nullptr);
+          /*NumAlreadyConverted=*/0U);
       Result != TemplateDeductionResult::Success)
     return Result;
 
   // Check that we produced the correct argument list.
   TemplateParameterList *TemplateParams = Template->getTemplateParameters();
-  auto isSame = [&](unsigned I, const TemplateArgument &P,
-                    const TemplateArgument &A) {
-    if (isSameTemplateArg(S.Context, P, A, PartialOrdering,
-                          /*PackExpansionMatchesPack=*/true))
-      return true;
-    Info.Param = makeTemplateParameter(TemplateParams->getParam(I));
-    Info.FirstArg = P;
-    Info.SecondArg = A;
-    return false;
-  };
   for (unsigned I = 0, E = TemplateParams->size(); I != E; ++I) {
-    const TemplateArgument &P = TemplateArgs[I];
-    if (P.isPackExpansion()) {
-      assert(I == TemplateArgs.size() - 1);
-      for (/**/; I != E; ++I) {
-        const TemplateArgument &A = CanonicalBuilder[I];
-        if (A.getKind() == TemplateArgument::Pack) {
-          for (const TemplateArgument &Ai : A.getPackAsArray())
-            if (!isSame(I, P, Ai))
-              return TemplateDeductionResult::NonDeducedMismatch;
-        } else if (!isSame(I, P, A)) {
-          return TemplateDeductionResult::NonDeducedMismatch;
-        }
-      }
-      break;
-    }
-    if (!isSame(I, P, CanonicalBuilder[I]))
+    TemplateArgument InstArg = CanonicalBuilder[I];
+    if (!isSameTemplateArg(S.Context, TemplateArgs[I], InstArg, PartialOrdering,
+                           /*PackExpansionMatchesPack=*/true)) {
+      Info.Param = makeTemplateParameter(TemplateParams->getParam(I));
+      Info.FirstArg = TemplateArgs[I];
+      Info.SecondArg = InstArg;
       return TemplateDeductionResult::NonDeducedMismatch;
+    }
   }
 
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
   if (!PartialOrdering) {
     if (auto Result = CheckDeducedArgumentConstraints(
             S, Template, SugaredBuilder, CanonicalBuilder, Info);
@@ -3445,6 +3327,7 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   // Unevaluated SFINAE context.
   EnterExpressionEvaluationContext Unevaluated(
       S, Sema::ExpressionEvaluationContext::Unevaluated);
+  Sema::SFINAETrap Trap(S);
 
   Sema::ContextRAII SavedContext(S, getAsDeclContextOrEnclosing(TD));
 
@@ -3453,15 +3336,20 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   //   explicitly specified, template argument deduction fails.
   SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
   if (auto Result = ConvertDeducedTemplateArguments(
-          S, TD, /*IsDeduced=*/false, Deduced, Info, SugaredBuilder,
-          CanonicalBuilder, /*PartialOrdering=*/false,
-          /*CurrentInstantiationScope=*/nullptr, /*NumAlreadyConverted=*/0,
-          /*IsIncomplete=*/nullptr);
+          S, TD, /*IsPartialOrdering=*/false, Deduced, Info, SugaredBuilder,
+          CanonicalBuilder);
+      Result != TemplateDeductionResult::Success)
+    return Result;
+
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
+  if (auto Result = CheckDeducedArgumentConstraints(S, TD, SugaredBuilder,
+                                                    CanonicalBuilder, Info);
       Result != TemplateDeductionResult::Success)
     return Result;
 
-  return ::CheckDeducedArgumentConstraints(S, TD, SugaredBuilder,
-                                           CanonicalBuilder, Info);
+  return TemplateDeductionResult::Success;
 }
 
 /// Perform template argument deduction to determine whether the given template
@@ -3508,20 +3396,16 @@ DeduceTemplateArguments(Sema &S, T *Partial,
   if (Inst.isInvalid())
     return TemplateDeductionResult::InstantiationDepth;
 
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
   TemplateDeductionResult Result;
   S.runWithSufficientStackSpace(Info.getLocation(), [&] {
     Result = ::FinishTemplateArgumentDeduction(S, Partial,
                                                /*IsPartialOrdering=*/false,
                                                TemplateArgs, Deduced, Info);
   });
-
-  if (Result != TemplateDeductionResult::Success)
-    return Result;
-
-  if (Trap.hasErrorOccurred())
-    return TemplateDeductionResult::SubstitutionFailure;
-
-  return TemplateDeductionResult::Success;
+  return Result;
 }
 
 TemplateDeductionResult
@@ -3577,18 +3461,14 @@ Sema::DeduceTemplateArgumentsFromType(TemplateDecl *TD, QualType FromType,
   if (Inst.isInvalid())
     return TemplateDeductionResult::InstantiationDepth;
 
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
   TemplateDeductionResult Result;
   runWithSufficientStackSpace(Info.getLocation(), [&] {
     Result = ::FinishTemplateArgumentDeduction(*this, TD, Deduced, Info);
   });
-
-  if (Result != TemplateDeductionResult::Success)
-    return Result;
-
-  if (Trap.hasErrorOccurred())
-    return TemplateDeductionResult::SubstitutionFailure;
-
-  return TemplateDeductionResult::Success;
+  return Result;
 }
 
 /// Determine whether the given type T is a simple-template-id type.
@@ -3994,8 +3874,7 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
     unsigned NumExplicitlySpecified, FunctionDecl *&Specialization,
     TemplateDeductionInfo &Info,
     SmallVectorImpl<OriginalCallArg> const *OriginalCallArgs,
-    bool PartialOverloading, bool PartialOrdering,
-    llvm::function_ref<bool()> CheckNonDependent) {
+    bool PartialOverloading, llvm::function_ref<bool()> CheckNonDependent) {
   // Unevaluated SFINAE context.
   EnterExpressionEvaluationContext Unevaluated(
       *this, Sema::ExpressionEvaluationContext::Unevaluated);
@@ -4018,10 +3897,9 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
   bool IsIncomplete = false;
   SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
   if (auto Result = ConvertDeducedTemplateArguments(
-          *this, FunctionTemplate, /*IsDeduced=*/true, Deduced, Info,
-          SugaredBuilder, CanonicalBuilder, PartialOrdering,
-          CurrentInstantiationScope, NumExplicitlySpecified,
-          PartialOverloading ? &IsIncomplete : nullptr);
+          *this, FunctionTemplate, /*IsDeduced*/ true, Deduced, Info,
+          SugaredBuilder, CanonicalBuilder, CurrentInstantiationScope,
+          NumExplicitlySpecified, PartialOverloading ? &IsIncomplete : nullptr);
       Result != TemplateDeductionResult::Success)
     return Result;
 
@@ -4179,11 +4057,10 @@ TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
   // keep track of these diagnostics. They'll be emitted if this specialization
   // is actually used.
   if (Info.diag_begin() != Info.diag_end()) {
-    SuppressedDiagnosticsMap::iterator
-      Pos = SuppressedDiagnostics.find(Specialization->getCanonicalDecl());
-    if (Pos == SuppressedDiagnostics.end())
-        SuppressedDiagnostics[Specialization->getCanonicalDecl()]
-          .append(Info.diag_begin(), Info.diag_end());
+    auto [Pos, Inserted] =
+        SuppressedDiagnostics.try_emplace(Specialization->getCanonicalDecl());
+    if (Inserted)
+      Pos->second.append(Info.diag_begin(), Info.diag_end());
   }
 
   return TemplateDeductionResult::Success;
@@ -4553,8 +4430,7 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
     TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
     FunctionDecl *&Specialization, TemplateDeductionInfo &Info,
     bool PartialOverloading, bool AggregateDeductionCandidate,
-    bool PartialOrdering, QualType ObjectType,
-    Expr::Classification ObjectClassification,
+    QualType ObjectType, Expr::Classification ObjectClassification,
     llvm::function_ref<bool(ArrayRef<QualType>)> CheckNonDependent) {
   if (FunctionTemplate->isInvalidDecl())
     return TemplateDeductionResult::Invalid;
@@ -4769,8 +4645,7 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
   runWithSufficientStackSpace(Info.getLocation(), [&] {
     Result = FinishTemplateArgumentDeduction(
         FunctionTemplate, Deduced, NumExplicitlySpecified, Specialization, Info,
-        &OriginalCallArgs, PartialOverloading, PartialOrdering,
-        [&, CallingCtx]() {
+        &OriginalCallArgs, PartialOverloading, [&, CallingCtx]() {
           ContextRAII SavedContext(*this, CallingCtx);
           return CheckNonDependent(ParamTypesForArgChecking);
         });
@@ -4882,10 +4757,9 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
 
   TemplateDeductionResult Result;
   runWithSufficientStackSpace(Info.getLocation(), [&] {
-    Result = FinishTemplateArgumentDeduction(
-        FunctionTemplate, Deduced, NumExplicitlySpecified, Specialization, Info,
-        /*OriginalCallArgs=*/nullptr, /*PartialOverloading=*/false,
-        /*PartialOrdering=*/true);
+    Result = FinishTemplateArgumentDeduction(FunctionTemplate, Deduced,
+                                             NumExplicitlySpecified,
+                                             Specialization, Info);
   });
   if (Result != TemplateDeductionResult::Success)
     return Result;
@@ -5065,10 +4939,9 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
   FunctionDecl *ConversionSpecialized = nullptr;
   TemplateDeductionResult Result;
   runWithSufficientStackSpace(Info.getLocation(), [&] {
-    Result = FinishTemplateArgumentDeduction(
-        ConversionTemplate, Deduced, 0, ConversionSpecialized, Info,
-        &OriginalCallArgs, /*PartialOverloading=*/false,
-        /*PartialOrdering=*/false);
+    Result = FinishTemplateArgumentDeduction(ConversionTemplate, Deduced, 0,
+                                             ConversionSpecialized, Info,
+                                             &OriginalCallArgs);
   });
   Specialization = cast_or_null<CXXConversionDecl>(ConversionSpecialized);
   return Result;
@@ -5645,8 +5518,7 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
   if (auto Result = ConvertDeducedTemplateArguments(
           S, FTD, /*IsDeduced=*/true, Deduced, Info, SugaredBuilder,
-          CanonicalBuilder, /*PartialOrdering=*/true,
-          /*CurrentInstantiationScope=*/nullptr,
+          CanonicalBuilder, /*CurrentInstantiationScope=*/nullptr,
           /*NumAlreadyConverted=*/0, &IsIncomplete);
       Result != TemplateDeductionResult::Success)
     return Result;
@@ -6236,23 +6108,14 @@ static bool isAtLeastAsSpecializedAs(Sema &S, QualType T1, QualType T2,
     return false;
 
   const auto *TST1 = cast<TemplateSpecializationType>(T1);
-
-  Sema::SFINAETrap Trap(S);
-
-  TemplateDeductionResult Result;
+  bool AtLeastAsSpecialized;
   S.runWithSufficientStackSpace(Info.getLocation(), [&] {
-    Result = ::FinishTemplateArgumentDeduction(
-        S, P2, /*IsPartialOrdering=*/true, TST1->template_arguments(), Deduced,
-        Info);
+    AtLeastAsSpecialized =
+        FinishTemplateArgumentDeduction(
+            S, P2, /*IsPartialOrdering=*/true, TST1->template_arguments(),
+            Deduced, Info) == TemplateDeductionResult::Success;
   });
-
-  if (Result != TemplateDeductionResult::Success)
-    return false;
-
-  if (Trap.hasErrorOccurred())
-    return false;
-
-  return true;
+  return AtLeastAsSpecialized;
 }
 
 namespace {
@@ -6490,9 +6353,8 @@ bool Sema::isMoreSpecializedThanPrimary(
 }
 
 bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
-    TemplateParameterList *P, TemplateDecl *PArg, TemplateDecl *AArg,
-    const DefaultArguments &DefaultArgs, SourceLocation ArgLoc,
-    bool PartialOrdering, bool *MatchedPackOnParmToNonPackOnArg) {
+    TemplateParameterList *P, TemplateDecl *AArg,
+    const DefaultArguments &DefaultArgs, SourceLocation Loc, bool IsDeduced) {
   // C++1z [temp.arg.template]p4: (DR 150)
   //   A template template-parameter P is at least as specialized as a
   //   template template-argument A if, given the following rewrite to two
@@ -6504,12 +6366,6 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   //
   TemplateParameterList *A = AArg->getTemplateParameters();
 
-  Sema::InstantiatingTemplate Inst(
-      *this, ArgLoc, Sema::InstantiatingTemplate::PartialOrderingTTP(), PArg,
-      SourceRange(P->getTemplateLoc(), P->getRAngleLoc()));
-  if (Inst.isInvalid())
-    return false;
-
   //   Given an invented class template X with the template parameter list of
   //   A (including default arguments):
   //    - Each function template has a single function parameter whose type is
@@ -6524,6 +6380,8 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   // templates.
   SmallVector<TemplateArgument, 4> PArgs;
   {
+    SFINAETrap Trap(*this);
+
     Context.getInjectedTemplateArgs(P, PArgs);
     TemplateArgumentListInfo PArgList(P->getLAngleLoc(),
                                       P->getRAngleLoc());
@@ -6543,17 +6401,18 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
     // C++1z [temp.arg.template]p3:
     //   If the rewrite produces an invalid type, then P is not at least as
     //   specialized as A.
-    SmallVector<TemplateArgument, 4> CanonicalPArgs;
-    if (CheckTemplateArgumentList(
-            AArg, ArgLoc, PArgList, DefaultArgs, false, PArgs, CanonicalPArgs,
-            /*UpdateArgsWithConversions=*/true,
-            /*ConstraintsNotSatisfied=*/nullptr,
-            /*PartialOrderingTTP=*/true, MatchedPackOnParmToNonPackOnArg))
+    SmallVector<TemplateArgument, 4> SugaredPArgs;
+    if (CheckTemplateArgumentList(AArg, Loc, PArgList, DefaultArgs, false,
+                                  SugaredPArgs, PArgs,
+                                  /*UpdateArgsWithConversions=*/true,
+                                  /*ConstraintsNotSatisfied=*/nullptr,
+                                  /*PartialOrderTTP=*/true) ||
+        Trap.hasErrorOccurred())
       return false;
   }
 
   // Determine whether P1 is at least as specialized as P2.
-  TemplateDeductionInfo Info(ArgLoc, A->getDepth());
+  TemplateDeductionInfo Info(Loc, A->getDepth());
   SmallVector<DeducedTemplateArgument, 4> Deduced;
   Deduced.resize(A->size());
 
@@ -6568,92 +6427,29 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   //   be inverted between Ps and As. On non-deduced context, matching needs to
   //   happen both ways, according to [temp.arg.template]p3, but this is
   //   currently implemented as a special case elsewhere.
-  switch (::DeduceTemplateArguments(
-      *this, A, AArgs, PArgs, Info, Deduced,
-      /*NumberOfArgumentsMustMatch=*/false, /*PartialOrdering=*/true,
-      PartialOrdering ? PackFold::ArgumentToParameter : PackFold::Both,
-      /*HasDeducedAnyParam=*/nullptr)) {
-  case clang::TemplateDeductionResult::Success:
-    if (MatchedPackOnParmToNonPackOnArg &&
-        Info.hasMatchedPackOnParmToNonPackOnArg())
-      *MatchedPackOnParmToNonPackOnArg = true;
-    break;
-
-  case TemplateDeductionResult::MiscellaneousDeductionFailure:
-    Diag(AArg->getLocation(), diag::err_template_param_list_different_arity)
-        << (A->size() > P->size()) << /*isTemplateTemplateParameter=*/true
-        << SourceRange(A->getTemplateLoc(), P->getRAngleLoc());
-    return false;
-  case TemplateDeductionResult::NonDeducedMismatch:
-    Diag(AArg->getLocation(), diag::err_non_deduced_mismatch)
-        << Info.FirstArg << Info.SecondArg;
-    return false;
-  case TemplateDeductionResult::Inconsistent:
-    Diag(getAsNamedDecl(Info.Param)->getLocation(),
-         diag::err_inconsistent_deduction)
-        << Info.FirstArg << Info.SecondArg;
-    return false;
-  case TemplateDeductionResult::AlreadyDiagnosed:
+  if (::DeduceTemplateArguments(*this, A, AArgs, PArgs, Info, Deduced,
+                                /*NumberOfArgumentsMustMatch=*/false,
+                                /*PartialOrdering=*/true,
+                                IsDeduced ? PackFold::ArgumentToParameter
+                                          : PackFold::ParameterToArgument,
+                                /*HasDeducedAnyParam=*/nullptr) !=
+      TemplateDeductionResult::Success)
     return false;
 
-  // None of these should happen for a plain deduction.
-  case TemplateDeductionResult::Invalid:
-  case TemplateDeductionResult::InstantiationDepth:
-  case TemplateDeductionResult::Incomplete:
-  case TemplateDeductionResult::IncompletePack:
-  case TemplateDeductionResult::Underqualified:
-  case TemplateDeductionResult::SubstitutionFailure:
-  case TemplateDeductionResult::DeducedMismatch:
-  case TemplateDeductionResult::DeducedMismatchNested:
-  case TemplateDeductionResult::TooManyArguments:
-  case TemplateDeductionResult::TooFewArguments:
-  case TemplateDeductionResult::InvalidExplicitArguments:
-  case TemplateDeductionResult::NonDependentConversionFailure:
-  case TemplateDeductionResult::ConstraintsNotSatisfied:
-  case TemplateDeductionResult::CUDATargetMismatch:
-    llvm_unreachable("Unexpected Result");
-  }
-
   SmallVector<TemplateArgument, 4> DeducedArgs(Deduced.begin(), Deduced.end());
+  Sema::InstantiatingTemplate Inst(*this, Info.getLocation(), AArg, DeducedArgs,
+                                   Info);
+  if (Inst.isInvalid())
+    return false;
 
-  TemplateDeductionResult TDK;
+  bool AtLeastAsSpecialized;
   runWithSufficientStackSpace(Info.getLocation(), [&] {
-    TDK = ::FinishTemplateArgumentDeduction(
-        *this, AArg, /*IsPartialOrdering=*/true, PArgs, Deduced, Info);
+    AtLeastAsSpecialized =
+        ::FinishTemplateArgumentDeduction(
+            *this, AArg, /*IsPartialOrdering=*/true, PArgs, Deduced, Info) ==
+        TemplateDeductionResult::Success;
   });
-  switch (TDK) {
-  case TemplateDeductionResult::Success:
-    return true;
-
-  // It doesn't seem possible to get a non-deduced mismatch when partial
-  // ordering TTPs.
-  case TemplateDeductionResult::NonDeducedMismatch:
-    llvm_unreachable("Unexpected NonDeducedMismatch");
-
-  // Substitution failures should have already been diagnosed.
-  case TemplateDeductionResult::AlreadyDiagnosed:
-  case TemplateDeductionResult::SubstitutionFailure:
-  case TemplateDeductionResult::InstantiationDepth:
-    return false;
-
-  // None of these should happen when just converting deduced arguments.
-  case TemplateDeductionResult::Invalid:
-  case TemplateDeductionResult::Incomplete:
-  case TemplateDeductionResult::IncompletePack:
-  case TemplateDeductionResult::Inconsistent:
-  case TemplateDeductionResult::Underqualified:
-  case TemplateDeductionResult::DeducedMismatch:
-  case TemplateDeductionResult::DeducedMismatchNested:
-  case TemplateDeductionResult::TooManyArguments:
-  case TemplateDeductionResult::TooFewArguments:
-  case TemplateDeductionResult::InvalidExplicitArguments:
-  case TemplateDeductionResult::NonDependentConversionFailure:
-  case TemplateDeductionResult::ConstraintsNotSatisfied:
-  case TemplateDeductionResult::MiscellaneousDeductionFailure:
-  case TemplateDeductionResult::CUDATargetMismatch:
-    llvm_unreachable("Unexpected Result");
-  }
-  llvm_unreachable("Unexpected TDK");
+  return AtLeastAsSpecialized;
 }
 
 namespace {
diff --git a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
index 2d3e58548fb7ac2c7e27d5bc66977fb429d04aef..3a87128a7db0fa23995ace8967083c8a3a81496f 100644
--- a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
+++ b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
@@ -765,7 +765,7 @@ buildAssociatedConstraints(Sema &SemaRef, FunctionTemplateDecl *F,
   }
   // Template arguments used to transform the template arguments in
   // DeducedResults.
-  SmallVector<TemplateArgument> TemplateArgsForBuildingRC(
+  SmallVector<TemplateArgument> InnerArgsForBuildingRC(
       F->getTemplateParameters()->size());
   // Transform the transformed template args
   MultiLevelTemplateArgumentList Args;
@@ -778,33 +778,30 @@ buildAssociatedConstraints(Sema &SemaRef, FunctionTemplateDecl *F,
       NamedDecl *TP = F->getTemplateParameters()->getParam(Index);
       MultiLevelTemplateArgumentList Args;
       Args.setKind(TemplateSubstitutionKind::Rewrite);
-      Args.addOuterTemplateArguments(TemplateArgsForBuildingRC);
+      Args.addOuterTemplateArguments(InnerArgsForBuildingRC);
       // Rebuild the template parameter with updated depth and index.
       NamedDecl *NewParam =
           transformTemplateParameter(SemaRef, F->getDeclContext(), TP, Args,
                                      /*NewIndex=*/FirstUndeducedParamIdx,
                                      getDepthAndIndex(TP).first + AdjustDepth);
       FirstUndeducedParamIdx += 1;
-      assert(TemplateArgsForBuildingRC[Index].isNull());
-      TemplateArgsForBuildingRC[Index] =
-          Context.getInjectedTemplateArg(NewParam);
+      assert(InnerArgsForBuildingRC[Index].isNull());
+      InnerArgsForBuildingRC[Index] = Context.getInjectedTemplateArg(NewParam);
       continue;
     }
     TemplateArgumentLoc Input =
         SemaRef.getTrivialTemplateArgumentLoc(D, QualType(), SourceLocation{});
     TemplateArgumentLoc Output;
     if (!SemaRef.SubstTemplateArgument(Input, Args, Output)) {
-      assert(TemplateArgsForBuildingRC[Index].isNull() &&
+      assert(InnerArgsForBuildingRC[Index].isNull() &&
              "InstantiatedArgs must be null before setting");
-      TemplateArgsForBuildingRC[Index] = Output.getArgument();
+      InnerArgsForBuildingRC[Index] = Output.getArgument();
     }
   }
 
-  // A list of template arguments for transforming the require-clause of F.
-  // It must contain the entire set of template argument lists.
-  MultiLevelTemplateArgumentList ArgsForBuildingRC;
-  ArgsForBuildingRC.setKind(clang::TemplateSubstitutionKind::Rewrite);
-  ArgsForBuildingRC.addOuterTemplateArguments(TemplateArgsForBuildingRC);
+  // A list of template arguments for transforming the require-clause using
+  // the transformed template arguments as the template argument list of F.
+  //
   // For 2), if the underlying deduction guide F is nested in a class template,
   // we need the entire template argument list, as the constraint AST in the
   // require-clause of F remains completely uninstantiated.
@@ -827,25 +824,15 @@ buildAssociatedConstraints(Sema &SemaRef, FunctionTemplateDecl *F,
   //   - The occurrence of U in the function parameter is [depth:0, index:0]
   //   - The template parameter of U is [depth:0, index:0]
   //
-  // We add the outer template arguments which is [int] to the multi-level arg
-  // list to ensure that the occurrence U in `C<U>` will be replaced with int
-  // during the substitution.
-  //
   // NOTE: The underlying deduction guide F is instantiated -- either from an
   // explicitly-written deduction guide member, or from a constructor.
-  // getInstantiatedFromMemberTemplate() can only handle the former case, so we
-  // check the DeclContext kind.
-  if (F->getLexicalDeclContext()->getDeclKind() ==
-      clang::Decl::ClassTemplateSpecialization) {
-    auto OuterLevelArgs = SemaRef.getTemplateInstantiationArgs(
-        F, F->getLexicalDeclContext(),
-        /*Final=*/false, /*Innermost=*/std::nullopt,
-        /*RelativeToPrimary=*/true,
-        /*Pattern=*/nullptr,
-        /*ForConstraintInstantiation=*/true);
-    for (auto It : OuterLevelArgs)
-      ArgsForBuildingRC.addOuterTemplateArguments(It.Args);
-  }
+  MultiLevelTemplateArgumentList ArgsForBuildingRC =
+      SemaRef.getTemplateInstantiationArgs(F, F->getLexicalDeclContext(),
+                                           /*Final=*/false,
+                                           /*Innermost=*/InnerArgsForBuildingRC,
+                                           /*RelativeToPrimary=*/true,
+                                           /*ForConstraintInstantiation=*/true);
+  ArgsForBuildingRC.setKind(clang::TemplateSubstitutionKind::Rewrite);
 
   ExprResult E = SemaRef.SubstExpr(RC, ArgsForBuildingRC);
   if (E.isInvalid())
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 2f60c0beb22e73e68d0599f7a46728b4ef62fa3c..8c7f694c09042e6b1f515638d1fa1a3e2500b0e2 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -52,38 +52,6 @@ using namespace sema;
 //===----------------------------------------------------------------------===/
 
 namespace {
-namespace TemplateInstArgsHelpers {
-struct Response {
-  const Decl *NextDecl = nullptr;
-  bool IsDone = false;
-  bool ClearRelativeToPrimary = true;
-  static Response Done() {
-    Response R;
-    R.IsDone = true;
-    return R;
-  }
-  static Response ChangeDecl(const Decl *ND) {
-    Response R;
-    R.NextDecl = ND;
-    return R;
-  }
-  static Response ChangeDecl(const DeclContext *Ctx) {
-    Response R;
-    R.NextDecl = Decl::castFromDeclContext(Ctx);
-    return R;
-  }
-
-  static Response UseNextDecl(const Decl *CurDecl) {
-    return ChangeDecl(CurDecl->getDeclContext());
-  }
-
-  static Response DontClearRelativeToPrimaryNextDecl(const Decl *CurDecl) {
-    Response R = Response::UseNextDecl(CurDecl);
-    R.ClearRelativeToPrimary = false;
-    return R;
-  }
-};
-
 // Retrieve the primary template for a lambda call operator. It's
 // unfortunate that we only have the mappings of call operators rather
 // than lambda classes.
@@ -171,374 +139,396 @@ bool isLambdaEnclosedByTypeAliasDecl(
               .TraverseType(Underlying);
 }
 
-// Add template arguments from a variable template instantiation.
-Response
-HandleVarTemplateSpec(const VarTemplateSpecializationDecl *VarTemplSpec,
-                      MultiLevelTemplateArgumentList &Result,
-                      bool SkipForSpecialization) {
-  // For a class-scope explicit specialization, there are no template arguments
-  // at this level, but there may be enclosing template arguments.
-  if (VarTemplSpec->isClassScopeExplicitSpecialization())
-    return Response::DontClearRelativeToPrimaryNextDecl(VarTemplSpec);
-
-  // We're done when we hit an explicit specialization.
-  if (VarTemplSpec->getSpecializationKind() == TSK_ExplicitSpecialization &&
-      !isa<VarTemplatePartialSpecializationDecl>(VarTemplSpec))
-    return Response::Done();
-
-  // If this variable template specialization was instantiated from a
-  // specialized member that is a variable template, we're done.
-  assert(VarTemplSpec->getSpecializedTemplate() && "No variable template?");
-  llvm::PointerUnion<VarTemplateDecl *, VarTemplatePartialSpecializationDecl *>
-      Specialized = VarTemplSpec->getSpecializedTemplateOrPartial();
-  if (VarTemplatePartialSpecializationDecl *Partial =
-          Specialized.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
-    if (!SkipForSpecialization)
-      Result.addOuterTemplateArguments(
-          Partial, VarTemplSpec->getTemplateInstantiationArgs().asArray(),
-          /*Final=*/false);
-    if (Partial->isMemberSpecialization())
-      return Response::Done();
-  } else {
-    VarTemplateDecl *Tmpl = Specialized.get<VarTemplateDecl *>();
-    if (!SkipForSpecialization)
-      Result.addOuterTemplateArguments(
-          Tmpl, VarTemplSpec->getTemplateInstantiationArgs().asArray(),
-          /*Final=*/false);
-    if (Tmpl->isMemberSpecialization())
-      return Response::Done();
+struct TemplateInstantiationArgumentCollecter
+    : DeclVisitor<TemplateInstantiationArgumentCollecter, Decl *> {
+  Sema &S;
+  MultiLevelTemplateArgumentList &Result;
+  std::optional<ArrayRef<TemplateArgument>> Innermost;
+  bool RelativeToPrimary;
+  bool ForConstraintInstantiation;
+
+  TemplateInstantiationArgumentCollecter(
+      Sema &S, MultiLevelTemplateArgumentList &Result,
+      std::optional<ArrayRef<TemplateArgument>> Innermost,
+      bool RelativeToPrimary, bool ForConstraintInstantiation)
+      : S(S), Result(Result), Innermost(Innermost),
+        RelativeToPrimary(RelativeToPrimary),
+        ForConstraintInstantiation(ForConstraintInstantiation) {}
+
+  Decl *Done() { return nullptr; }
+
+  Decl *ChangeDecl(const Decl *D) {
+    RelativeToPrimary = false;
+    return const_cast<Decl *>(D);
   }
-  return Response::DontClearRelativeToPrimaryNextDecl(VarTemplSpec);
-}
 
-// If we have a template template parameter with translation unit context,
-// then we're performing substitution into a default template argument of
-// this template template parameter before we've constructed the template
-// that will own this template template parameter. In this case, we
-// use empty template parameter lists for all of the outer templates
-// to avoid performing any substitutions.
-Response
-HandleDefaultTempArgIntoTempTempParam(const TemplateTemplateParmDecl *TTP,
-                                      MultiLevelTemplateArgumentList &Result) {
-  for (unsigned I = 0, N = TTP->getDepth() + 1; I != N; ++I)
-    Result.addOuterTemplateArguments(std::nullopt);
-  return Response::Done();
-}
+  Decl *ChangeDecl(const DeclContext *DC) {
+    return ChangeDecl(Decl::castFromDeclContext(DC));
+  }
 
-Response HandlePartialClassTemplateSpec(
-    const ClassTemplatePartialSpecializationDecl *PartialClassTemplSpec,
-    MultiLevelTemplateArgumentList &Result, bool SkipForSpecialization) {
-  if (!SkipForSpecialization)
-      Result.addOuterRetainedLevels(PartialClassTemplSpec->getTemplateDepth());
-  return Response::Done();
-}
+  Decl *UseNextDecl(const Decl *D) { return ChangeDecl(D->getDeclContext()); }
 
-// Add template arguments from a class template instantiation.
-Response
-HandleClassTemplateSpec(const ClassTemplateSpecializationDecl *ClassTemplSpec,
-                        MultiLevelTemplateArgumentList &Result,
-                        bool SkipForSpecialization) {
-  if (!ClassTemplSpec->isClassScopeExplicitSpecialization()) {
-    // We're done when we hit an explicit specialization.
-    if (ClassTemplSpec->getSpecializationKind() == TSK_ExplicitSpecialization &&
-        !isa<ClassTemplatePartialSpecializationDecl>(ClassTemplSpec))
-      return Response::Done();
+  void AddInnermostTemplateArguments(const Decl *D) {
+    assert(Innermost);
+    Result.addOuterTemplateArguments(const_cast<Decl *>(D), *Innermost,
+                                     /*Final=*/false);
+    Innermost.reset();
+  }
 
-    if (!SkipForSpecialization)
-      Result.addOuterTemplateArguments(
-          const_cast<ClassTemplateSpecializationDecl *>(ClassTemplSpec),
-          ClassTemplSpec->getTemplateInstantiationArgs().asArray(),
-          /*Final=*/false);
+  void AddOuterTemplateArguments(const Decl *D, ArrayRef<TemplateArgument> Args,
+                                 bool Final) {
+    Result.addOuterTemplateArguments(const_cast<Decl *>(D), Args, Final);
+  }
 
-    // If this class template specialization was instantiated from a
-    // specialized member that is a class template, we're done.
-    assert(ClassTemplSpec->getSpecializedTemplate() && "No class template?");
-    if (ClassTemplSpec->getSpecializedTemplate()->isMemberSpecialization())
-      return Response::Done();
-
-    // If this was instantiated from a partial template specialization, we need
-    // to get the next level of declaration context from the partial
-    // specialization, as the ClassTemplateSpecializationDecl's
-    // DeclContext/LexicalDeclContext will be for the primary template.
-    if (auto *InstFromPartialTempl = ClassTemplSpec->getSpecializedTemplateOrPartial()
-                      .dyn_cast<ClassTemplatePartialSpecializationDecl *>())
-      return Response::ChangeDecl(InstFromPartialTempl->getLexicalDeclContext());
+  Decl *VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *TTPD) {
+    if (Innermost)
+      AddInnermostTemplateArguments(TTPD);
+    else if (ForConstraintInstantiation)
+      AddOuterTemplateArguments(nullptr, std::nullopt, /*Final=*/false);
+
+    for (unsigned Depth = TTPD->getDepth() + 1; Depth--;)
+      AddOuterTemplateArguments(nullptr, std::nullopt, /*Final=*/false);
+
+    return Done();
   }
-  return Response::UseNextDecl(ClassTemplSpec);
-}
 
-Response HandleFunction(Sema &SemaRef, const FunctionDecl *Function,
-                        MultiLevelTemplateArgumentList &Result,
-                        const FunctionDecl *Pattern, bool RelativeToPrimary,
-                        bool ForConstraintInstantiation,
-                        bool ForDefaultArgumentSubstitution) {
-  // Add template arguments from a function template specialization.
-  if (!RelativeToPrimary &&
-      Function->getTemplateSpecializationKindForInstantiation() ==
-          TSK_ExplicitSpecialization)
-    return Response::Done();
-
-  if (!RelativeToPrimary &&
-      Function->getTemplateSpecializationKind() == TSK_ExplicitSpecialization) {
-    // This is an implicit instantiation of an explicit specialization. We
-    // don't get any template arguments from this function but might get
-    // some from an enclosing template.
-    return Response::UseNextDecl(Function);
-  } else if (const TemplateArgumentList *TemplateArgs =
-                 Function->getTemplateSpecializationArgs()) {
-    // Add the template arguments for this specialization.
-    Result.addOuterTemplateArguments(const_cast<FunctionDecl *>(Function),
-                                     TemplateArgs->asArray(),
-                                     /*Final=*/false);
+  Decl *VisitFunctionTemplateDecl(FunctionTemplateDecl *FTD) {
+    assert(
+        (ForConstraintInstantiation || Result.getNumSubstitutedLevels() == 0) &&
+        "outer template not instantiated?");
+
+    if (Innermost)
+      AddInnermostTemplateArguments(FTD);
+    else if (ForConstraintInstantiation)
+      AddOuterTemplateArguments(FTD, FTD->getInjectedTemplateArgs(),
+                                /*Final=*/false);
+
+    if (FTD->isMemberSpecialization())
+      return Done();
+
+    if (FTD->getFriendObjectKind())
+      return ChangeDecl(FTD->getLexicalDeclContext());
+    return UseNextDecl(FTD);
+  }
 
-    if (RelativeToPrimary &&
-        (Function->getTemplateSpecializationKind() ==
-             TSK_ExplicitSpecialization ||
-         (Function->getFriendObjectKind() &&
-          !Function->getPrimaryTemplate()->getFriendObjectKind())))
-      return Response::UseNextDecl(Function);
-
-    // If this function was instantiated from a specialized member that is
-    // a function template, we're done.
-    assert(Function->getPrimaryTemplate() && "No function template?");
-    if (!ForDefaultArgumentSubstitution &&
-        Function->getPrimaryTemplate()->isMemberSpecialization())
-      return Response::Done();
-
-    // If this function is a generic lambda specialization, we are done.
-    if (!ForConstraintInstantiation &&
-        isGenericLambdaCallOperatorOrStaticInvokerSpecialization(Function))
-      return Response::Done();
-
-  } else if (Function->getDescribedFunctionTemplate()) {
+  Decl *VisitVarTemplateDecl(VarTemplateDecl *VTD) {
     assert(
         (ForConstraintInstantiation || Result.getNumSubstitutedLevels() == 0) &&
-        "Outer template not instantiated?");
+        "outer template not instantiated?");
+
+    if (Innermost)
+      AddInnermostTemplateArguments(VTD);
+    else if (ForConstraintInstantiation)
+      AddOuterTemplateArguments(VTD, VTD->getInjectedTemplateArgs(),
+                                /*Final=*/false);
+
+    if (VTD->isMemberSpecialization())
+      return Done();
+
+    return UseNextDecl(VTD);
   }
-  // If this is a friend or local declaration and it declares an entity at
-  // namespace scope, take arguments from its lexical parent
-  // instead of its semantic parent, unless of course the pattern we're
-  // instantiating actually comes from the file's context!
-  if ((Function->getFriendObjectKind() || Function->isLocalExternDecl()) &&
-      Function->getNonTransparentDeclContext()->isFileContext() &&
-      (!Pattern || !Pattern->getLexicalDeclContext()->isFileContext())) {
-    return Response::ChangeDecl(Function->getLexicalDeclContext());
+
+  Decl *VisitVarTemplatePartialSpecializationDecl(
+      VarTemplatePartialSpecializationDecl *VTPSD) {
+    assert(
+        (ForConstraintInstantiation || Result.getNumSubstitutedLevels() == 0) &&
+        "outer template not instantiated?");
+
+    if (Innermost)
+      AddInnermostTemplateArguments(VTPSD);
+    else if (ForConstraintInstantiation)
+      AddOuterTemplateArguments(VTPSD, VTPSD->getTemplateArgs().asArray(),
+                                /*Final=*/false);
+
+    if (VTPSD->isMemberSpecialization())
+      return Done();
+
+    return UseNextDecl(VTPSD);
   }
 
-  if (ForConstraintInstantiation && Function->getFriendObjectKind())
-    return Response::ChangeDecl(Function->getLexicalDeclContext());
-  return Response::UseNextDecl(Function);
-}
+  Decl *VisitClassTemplateDecl(ClassTemplateDecl *CTD) {
+    assert(
+        (ForConstraintInstantiation || Result.getNumSubstitutedLevels() == 0) &&
+        "outer template not instantiated?");
 
-Response HandleFunctionTemplateDecl(const FunctionTemplateDecl *FTD,
-                                    MultiLevelTemplateArgumentList &Result) {
-  if (!isa<ClassTemplateSpecializationDecl>(FTD->getDeclContext())) {
-    Result.addOuterTemplateArguments(
-        const_cast<FunctionTemplateDecl *>(FTD),
-        const_cast<FunctionTemplateDecl *>(FTD)->getInjectedTemplateArgs(),
-        /*Final=*/false);
-
-    NestedNameSpecifier *NNS = FTD->getTemplatedDecl()->getQualifier();
-
-    while (const Type *Ty = NNS ? NNS->getAsType() : nullptr) {
-      if (NNS->isInstantiationDependent()) {
-        if (const auto *TSTy = Ty->getAs<TemplateSpecializationType>()) {
-          ArrayRef<TemplateArgument> Arguments = TSTy->template_arguments();
-          // Prefer template arguments from the injected-class-type if possible.
-          // For example,
-          // ```cpp
-          // template <class... Pack> struct S {
-          //   template <class T> void foo();
-          // };
-          // template <class... Pack> template <class T>
-          //           ^^^^^^^^^^^^^ InjectedTemplateArgs
-          //           They're of kind TemplateArgument::Pack, not of
-          //           TemplateArgument::Type.
-          // void S<Pack...>::foo() {}
-          //        ^^^^^^^
-          //        TSTy->template_arguments() (which are of PackExpansionType)
-          // ```
-          // This meets the contract in
-          // TreeTransform::TryExpandParameterPacks that the template arguments
-          // for unexpanded parameters should be of a Pack kind.
-          if (TSTy->isCurrentInstantiation()) {
-            auto *RD = TSTy->getCanonicalTypeInternal()->getAsCXXRecordDecl();
-            if (ClassTemplateDecl *CTD = RD->getDescribedClassTemplate())
-              Arguments = CTD->getInjectedTemplateArgs();
-            else if (auto *Specialization =
-                         dyn_cast<ClassTemplateSpecializationDecl>(RD))
-              Arguments =
-                  Specialization->getTemplateInstantiationArgs().asArray();
-          }
-          Result.addOuterTemplateArguments(
-              TSTy->getTemplateName().getAsTemplateDecl(), Arguments,
-              /*Final=*/false);
-        }
-      }
+    if (Innermost)
+      AddInnermostTemplateArguments(CTD);
+    else if (ForConstraintInstantiation)
+      AddOuterTemplateArguments(CTD, CTD->getInjectedTemplateArgs(),
+                                /*Final=*/false);
 
-      NNS = NNS->getPrefix();
-    }
+    if (CTD->isMemberSpecialization())
+      return Done();
+
+    if (CTD->getFriendObjectKind())
+      return ChangeDecl(CTD->getLexicalDeclContext());
+    return UseNextDecl(CTD);
   }
 
-  return Response::ChangeDecl(FTD->getLexicalDeclContext());
-}
+  Decl *VisitClassTemplatePartialSpecializationDecl(
+      ClassTemplatePartialSpecializationDecl *CTPSD) {
+    assert(
+        (ForConstraintInstantiation || Result.getNumSubstitutedLevels() == 0) &&
+        "outer template not instantiated?");
+
+    if (Innermost)
+      AddInnermostTemplateArguments(CTPSD);
+    else if (ForConstraintInstantiation)
+      AddOuterTemplateArguments(CTPSD, CTPSD->getTemplateArgs().asArray(),
+                                /*Final=*/false);
+
+    if (CTPSD->isMemberSpecialization())
+      return Done();
+
+    return UseNextDecl(CTPSD);
+  }
+
+  Decl *VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *TATD) {
+    assert(
+        (ForConstraintInstantiation || Result.getNumSubstitutedLevels() == 0) &&
+        "outer template not instantiated?");
+    if (Innermost)
+      AddInnermostTemplateArguments(TATD);
+    else if (ForConstraintInstantiation)
+      AddOuterTemplateArguments(TATD, TATD->getInjectedTemplateArgs(),
+                                /*Final=*/false);
+
+    return UseNextDecl(TATD);
+  }
 
-Response HandleRecordDecl(Sema &SemaRef, const CXXRecordDecl *Rec,
-                          MultiLevelTemplateArgumentList &Result,
-                          ASTContext &Context,
-                          bool ForConstraintInstantiation) {
-  if (ClassTemplateDecl *ClassTemplate = Rec->getDescribedClassTemplate()) {
+  Decl *VisitConceptDecl(ConceptDecl *CD) {
     assert(
         (ForConstraintInstantiation || Result.getNumSubstitutedLevels() == 0) &&
-        "Outer template not instantiated?");
-    if (ClassTemplate->isMemberSpecialization())
-      return Response::Done();
-    if (ForConstraintInstantiation)
-      Result.addOuterTemplateArguments(const_cast<CXXRecordDecl *>(Rec),
-                                       ClassTemplate->getInjectedTemplateArgs(),
-                                       /*Final=*/false);
+        "outer template not instantiated?");
+    if (Innermost)
+      AddInnermostTemplateArguments(CD);
+
+    return UseNextDecl(CD);
   }
 
-  if (const MemberSpecializationInfo *MSInfo =
-          Rec->getMemberSpecializationInfo())
-    if (MSInfo->getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
-      return Response::Done();
-
-  bool IsFriend = Rec->getFriendObjectKind() ||
-                  (Rec->getDescribedClassTemplate() &&
-                   Rec->getDescribedClassTemplate()->getFriendObjectKind());
-  if (ForConstraintInstantiation && IsFriend &&
-      Rec->getNonTransparentDeclContext()->isFileContext()) {
-    return Response::ChangeDecl(Rec->getLexicalDeclContext());
+  Decl *VisitFunctionDecl(FunctionDecl *FD) {
+    assert(!FD->getDescribedFunctionTemplate() &&
+           "not for templated declarations");
+
+    if (!RelativeToPrimary) {
+      // Add template arguments from a function template specialization.
+      if (const MemberSpecializationInfo *MSI =
+              FD->getMemberSpecializationInfo();
+          MSI &&
+          MSI->getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
+        return Done();
+
+      // This is an implicit instantiation of an explicit specialization. We
+      // don't get any template arguments from this function but might get
+      // some from an enclosing template.
+      if (FD->getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
+        return UseNextDecl(FD);
+    }
+
+    if (const TemplateArgumentList *TemplateArgs =
+            FD->getTemplateSpecializationArgs()) {
+      // Add the template arguments for this specialization.
+      if (Innermost)
+        AddInnermostTemplateArguments(FD);
+      else
+        AddOuterTemplateArguments(FD, TemplateArgs->asArray(), /*Final=*/false);
+
+      if (FD->getTemplateSpecializationKind() == TSK_ExplicitSpecialization ||
+          (FD->getFriendObjectKind() &&
+           !FD->getPrimaryTemplate()->getFriendObjectKind()))
+        return UseNextDecl(FD);
+
+      // If this function was instantiated from a specialized member that is
+      // a function template, we're done.
+      assert(FD->getPrimaryTemplate() && "No function template?");
+      if (FD->getPrimaryTemplate()->hasMemberSpecialization())
+        return Done();
+
+      // If this function is a generic lambda specialization, we are done.
+      if (!ForConstraintInstantiation &&
+          isGenericLambdaCallOperatorOrStaticInvokerSpecialization(FD))
+        return Done();
+    }
+
+    // If this is a friend or local declaration and it declares an entity at
+    // namespace scope, take arguments from its lexical parent
+    // instead of its semantic parent, unless of course the pattern we're
+    // instantiating actually comes from the file's context!
+    if ((FD->getFriendObjectKind() || FD->isLocalExternDecl()) &&
+        FD->getNonTransparentDeclContext()->isFileContext()) {
+      return ChangeDecl(FD->getLexicalDeclContext());
+    }
+
+    if (ForConstraintInstantiation && FD->getFriendObjectKind())
+      return ChangeDecl(FD->getLexicalDeclContext());
+    return UseNextDecl(FD);
   }
 
-  // This is to make sure we pick up the VarTemplateSpecializationDecl or the
-  // TypeAliasTemplateDecl that this lambda is defined inside of.
-  if (Rec->isLambda()) {
-    if (const Decl *LCD = Rec->getLambdaContextDecl())
-      return Response::ChangeDecl(LCD);
-    // Retrieve the template arguments for a using alias declaration.
-    // This is necessary for constraint checking, since we always keep
-    // constraints relative to the primary template.
-    if (auto TypeAlias = getEnclosingTypeAliasTemplateDecl(SemaRef);
-        ForConstraintInstantiation && TypeAlias) {
-      if (isLambdaEnclosedByTypeAliasDecl(Rec->getLambdaCallOperator(),
-                                          TypeAlias.PrimaryTypeAliasDecl)) {
-        Result.addOuterTemplateArguments(TypeAlias.Template,
-                                         TypeAlias.AssociatedTemplateArguments,
-                                         /*Final=*/false);
-        // Visit the parent of the current type alias declaration rather than
-        // the lambda thereof.
-        // E.g., in the following example:
-        // struct S {
-        //  template <class> using T = decltype([]<Concept> {} ());
-        // };
-        // void foo() {
-        //   S::T var;
-        // }
-        // The instantiated lambda expression (which we're visiting at 'var')
-        // has a function DeclContext 'foo' rather than the Record DeclContext
-        // S. This seems to be an oversight to me that we may want to set a
-        // Sema Context from the CXXScopeSpec before substituting into T.
-        return Response::ChangeDecl(TypeAlias.Template->getDeclContext());
+  Decl *VisitCXXRecordDecl(CXXRecordDecl *RD) {
+    assert(!RD->getDescribedClassTemplate() &&
+           "not for templated declarations");
+
+    if (const MemberSpecializationInfo *MSI = RD->getMemberSpecializationInfo();
+        MSI &&
+        MSI->getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
+      return Done();
+
+    if (ForConstraintInstantiation && RD->getFriendObjectKind() &&
+        RD->getNonTransparentDeclContext()->isFileContext()) {
+      return ChangeDecl(RD->getLexicalDeclContext());
+    }
+
+    // This is to make sure we pick up the VarTemplateSpecializationDecl or the
+    // TypeAliasTemplateDecl that this lambda is defined inside of.
+    if (RD->isLambda()) {
+      if (Decl *LCD = RD->getLambdaContextDecl())
+        return ChangeDecl(LCD);
+      // Retrieve the template arguments for a using alias declaration.
+      // This is necessary for constraint checking, since we always keep
+      // constraints relative to the primary template.
+      if (auto TypeAlias = getEnclosingTypeAliasTemplateDecl(S);
+          ForConstraintInstantiation && TypeAlias) {
+        if (isLambdaEnclosedByTypeAliasDecl(RD->getLambdaCallOperator(),
+                                            TypeAlias.PrimaryTypeAliasDecl)) {
+          AddOuterTemplateArguments(TypeAlias.Template,
+                                    TypeAlias.AssociatedTemplateArguments,
+                                    /*Final=*/false);
+          // Visit the parent of the current type alias declaration rather than
+          // the lambda thereof.
+          // E.g., in the following example:
+          // struct S {
+          //  template <class> using T = decltype([]<Concept> {} ());
+          // };
+          // void foo() {
+          //   S::T var;
+          // }
+          // The instantiated lambda expression (which we're visiting at 'var')
+          // has a function DeclContext 'foo' rather than the Record DeclContext
+          // S. This seems to be an oversight to me that we may want to set a
+          // Sema Context from the CXXScopeSpec before substituting into T.
+          return ChangeDecl(TypeAlias.Template->getDeclContext());
+        }
       }
     }
+
+    return UseNextDecl(RD);
   }
 
-  return Response::UseNextDecl(Rec);
-}
+  Decl *
+  VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *CTSD) {
+    // For a class-scope explicit specialization, there are no template
+    // arguments at this level, but there may be enclosing template arguments.
+    if (CTSD->isClassScopeExplicitSpecialization())
+      return UseNextDecl(CTSD);
 
-Response HandleImplicitConceptSpecializationDecl(
-    const ImplicitConceptSpecializationDecl *CSD,
-    MultiLevelTemplateArgumentList &Result) {
-  Result.addOuterTemplateArguments(
-      const_cast<ImplicitConceptSpecializationDecl *>(CSD),
-      CSD->getTemplateArguments(),
-      /*Final=*/false);
-  return Response::UseNextDecl(CSD);
-}
+    // We're done when we hit an explicit specialization.
+    if (CTSD->getSpecializationKind() == TSK_ExplicitSpecialization)
+      return Done();
+
+    if (Innermost)
+      AddInnermostTemplateArguments(CTSD);
+    else
+      AddOuterTemplateArguments(CTSD,
+                                CTSD->getTemplateInstantiationArgs().asArray(),
+                                /*Final=*/false);
+
+    // If this class template specialization was instantiated from a
+    // specialized member that is a class template, we're done.
+    assert(CTSD->getSpecializedTemplate() && "No class template?");
+    llvm::PointerUnion<ClassTemplateDecl *,
+                       ClassTemplatePartialSpecializationDecl *>
+        Specialized = CTSD->getSpecializedTemplateOrPartial();
+    if (auto *CTPSD =
+            Specialized.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
+      if (CTPSD->hasMemberSpecialization())
+        return Done();
+    } else {
+      auto *CTD = Specialized.get<ClassTemplateDecl *>();
+      if (CTD->hasMemberSpecialization())
+        return Done();
+    }
+    return UseNextDecl(CTSD);
+  }
+
+  Decl *
+  VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *VTSD) {
+    // For a class-scope explicit specialization, there are no template
+    // arguments at this level, but there may be enclosing template arguments.
+    if (VTSD->isClassScopeExplicitSpecialization())
+      return UseNextDecl(VTSD);
+
+    // We're done when we hit an explicit specialization.
+    if (VTSD->getSpecializationKind() == TSK_ExplicitSpecialization)
+      return Done();
+
+    if (Innermost)
+      AddInnermostTemplateArguments(VTSD);
+    else
+      AddOuterTemplateArguments(VTSD,
+                                VTSD->getTemplateInstantiationArgs().asArray(),
+                                /*Final=*/false);
+
+    // If this variable template specialization was instantiated from a
+    // specialized member that is a variable template, we're done.
+    assert(VTSD->getSpecializedTemplate() && "No variable template?");
+    llvm::PointerUnion<VarTemplateDecl *,
+                       VarTemplatePartialSpecializationDecl *>
+        Specialized = VTSD->getSpecializedTemplateOrPartial();
+    if (auto *VTPSD =
+            Specialized.dyn_cast<VarTemplatePartialSpecializationDecl *>()) {
+      if (VTPSD->hasMemberSpecialization())
+        return Done();
+    } else {
+      auto *VTD = Specialized.get<VarTemplateDecl *>();
+      if (VTD->hasMemberSpecialization())
+        return Done();
+    }
+    return UseNextDecl(VTSD);
+  }
+
+  Decl *VisitImplicitConceptSpecializationDecl(
+      ImplicitConceptSpecializationDecl *ICSD) {
+    AddOuterTemplateArguments(ICSD, ICSD->getTemplateArguments(),
+                              /*Final=*/false);
+    return UseNextDecl(ICSD);
+  }
+
+  Decl *VisitDecl(Decl *D) {
+    if (D->isFileContextDecl())
+      return Done();
+
+    if (isa<DeclContext>(D))
+      RelativeToPrimary = false;
+
+    return UseNextDecl(D);
+  }
+
+  Decl *Visit(Decl *D) {
+    if (TemplateDecl *TD = D->getDescribedTemplate())
+      D = TD;
+    return DeclVisitor::Visit(D);
+  }
+};
 
-Response HandleGenericDeclContext(const Decl *CurDecl) {
-  return Response::UseNextDecl(CurDecl);
-}
-} // namespace TemplateInstArgsHelpers
 } // namespace
 
 MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
     const NamedDecl *ND, const DeclContext *DC, bool Final,
     std::optional<ArrayRef<TemplateArgument>> Innermost, bool RelativeToPrimary,
-    const FunctionDecl *Pattern, bool ForConstraintInstantiation,
-    bool SkipForSpecialization, bool ForDefaultArgumentSubstitution) {
+    bool ForConstraintInstantiation) {
   assert((ND || DC) && "Can't find arguments for a decl if one isn't provided");
   // Accumulate the set of template argument lists in this structure.
   MultiLevelTemplateArgumentList Result;
-
-  using namespace TemplateInstArgsHelpers;
   const Decl *CurDecl = ND;
 
   if (!CurDecl)
     CurDecl = Decl::castFromDeclContext(DC);
 
-  if (Innermost) {
-    Result.addOuterTemplateArguments(const_cast<NamedDecl *>(ND), *Innermost,
-                                     Final);
-    // Populate placeholder template arguments for TemplateTemplateParmDecls.
-    // This is essential for the case e.g.
-    //
-    // template <class> concept Concept = false;
-    // template <template <Concept C> class T> void foo(T<int>)
-    //
-    // where parameter C has a depth of 1 but the substituting argument `int`
-    // has a depth of 0.
-    if (const auto *TTP = dyn_cast<TemplateTemplateParmDecl>(CurDecl))
-      HandleDefaultTempArgIntoTempTempParam(TTP, Result);
-    CurDecl = Response::UseNextDecl(CurDecl).NextDecl;
-  }
-
-  while (!CurDecl->isFileContextDecl()) {
-    Response R;
-    if (const auto *VarTemplSpec =
-            dyn_cast<VarTemplateSpecializationDecl>(CurDecl)) {
-      R = HandleVarTemplateSpec(VarTemplSpec, Result, SkipForSpecialization);
-    } else if (const auto *PartialClassTemplSpec =
-                   dyn_cast<ClassTemplatePartialSpecializationDecl>(CurDecl)) {
-      R = HandlePartialClassTemplateSpec(PartialClassTemplSpec, Result,
-                                         SkipForSpecialization);
-    } else if (const auto *ClassTemplSpec =
-                   dyn_cast<ClassTemplateSpecializationDecl>(CurDecl)) {
-      R = HandleClassTemplateSpec(ClassTemplSpec, Result,
-                                  SkipForSpecialization);
-    } else if (const auto *Function = dyn_cast<FunctionDecl>(CurDecl)) {
-      R = HandleFunction(*this, Function, Result, Pattern, RelativeToPrimary,
-                         ForConstraintInstantiation,
-                         ForDefaultArgumentSubstitution);
-    } else if (const auto *Rec = dyn_cast<CXXRecordDecl>(CurDecl)) {
-      R = HandleRecordDecl(*this, Rec, Result, Context,
-                           ForConstraintInstantiation);
-    } else if (const auto *CSD =
-                   dyn_cast<ImplicitConceptSpecializationDecl>(CurDecl)) {
-      R = HandleImplicitConceptSpecializationDecl(CSD, Result);
-    } else if (const auto *FTD = dyn_cast<FunctionTemplateDecl>(CurDecl)) {
-      R = HandleFunctionTemplateDecl(FTD, Result);
-    } else if (const auto *CTD = dyn_cast<ClassTemplateDecl>(CurDecl)) {
-      R = Response::ChangeDecl(CTD->getLexicalDeclContext());
-    } else if (!isa<DeclContext>(CurDecl)) {
-      R = Response::DontClearRelativeToPrimaryNextDecl(CurDecl);
-      if (const auto *TTP = dyn_cast<TemplateTemplateParmDecl>(CurDecl)) {
-        R = HandleDefaultTempArgIntoTempTempParam(TTP, Result);
-      }
-    } else {
-      R = HandleGenericDeclContext(CurDecl);
-    }
-
-    if (R.IsDone)
-      return Result;
-    if (R.ClearRelativeToPrimary)
-      RelativeToPrimary = false;
-    assert(R.NextDecl);
-    CurDecl = R.NextDecl;
-  }
-
+  TemplateInstantiationArgumentCollecter Collecter(
+      *this, Result, Innermost, RelativeToPrimary, ForConstraintInstantiation);
+  do {
+    CurDecl = Collecter.Visit(const_cast<Decl *>(CurDecl));
+  } while (CurDecl);
   return Result;
 }
 
@@ -572,7 +562,6 @@ bool Sema::CodeSynthesisContext::isInstantiationRecord() const {
   case LambdaExpressionSubstitution:
   case BuildingDeductionGuides:
   case TypeAliasTemplateInstantiation:
-  case PartialOrderingTTP:
     return false;
 
   // This function should never be called when Kind's value is Memoization.
@@ -805,11 +794,6 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
           SemaRef, CodeSynthesisContext::BuildingDeductionGuides,
           PointOfInstantiation, InstantiationRange, Entity) {}
 
-Sema::InstantiatingTemplate::InstantiatingTemplate(
-    Sema &SemaRef, SourceLocation ArgLoc, PartialOrderingTTP,
-    TemplateDecl *PArg, SourceRange InstantiationRange)
-    : InstantiatingTemplate(SemaRef, CodeSynthesisContext::PartialOrderingTTP,
-                            ArgLoc, InstantiationRange, PArg) {}
 
 void Sema::pushCodeSynthesisContext(CodeSynthesisContext Ctx) {
   Ctx.SavedInNonInstantiationSFINAEContext = InNonInstantiationSFINAEContext;
@@ -1249,14 +1233,6 @@ void Sema::PrintInstantiationStack() {
           << cast<TypeAliasTemplateDecl>(Active->Entity)
           << Active->InstantiationRange;
       break;
-    case CodeSynthesisContext::PartialOrderingTTP:
-      Diags.Report(Active->PointOfInstantiation,
-                   diag::note_template_arg_template_params_mismatch);
-      if (SourceLocation ParamLoc = Active->Entity->getLocation();
-          ParamLoc.isValid())
-        Diags.Report(ParamLoc, diag::note_template_prev_declaration)
-            << /*isTemplateTemplateParam=*/true << Active->InstantiationRange;
-      break;
     }
   }
 }
@@ -1299,7 +1275,6 @@ std::optional<TemplateDeductionInfo *> Sema::isSFINAEContext() const {
     case CodeSynthesisContext::PriorTemplateArgumentSubstitution:
     case CodeSynthesisContext::DefaultTemplateArgumentChecking:
     case CodeSynthesisContext::RewritingOperatorAsSpaceship:
-    case CodeSynthesisContext::PartialOrderingTTP:
       // A default template argument instantiation and substitution into
       // template parameters with arguments for prior parameters may or may
       // not be a SFINAE context; look further up the stack.
@@ -1688,10 +1663,8 @@ namespace {
 
     CXXRecordDecl::LambdaDependencyKind
     ComputeLambdaDependency(LambdaScopeInfo *LSI) {
-      if (auto TypeAlias =
-              TemplateInstArgsHelpers::getEnclosingTypeAliasTemplateDecl(
-                  getSema());
-          TypeAlias && TemplateInstArgsHelpers::isLambdaEnclosedByTypeAliasDecl(
+      if (auto TypeAlias = getEnclosingTypeAliasTemplateDecl(getSema());
+          TypeAlias && isLambdaEnclosedByTypeAliasDecl(
                            LSI->CallOperator, TypeAlias.PrimaryTypeAliasDecl)) {
         unsigned TypeAliasDeclDepth = TypeAlias.Template->getTemplateDepth();
         if (TypeAliasDeclDepth >= TemplateArgs.getNumSubstitutedLevels())
@@ -1709,12 +1682,18 @@ namespace {
       if (SemaRef.RebuildingImmediateInvocation)
         return E;
       LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true,
-                                    /*InstantiatingLambda=*/true);
+                                    /*InstantiatingLambdaOrBlock=*/true);
       Sema::ConstraintEvalRAII<TemplateInstantiator> RAII(*this);
 
       return inherited::TransformLambdaExpr(E);
     }
 
+    ExprResult TransformBlockExpr(BlockExpr *E) {
+      LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true,
+                                    /*InstantiatingLambdaOrBlock=*/true);
+      return inherited::TransformBlockExpr(E);
+    }
+
     ExprResult RebuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
                                  LambdaScopeInfo *LSI) {
       CXXMethodDecl *MD = LSI->CallOperator;
@@ -1730,8 +1709,8 @@ namespace {
           // RecoveryExpr that wraps the uninstantiated default argument so
           // that downstream diagnostics are omitted.
           ExprResult ErrorResult = SemaRef.CreateRecoveryExpr(
-              UninstExpr->getBeginLoc(), UninstExpr->getEndLoc(),
-              { UninstExpr }, UninstExpr->getType());
+              UninstExpr->getBeginLoc(), UninstExpr->getEndLoc(), {UninstExpr},
+              UninstExpr->getType());
           if (ErrorResult.isUsable())
             PVD->setDefaultArg(ErrorResult.get());
         }
@@ -2503,7 +2482,7 @@ QualType TemplateInstantiator::TransformFunctionProtoType(TypeLocBuilder &TLB,
                                  CXXRecordDecl *ThisContext,
                                  Qualifiers ThisTypeQuals,
                                  Fn TransformExceptionSpec) {
-  // If this is a lambda, the transformation MUST be done in the
+  // If this is a lambda or block, the transformation MUST be done in the
   // CurrentInstantiationScope since it introduces a mapping of
   // the original to the newly created transformed parameters.
   //
@@ -2512,7 +2491,7 @@ QualType TemplateInstantiator::TransformFunctionProtoType(TypeLocBuilder &TLB,
   // a second one.
   LocalInstantiationScope *Current = getSema().CurrentInstantiationScope;
   std::optional<LocalInstantiationScope> Scope;
-  if (!Current || !Current->isLambda())
+  if (!Current || !Current->isLambdaOrBlock())
     Scope.emplace(SemaRef, /*CombineWithOuterScope=*/true);
 
   return inherited::TransformFunctionProtoType(
@@ -4017,11 +3996,11 @@ bool Sema::usesPartialOrExplicitSpecialization(
 /// Get the instantiation pattern to use to instantiate the definition of a
 /// given ClassTemplateSpecializationDecl (either the pattern of the primary
 /// template or of a partial specialization).
-static ActionResult<CXXRecordDecl *> getPatternForClassTemplateSpecialization(
+static ActionResult<CXXRecordDecl *>
+getPatternForClassTemplateSpecialization(
     Sema &S, SourceLocation PointOfInstantiation,
     ClassTemplateSpecializationDecl *ClassTemplateSpec,
-    TemplateSpecializationKind TSK,
-    bool PrimaryHasMatchedPackOnParmToNonPackOnArg) {
+    TemplateSpecializationKind TSK) {
   Sema::InstantiatingTemplate Inst(S, PointOfInstantiation, ClassTemplateSpec);
   if (Inst.isInvalid())
     return {/*Invalid=*/true};
@@ -4044,7 +4023,7 @@ static ActionResult<CXXRecordDecl *> getPatternForClassTemplateSpecialization(
     //   specialization with the template argument lists of the partial
     //   specializations.
     typedef PartialSpecMatchResult MatchResult;
-    SmallVector<MatchResult, 4> Matched, ExtraMatched;
+    SmallVector<MatchResult, 4> Matched;
     SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
     Template->getPartialSpecializations(PartialSpecs);
     TemplateSpecCandidateSet FailedCandidates(PointOfInstantiation);
@@ -4061,13 +4040,11 @@ static ActionResult<CXXRecordDecl *> getPatternForClassTemplateSpecialization(
             MakeDeductionFailureInfo(S.Context, Result, Info));
         (void)Result;
       } else {
-        auto &List =
-            Info.hasMatchedPackOnParmToNonPackOnArg() ? ExtraMatched : Matched;
-        List.push_back(MatchResult{Partial, Info.takeCanonical()});
+        Matched.push_back(PartialSpecMatchResult());
+        Matched.back().Partial = Partial;
+        Matched.back().Args = Info.takeCanonical();
       }
     }
-    if (Matched.empty() && PrimaryHasMatchedPackOnParmToNonPackOnArg)
-      Matched = std::move(ExtraMatched);
 
     // If we're dealing with a member template where the template parameters
     // have been instantiated, this provides the original template parameters
@@ -4139,39 +4116,32 @@ static ActionResult<CXXRecordDecl *> getPatternForClassTemplateSpecialization(
 
   CXXRecordDecl *Pattern = nullptr;
   Specialized = ClassTemplateSpec->getSpecializedTemplateOrPartial();
-  if (auto *PartialSpec =
-          Specialized.dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
-    // Instantiate using the best class template partial specialization.
-    while (PartialSpec->getInstantiatedFromMember()) {
-      // If we've found an explicit specialization of this class template,
-      // stop here and use that as the pattern.
-      if (PartialSpec->isMemberSpecialization())
+  if (auto *CTD = Specialized.dyn_cast<ClassTemplateDecl *>()) {
+    while (!CTD->hasMemberSpecialization()) {
+      if (auto *NewCTD = CTD->getInstantiatedFromMemberTemplate())
+        CTD = NewCTD;
+      else
         break;
-
-      PartialSpec = PartialSpec->getInstantiatedFromMember();
     }
-    Pattern = PartialSpec;
-  } else {
-    ClassTemplateDecl *Template = ClassTemplateSpec->getSpecializedTemplate();
-    while (Template->getInstantiatedFromMemberTemplate()) {
-      // If we've found an explicit specialization of this class template,
-      // stop here and use that as the pattern.
-      if (Template->isMemberSpecialization())
+    Pattern = CTD->getTemplatedDecl();
+  } else if (auto *CTPSD =
+                 Specialized
+                     .dyn_cast<ClassTemplatePartialSpecializationDecl *>()) {
+    while (!CTPSD->hasMemberSpecialization()) {
+      if (auto *NewCTPSD = CTPSD->getInstantiatedFromMemberTemplate())
+        CTPSD = NewCTPSD;
+      else
         break;
-
-      Template = Template->getInstantiatedFromMemberTemplate();
     }
-    Pattern = Template->getTemplatedDecl();
+    Pattern = CTPSD;
   }
-
   return Pattern;
 }
 
 bool Sema::InstantiateClassTemplateSpecialization(
     SourceLocation PointOfInstantiation,
     ClassTemplateSpecializationDecl *ClassTemplateSpec,
-    TemplateSpecializationKind TSK, bool Complain,
-    bool PrimaryHasMatchedPackOnParmToNonPackOnArg) {
+    TemplateSpecializationKind TSK, bool Complain) {
   // Perform the actual instantiation on the canonical declaration.
   ClassTemplateSpec = cast<ClassTemplateSpecializationDecl>(
       ClassTemplateSpec->getCanonicalDecl());
@@ -4179,9 +4149,8 @@ bool Sema::InstantiateClassTemplateSpecialization(
     return true;
 
   ActionResult<CXXRecordDecl *> Pattern =
-      getPatternForClassTemplateSpecialization(
-          *this, PointOfInstantiation, ClassTemplateSpec, TSK,
-          PrimaryHasMatchedPackOnParmToNonPackOnArg);
+      getPatternForClassTemplateSpecialization(*this, PointOfInstantiation,
+                                               ClassTemplateSpec, TSK);
   if (!Pattern.isUsable())
     return Pattern.isInvalid();
 
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 6b1af35f5c80a8b1df2325571c8dfbeedf3aae7b..3e948232057afe68948e43c528a3155b2a8795c4 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -12,6 +12,7 @@
 #include "TreeTransform.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/ASTLambda.h"
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclVisitor.h"
@@ -4688,6 +4689,36 @@ bool Sema::InstantiateDefaultArgument(SourceLocation CallLoc, FunctionDecl *FD,
                                       ParmVarDecl *Param) {
   assert(Param->hasUninstantiatedDefaultArg());
 
+  NamedDecl *Pattern = FD;
+  std::optional<ArrayRef<TemplateArgument>> Innermost;
+
+  // C++ [dcl.fct.default]p4
+  //   For non-template functions, default arguments can be added in later
+  //   declarations of a function that inhabit the same scope.
+  //
+  // C++ [dcl.fct.default]p6
+  //   Except for member functions of templated classes, the default arguments
+  //   in a member function definition that appears outside of the class
+  //   definition are added to the set of default arguments provided by the
+  //   member function declaration in the class definition; the program is
+  //   ill-formed if a default constructor, copy or move constructor, or copy
+  //   or move assignment operator is so declared. Default arguments for a
+  //   member function of a templated class shall be specified on the initial
+  //   declaration of the member function within the templated class.
+  //
+  // We need to collect the template arguments from the context of the function
+  // where the default argument was defined. For a specialization of a function
+  // template explicitly specialized for an implicit instantiation of a class
+  // template, that context is the (implicitly instantiated) declaration in the
+  // definition of the class template specialization.
+  if (FD->isCXXClassMember() &&
+      !isGenericLambdaCallOperatorOrStaticInvokerSpecialization(FD)) {
+    if (FunctionTemplateDecl *FTD = FD->getPrimaryTemplate()) {
+      Pattern = FTD->getFirstDecl();
+      Innermost = FD->getTemplateSpecializationArgs()->asArray();
+    }
+  }
+
   // Instantiate the expression.
   //
   // FIXME: Pass in a correct Pattern argument, otherwise
@@ -4705,12 +4736,10 @@ bool Sema::InstantiateDefaultArgument(SourceLocation CallLoc, FunctionDecl *FD,
   //
   // template<typename T>
   // A<T> Foo(int a = A<T>::FooImpl());
-  MultiLevelTemplateArgumentList TemplateArgs = getTemplateInstantiationArgs(
-      FD, FD->getLexicalDeclContext(),
-      /*Final=*/false, /*Innermost=*/std::nullopt,
-      /*RelativeToPrimary=*/true, /*Pattern=*/nullptr,
-      /*ForConstraintInstantiation=*/false, /*SkipForSpecialization=*/false,
-      /*ForDefaultArgumentSubstitution=*/true);
+  MultiLevelTemplateArgumentList TemplateArgs =
+      getTemplateInstantiationArgs(Pattern, Pattern->getLexicalDeclContext(),
+                                   /*Final=*/false, Innermost,
+                                   /*RelativeToPrimary=*/true);
 
   if (SubstDefaultArgument(CallLoc, Param, TemplateArgs, /*ForCallExpr*/ true))
     return true;
@@ -4751,7 +4780,7 @@ void Sema::InstantiateExceptionSpec(SourceLocation PointOfInstantiation,
   MultiLevelTemplateArgumentList TemplateArgs =
       getTemplateInstantiationArgs(Decl, Decl->getLexicalDeclContext(),
                                    /*Final=*/false, /*Innermost=*/std::nullopt,
-                                   /*RelativeToPrimary*/ true);
+                                   /*RelativeToPrimary=*/true);
 
   // FIXME: We can't use getTemplateInstantiationPattern(false) in general
   // here, because for a non-defining friend declaration in a class template,
@@ -5199,8 +5228,7 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
     SetDeclDefaulted(Function, PatternDecl->getLocation());
   } else {
     MultiLevelTemplateArgumentList TemplateArgs = getTemplateInstantiationArgs(
-        Function, Function->getLexicalDeclContext(), /*Final=*/false,
-        /*Innermost=*/std::nullopt, false, PatternDecl);
+        Function, Function->getLexicalDeclContext());
 
     // Substitute into the qualifier; we can get a substitution failure here
     // through evil use of alias templates.
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index c6d850c950613d206c86d4efe1392e460396fce5..2d1a6951a7fedd7aafc39817d31bc02bf1c9c770 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -18,6 +18,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
+#include "llvm/Support/SaveAndRestore.h"
 #include <optional>
 
 using namespace clang;
@@ -36,7 +37,7 @@ namespace {
 
     SmallVectorImpl<UnexpandedParameterPack> &Unexpanded;
 
-    bool InLambda = false;
+    bool InLambdaOrBlock = false;
     unsigned DepthLimit = (unsigned)-1;
 
 #ifndef NDEBUG
@@ -140,7 +141,7 @@ namespace {
     /// do not contain unexpanded parameter packs.
     bool TraverseStmt(Stmt *S) {
       Expr *E = dyn_cast_or_null<Expr>(S);
-      if ((E && E->containsUnexpandedParameterPack()) || InLambda)
+      if ((E && E->containsUnexpandedParameterPack()) || InLambdaOrBlock)
         return inherited::TraverseStmt(S);
 
       return true;
@@ -149,7 +150,8 @@ namespace {
     /// Suppress traversal into types that do not contain
     /// unexpanded parameter packs.
     bool TraverseType(QualType T) {
-      if ((!T.isNull() && T->containsUnexpandedParameterPack()) || InLambda)
+      if ((!T.isNull() && T->containsUnexpandedParameterPack()) ||
+          InLambdaOrBlock)
         return inherited::TraverseType(T);
 
       return true;
@@ -160,7 +162,7 @@ namespace {
     bool TraverseTypeLoc(TypeLoc TL) {
       if ((!TL.getType().isNull() &&
            TL.getType()->containsUnexpandedParameterPack()) ||
-          InLambda)
+          InLambdaOrBlock)
         return inherited::TraverseTypeLoc(TL);
 
       return true;
@@ -262,20 +264,28 @@ namespace {
       if (!Lambda->containsUnexpandedParameterPack())
         return true;
 
-      bool WasInLambda = InLambda;
+      SaveAndRestore _(InLambdaOrBlock, true);
       unsigned OldDepthLimit = DepthLimit;
 
-      InLambda = true;
       if (auto *TPL = Lambda->getTemplateParameterList())
         DepthLimit = TPL->getDepth();
 
       inherited::TraverseLambdaExpr(Lambda);
 
-      InLambda = WasInLambda;
       DepthLimit = OldDepthLimit;
       return true;
     }
 
+    /// Analogously for blocks.
+    bool TraverseBlockExpr(BlockExpr *Block) {
+      if (!Block->containsUnexpandedParameterPack())
+        return true;
+
+      SaveAndRestore _(InLambdaOrBlock, true);
+      inherited::TraverseBlockExpr(Block);
+      return true;
+    }
+
     /// Suppress traversal within pack expansions in lambda captures.
     bool TraverseLambdaCapture(LambdaExpr *Lambda, const LambdaCapture *C,
                                Expr *Init) {
@@ -323,11 +333,11 @@ Sema::DiagnoseUnexpandedParameterPacks(SourceLocation Loc,
 
   // If we are within a lambda expression and referencing a pack that is not
   // declared within the lambda itself, that lambda contains an unexpanded
-  // parameter pack, and we are done.
+  // parameter pack, and we are done. Analogously for blocks.
   // FIXME: Store 'Unexpanded' on the lambda so we don't need to recompute it
   // later.
-  SmallVector<UnexpandedParameterPack, 4> LambdaParamPackReferences;
-  if (auto *LSI = getEnclosingLambda()) {
+  SmallVector<UnexpandedParameterPack, 4> ParamPackReferences;
+  if (sema::CapturingScopeInfo *CSI = getEnclosingLambdaOrBlock()) {
     for (auto &Pack : Unexpanded) {
       auto DeclaresThisPack = [&](NamedDecl *LocalPack) {
         if (auto *TTPT = Pack.first.dyn_cast<const TemplateTypeParmType *>()) {
@@ -336,11 +346,11 @@ Sema::DiagnoseUnexpandedParameterPacks(SourceLocation Loc,
         }
         return declaresSameEntity(Pack.first.get<NamedDecl *>(), LocalPack);
       };
-      if (llvm::any_of(LSI->LocalPacks, DeclaresThisPack))
-        LambdaParamPackReferences.push_back(Pack);
+      if (llvm::any_of(CSI->LocalPacks, DeclaresThisPack))
+        ParamPackReferences.push_back(Pack);
     }
 
-    if (LambdaParamPackReferences.empty()) {
+    if (ParamPackReferences.empty()) {
       // Construct in lambda only references packs declared outside the lambda.
       // That's OK for now, but the lambda itself is considered to contain an
       // unexpanded pack in this case, which will require expansion outside the
@@ -363,16 +373,16 @@ Sema::DiagnoseUnexpandedParameterPacks(SourceLocation Loc,
         }
         // Coumpound-statements outside the lambda are OK for now; we'll check
         // for those when we finish handling the lambda.
-        if (Func == LSI)
+        if (Func == CSI)
           break;
       }
 
       if (!EnclosingStmtExpr) {
-        LSI->ContainsUnexpandedParameterPack = true;
+        CSI->ContainsUnexpandedParameterPack = true;
         return false;
       }
     } else {
-      Unexpanded = LambdaParamPackReferences;
+      Unexpanded = ParamPackReferences;
     }
   }
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 5625a075c46a8d8ce2b401b93e43c39947fbfa22..9630805d61d79b69890ff3d68246532552825d40 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -12004,6 +12004,34 @@ void OpenACCClauseTransform<Derived>::VisitAsyncClause(
                                          : nullptr,
       ParsedClause.getEndLoc());
 }
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitWorkerClause(
+    const OpenACCWorkerClause &C) {
+  if (C.hasIntExpr()) {
+    // restrictions on this expression are all "does it exist in certain
+    // situations" that are not possible to be dependent, so the only check we
+    // have is that it transforms, and is an int expression.
+    ExprResult Res = Self.TransformExpr(const_cast<Expr *>(C.getIntExpr()));
+    if (!Res.isUsable())
+      return;
+
+    Res = Self.getSema().OpenACC().ActOnIntExpr(OpenACCDirectiveKind::Invalid,
+                                                C.getClauseKind(),
+                                                C.getBeginLoc(), Res.get());
+    if (!Res.isUsable())
+      return;
+    ParsedClause.setIntExprDetails(Res.get());
+  }
+
+  NewClause = OpenACCWorkerClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(),
+      ParsedClause.getNumIntExprs() != 0 ? ParsedClause.getIntExprs()[0]
+                                         : nullptr,
+      ParsedClause.getEndLoc());
+}
+
 template <typename Derived>
 void OpenACCClauseTransform<Derived>::VisitWaitClause(
     const OpenACCWaitClause &C) {
@@ -12157,6 +12185,29 @@ void OpenACCClauseTransform<Derived>::VisitTileClause(
       ParsedClause.getLParenLoc(), ParsedClause.getIntExprs(),
       ParsedClause.getEndLoc());
 }
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitGangClause(
+    const OpenACCGangClause &C) {
+  llvm::SmallVector<OpenACCGangKind> TransformedGangKinds;
+  llvm::SmallVector<Expr *> TransformedIntExprs;
+
+  for (unsigned I = 0; I < C.getNumExprs(); ++I) {
+    ExprResult ER = Self.TransformExpr(const_cast<Expr *>(C.getExpr(I).second));
+    if (!ER.isUsable())
+      continue;
+
+    ER = Self.getSema().OpenACC().CheckGangExpr(C.getExpr(I).first, ER.get());
+    if (!ER.isUsable())
+      continue;
+    TransformedGangKinds.push_back(C.getExpr(I).first);
+    TransformedIntExprs.push_back(ER.get());
+  }
+
+  NewClause = OpenACCGangClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(), TransformedGangKinds, TransformedIntExprs,
+      ParsedClause.getEndLoc());
+}
 } // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 0ae4cdddb570b5086715e7302b3a6393287ddf12..28be3418270be3f5e742fd5bd9cdbd10de51025f 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -10003,7 +10003,8 @@ void ASTReader::finishPendingActions() {
 
     auto RTD = cast<RedeclarableTemplateDecl>(D)->getCanonicalDecl();
     for (auto *R = getMostRecentExistingDecl(RTD); R; R = R->getPreviousDecl())
-      cast<RedeclarableTemplateDecl>(R)->Common = RTD->Common;
+      cast<RedeclarableTemplateDecl>(R)->setCommonPtr(
+          RTD->getCommonPtrInternal());
   }
   PendingDefinitions.clear();
 
@@ -12372,10 +12373,27 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     return OpenACCTileClause::Create(getContext(), BeginLoc, LParenLoc,
                                      SizeExprs, EndLoc);
   }
+  case OpenACCClauseKind::Gang: {
+    SourceLocation LParenLoc = readSourceLocation();
+    unsigned NumExprs = readInt();
+    llvm::SmallVector<OpenACCGangKind> GangKinds;
+    llvm::SmallVector<Expr *> Exprs;
+    for (unsigned I = 0; I < NumExprs; ++I) {
+      GangKinds.push_back(readEnum<OpenACCGangKind>());
+      Exprs.push_back(readSubExpr());
+    }
+    return OpenACCGangClause::Create(getContext(), BeginLoc, LParenLoc,
+                                     GangKinds, Exprs, EndLoc);
+  }
+  case OpenACCClauseKind::Worker: {
+    SourceLocation LParenLoc = readSourceLocation();
+    Expr *WorkerExpr = readBool() ? readSubExpr() : nullptr;
+    return OpenACCWorkerClause::Create(getContext(), BeginLoc, LParenLoc,
+                                       WorkerExpr, EndLoc);
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
-  case OpenACCClauseKind::Worker:
   case OpenACCClauseKind::Vector:
   case OpenACCClauseKind::NoHost:
   case OpenACCClauseKind::UseDevice:
@@ -12388,7 +12406,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
   }
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 7cead2728ca938ba375a361155674eab2e75f034..1ccc810f415eb45b5ab50c2523fc8825fda2c5d5 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2416,11 +2416,13 @@ ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
   // Make sure we've allocated the Common pointer first. We do this before
   // VisitTemplateDecl so that getCommonPtr() can be used during initialization.
   RedeclarableTemplateDecl *CanonD = D->getCanonicalDecl();
-  if (!CanonD->Common) {
-    CanonD->Common = CanonD->newCommon(Reader.getContext());
+  if (!CanonD->getCommonPtrInternal()) {
+    CanonD->setCommonPtr(CanonD->newCommon(Reader.getContext()));
     Reader.PendingDefinitions.insert(CanonD);
   }
-  D->Common = CanonD->Common;
+  D->setCommonPtr(CanonD->getCommonPtrInternal());
+  if (Record.readInt())
+    D->setMemberSpecialization();
 
   // If this is the first declaration of the template, fill in the information
   // for the 'common' pointer.
@@ -2429,8 +2431,6 @@ ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
       assert(RTD->getKind() == D->getKind() &&
              "InstantiatedFromMemberTemplate kind mismatch");
       D->setInstantiatedFromMemberTemplate(RTD);
-      if (Record.readInt())
-        D->setMemberSpecialization();
     }
   }
 
@@ -2562,12 +2562,12 @@ void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
   D->TemplateParams = Params;
 
   RedeclarableResult Redecl = VisitClassTemplateSpecializationDeclImpl(D);
+  D->InstantiatedFromMember.setInt(Record.readInt());
 
   // These are read/set from/to the first declaration.
   if (ThisDeclID == Redecl.getFirstID()) {
     D->InstantiatedFromMember.setPointer(
-      readDeclAs<ClassTemplatePartialSpecializationDecl>());
-    D->InstantiatedFromMember.setInt(Record.readInt());
+        readDeclAs<ClassTemplatePartialSpecializationDecl>());
   }
 }
 
@@ -2660,12 +2660,12 @@ void ASTDeclReader::VisitVarTemplatePartialSpecializationDecl(
   D->TemplateParams = Params;
 
   RedeclarableResult Redecl = VisitVarTemplateSpecializationDeclImpl(D);
+  D->InstantiatedFromMember.setInt(Record.readInt());
 
   // These are read/set from/to the first declaration.
   if (ThisDeclID == Redecl.getFirstID()) {
     D->InstantiatedFromMember.setPointer(
         readDeclAs<VarTemplatePartialSpecializationDecl>());
-    D->InstantiatedFromMember.setInt(Record.readInt());
   }
 }
 
@@ -2888,7 +2888,7 @@ void ASTDeclReader::mergeRedeclarableTemplate(RedeclarableTemplateDecl *D,
   // If we merged the template with a prior declaration chain, merge the
   // common pointer.
   // FIXME: Actually merge here, don't just overwrite.
-  D->Common = D->getCanonicalDecl()->Common;
+  D->setCommonPtr(D->getCanonicalDecl()->getCommonPtrInternal());
 }
 
 /// "Cast" to type T, asserting if we don't have an implicit conversion.
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 286dec0fdfeeaae1fbcb2b29df206be23a75e3d0..fc489d9932d53a90e2bd5382d0b8c6d01f0e008e 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8217,10 +8217,27 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
       AddStmt(E);
     return;
   }
+  case OpenACCClauseKind::Gang: {
+    const auto *GC = cast<OpenACCGangClause>(C);
+    writeSourceLocation(GC->getLParenLoc());
+    writeUInt32(GC->getNumExprs());
+    for (unsigned I = 0; I < GC->getNumExprs(); ++I) {
+      writeEnum(GC->getExpr(I).first);
+      AddStmt(const_cast<Expr *>(GC->getExpr(I).second));
+    }
+    return;
+  }
+  case OpenACCClauseKind::Worker: {
+    const auto *WC = cast<OpenACCWorkerClause>(C);
+    writeSourceLocation(WC->getLParenLoc());
+    writeBool(WC->hasIntExpr());
+    if (WC->hasIntExpr())
+      AddStmt(const_cast<Expr *>(WC->getIntExpr()));
+    return;
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
-  case OpenACCClauseKind::Worker:
   case OpenACCClauseKind::Vector:
   case OpenACCClauseKind::NoHost:
   case OpenACCClauseKind::UseDevice:
@@ -8233,7 +8250,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
   }
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index b71684569609acefdbc949b806d054509344b312..f21cbd11b6ab8955c8ffc74ef5ce611395ee36cf 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1713,14 +1713,13 @@ void ASTDeclWriter::VisitRequiresExprBodyDecl(RequiresExprBodyDecl *D) {
 void ASTDeclWriter::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
   VisitRedeclarable(D);
 
+  Record.push_back(D->isMemberSpecialization());
+
   // Emit data to initialize CommonOrPrev before VisitTemplateDecl so that
   // getCommonPtr() can be used while this is still initializing.
-  if (D->isFirstDecl()) {
+  if (D->isFirstDecl())
     // This declaration owns the 'common' pointer, so serialize that data now.
     Record.AddDeclRef(D->getInstantiatedFromMemberTemplate());
-    if (D->getInstantiatedFromMemberTemplate())
-      Record.push_back(D->isMemberSpecialization());
-  }
 
   VisitTemplateDecl(D);
   Record.push_back(D->getIdentifierNamespace());
@@ -1806,11 +1805,10 @@ void ASTDeclWriter::VisitClassTemplatePartialSpecializationDecl(
 
   VisitClassTemplateSpecializationDecl(D);
 
+  Record.push_back(D->isMemberSpecialization());
   // These are read/set from/to the first declaration.
-  if (D->getPreviousDecl() == nullptr) {
+  if (D->isFirstDecl())
     Record.AddDeclRef(D->getInstantiatedFromMember());
-    Record.push_back(D->isMemberSpecialization());
-  }
 
   Code = serialization::DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION;
 }
@@ -1874,12 +1872,11 @@ void ASTDeclWriter::VisitVarTemplatePartialSpecializationDecl(
   Record.AddTemplateParameterList(D->getTemplateParameters());
 
   VisitVarTemplateSpecializationDecl(D);
+  Record.push_back(D->isMemberSpecialization());
 
   // These are read/set from/to the first declaration.
-  if (D->getPreviousDecl() == nullptr) {
+  if (D->isFirstDecl())
     Record.AddDeclRef(D->getInstantiatedFromMember());
-    Record.push_back(D->isMemberSpecialization());
-  }
 
   Code = serialization::DECL_VAR_TEMPLATE_PARTIAL_SPECIALIZATION;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
index f0dc5efd75f7d6597d7aee8873c19891f0f89947..7a85d9e20730680f181e865ba9f46d5ae358b1a3 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
@@ -61,6 +61,10 @@ void PointerSubChecker::checkPreStmt(const BinaryOperator *B,
   if (LR->getSymbolicBase() || RR->getSymbolicBase())
     return;
 
+  if (!B->getLHS()->getType()->isPointerType() ||
+      !B->getRHS()->getType()->isPointerType())
+    return;
+
   const auto *ElemLR = dyn_cast<ElementRegion>(LR);
   const auto *ElemRR = dyn_cast<ElementRegion>(RR);
 
diff --git a/clang/test/AST/ByteCode/cxx1z.cpp b/clang/test/AST/ByteCode/cxx1z.cpp
index 1a06597fa348fe8bf6762353b12dcd9f7033d21e..57f99235a2b2015446747d938032b55743b7c8dd 100644
--- a/clang/test/AST/ByteCode/cxx1z.cpp
+++ b/clang/test/AST/ByteCode/cxx1z.cpp
@@ -13,3 +13,7 @@ namespace Temp {
 
 char arr[3];
 A<const char*, &arr[1]> d; // both-error {{refers to subobject '&arr[1]'}}
+
+void Func() {
+  A<const char*, __func__> a; // both-error {{pointer to subobject of predefined '__func__' variable}}
+}
diff --git a/clang/test/AST/ByteCode/vectors.cpp b/clang/test/AST/ByteCode/vectors.cpp
index 7662abeeb7f596582f79c31c2259500c1e3a2853..a0aace44f3c981eefbda1c9bba618af8ad36347d 100644
--- a/clang/test/AST/ByteCode/vectors.cpp
+++ b/clang/test/AST/ByteCode/vectors.cpp
@@ -125,3 +125,21 @@ constexpr int a2() {
 }
 
 static_assert(a2() == 0);
+
+namespace {
+  /// convertvector expr with a per-element floating-point cast
+
+  typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
+  typedef double __m128d __attribute__((__vector_size__(16), __aligned__(16)));
+  typedef float __v4sf __attribute__((__vector_size__(16)));
+  typedef double __v2df __attribute__((__vector_size__(16)));
+
+  static inline constexpr __m128d
+  _mm_cvtps_pd(__m128 __a) {
+    return __builtin_convertvector(__builtin_shufflevector(__a, __a, 0, 1), __v2df);
+  }
+
+  constexpr __m128 kf1 {-1.0f,+2.0f,-3.0f,+4.0f};
+  constexpr __m128d v_mm_cvtps_pd = _mm_cvtps_pd(kf1);
+  static_assert(v_mm_cvtps_pd[0] == -1.0 && v_mm_cvtps_pd[1] == +2.0);
+}
diff --git a/clang/test/AST/ast-print-openacc-loop-construct.cpp b/clang/test/AST/ast-print-openacc-loop-construct.cpp
index aee4591cab428f56fb92f49a1eace6d277506c60..ee11435aaa4b1c88bf19828f3c58938016d68a0e 100644
--- a/clang/test/AST/ast-print-openacc-loop-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-loop-construct.cpp
@@ -95,4 +95,125 @@ void foo() {
     for(;;)
       for(;;)
         for(;;);
+
+// CHECK: #pragma acc loop gang(dim: 2)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc loop gang(dim:2)
+  for(;;);
+
+// CHECK: #pragma acc loop gang(static: i)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc loop gang(static:i)
+  for(;;);
+
+// CHECK: #pragma acc loop gang(static: i) gang(dim: 2)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc loop gang(static:i) gang(dim:2)
+  for(;;);
+
+// CHECK: #pragma acc parallel
+// CHECK-NEXT: #pragma acc loop gang(dim: 2)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc parallel
+#pragma acc loop gang(dim:2)
+  for(;;);
+
+// CHECK: #pragma acc parallel
+// CHECK-NEXT: #pragma acc loop gang(static: i)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc parallel
+#pragma acc loop gang(static:i)
+  for(;;);
+
+// CHECK: #pragma acc parallel
+// CHECK-NEXT: #pragma acc loop gang(static: i) gang(dim: 2)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc parallel
+#pragma acc loop gang(static:i) gang(dim:2)
+  for(;;);
+
+// CHECK: #pragma acc kernels
+// CHECK-NEXT: #pragma acc loop gang(num: i) gang(static: i)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc kernels
+#pragma acc loop gang(i) gang(static:i)
+  for(;;);
+
+// CHECK: #pragma acc kernels
+// CHECK-NEXT: #pragma acc loop gang(num: i) gang(static: i)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc kernels
+#pragma acc loop gang(num:i) gang(static:i)
+  for(;;);
+
+// CHECK: #pragma acc serial
+// CHECK-NEXT: #pragma acc loop gang(static: i)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc serial
+#pragma acc loop gang(static:i)
+  for(;;);
+
+// CHECK: #pragma acc serial
+// CHECK-NEXT: #pragma acc loop gang(static: *)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc serial
+#pragma acc loop gang(static:*)
+  for(;;);
+
+// CHECK: #pragma acc serial
+// CHECK-NEXT: #pragma acc loop
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc serial
+#pragma acc loop gang
+  for(;;);
+
+// CHECK: #pragma acc loop worker
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc parallel
+#pragma acc loop worker
+  for(;;);
+
+// CHECK: #pragma acc parallel
+// CHECK-NEXT: #pragma acc loop worker
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc parallel
+#pragma acc loop worker
+  for(;;);
+
+// CHECK: #pragma acc serial
+// CHECK-NEXT: #pragma acc loop worker
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc serial
+#pragma acc loop worker
+  for(;;);
+
+// CHECK: #pragma acc kernels
+// CHECK-NEXT: #pragma acc loop worker(num: 5)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc kernels
+#pragma acc loop worker(5)
+  for(;;);
+
+// CHECK: #pragma acc kernels
+// CHECK-NEXT: #pragma acc loop worker(num: 5)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc kernels
+#pragma acc loop worker(num:5)
+  for(;;);
 }
diff --git a/clang/test/Analysis/pointer-sub.c b/clang/test/Analysis/pointer-sub.c
index 1c9d676ebb8f24ae0a39025ea1452d4b5ed8d5ba..25fb7f043d468cc6a9bf4e005321df3ccc5a975a 100644
--- a/clang/test/Analysis/pointer-sub.c
+++ b/clang/test/Analysis/pointer-sub.c
@@ -1,5 +1,7 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=security.PointerSub -analyzer-output=text-minimal -verify %s
 
+typedef int * Ptr;
+
 void f1(void) {
   int x, y, z[10];
   int d = &y - &x; // expected-warning{{Subtraction of two pointers that do not point into the same array is undefined behavior}}
@@ -10,6 +12,12 @@ void f1(void) {
   d = &x - (&x + 1); // no-warning
   d = (&x + 0) - &x; // no-warning
   d = (z + 10) - z; // no-warning
+  d = (long long)&y - (long long)&x; // no-warning
+  long long l = 1;
+  d = l - (long long)&y; // no-warning
+  Ptr p1 = &x;
+  Ptr p2 = &y;
+  d = p1 - p2; // expected-warning{{Subtraction of two pointers that do not point into the same array is undefined behavior}}
 }
 
 void f2(void) {
@@ -28,6 +36,10 @@ void f2(void) {
 
   d = (int *)((char *)(&a[4]) + sizeof(int)) - &a[4]; // no-warning (pointers into the same array data)
   d = (int *)((char *)(&a[4]) + 1) - &a[4]; // expected-warning{{Subtraction of two pointers that}}
+
+  long long a1 = (long long)&a[1];
+  long long b1 = (long long)&b[1];
+  d = a1 - b1;
 }
 
 void f3(void) {
diff --git a/clang/test/C/C2y/n3298.c b/clang/test/C/C2y/n3298.c
new file mode 100644
index 0000000000000000000000000000000000000000..b39d0528963514c0b7b38d09038e0b9ba500ee08
--- /dev/null
+++ b/clang/test/C/C2y/n3298.c
@@ -0,0 +1,104 @@
+// RUN: %clang_cc1 -verify=ped -std=c23 -Wall -pedantic %s
+// RUN: %clang_cc1 -verify=yay -std=c2y -Wall -pedantic %s
+// RUN: %clang_cc1 -verify=pre -std=c2y -Wpre-c2y-compat -Wall -pedantic %s
+// RUN: %clang_cc1 -verify=gnu -Wall -Wgnu -x c++ %s
+// RUN: %clang_cc1 -verify=yay -Wall -Wgnu -Wno-gnu-imaginary-constant -x c++ %s
+
+
+/* WG14 N3298: Yes
+ * Introduce complex literals v. 2
+ *
+ * This introduces two suffixes for making complex literals: i and j (and I and
+ * J), which can be combined in any order with the other floating literal
+ * suffixes.
+ *
+ * We support these suffixes in older language modes as a conforming extension.
+ * It used to be a GNU extension, but now it's a C2y extension.
+ */
+
+// yay-no-diagnostics
+
+static_assert(_Generic(12.0i, _Complex double : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0fi, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0li, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0if, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0il, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+
+static_assert(_Generic(12.0I, _Complex double : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0fI, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0lI, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0If, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0Il, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+
+static_assert(_Generic(12.0j, _Complex double : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0fj, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0lj, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0jf, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0jl, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+
+static_assert(_Generic(12.0J, _Complex double : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0fJ, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0lJ, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0Jf, _Complex float : 1, default : 0));       /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+static_assert(_Generic(12.0Jl, _Complex long double : 1, default : 0)); /* gnu-warning {{imaginary constants are a GNU extension}}
+                                                                           ped-warning {{imaginary constants are a C2y extension}}
+                                                                           pre-warning {{imaginary constants are incompatible with C standards before C2y}}
+                                                                         */
+
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index 2d84b0d73053f6663bed457facb7e994e9421900..98829d53db934f7e5535e340955408c44b0853e5 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -127,6 +127,7 @@ if( NOT CLANG_BUILT_STANDALONE )
     llvm-dwarfdump
     llvm-ifs
     llvm-lto2
+    llvm-mc
     llvm-modextract
     llvm-nm
     llvm-objcopy
diff --git a/clang/test/CXX/class.derived/p2.cpp b/clang/test/CXX/class.derived/p2.cpp
index 87e0f7486154566a94ceb304a61f06320b29e16a..401ee37ad6b64c801707d49db2668b8748437c8a 100644
--- a/clang/test/CXX/class.derived/p2.cpp
+++ b/clang/test/CXX/class.derived/p2.cpp
@@ -6,4 +6,16 @@ namespace PR5840 {
   struct Base {};
   int Base = 10;
   struct Derived : Base {};
-}
+} // namespace PR5840
+
+namespace issue_16855 {
+  struct x {};
+  namespace
+  {
+      namespace x
+      {
+          struct y : x
+          {};
+      } // namespace x
+  }
+} // namespace issue_16855
diff --git a/clang/test/CXX/drs/cwg14xx.cpp b/clang/test/CXX/drs/cwg14xx.cpp
index 5301185d04698291b7794ba9106c59f966261faf..cb2f34bf5e427fc943bbe85b89c0b7a81e78b59f 100644
--- a/clang/test/CXX/drs/cwg14xx.cpp
+++ b/clang/test/CXX/drs/cwg14xx.cpp
@@ -627,7 +627,7 @@ int i = N::f();
 
 namespace cwg1479 { // cwg1479: 3.1
 #if __cplusplus >= 201103L
-  int operator"" _a(const char*, std::size_t = 0);
+  int operator""_a(const char*, std::size_t = 0);
   // since-cxx11-error@-1 {{literal operator cannot have a default argument}}
 #endif
 }
diff --git a/clang/test/CXX/drs/cwg25xx.cpp b/clang/test/CXX/drs/cwg25xx.cpp
index 1924008f15ba5802014b6b248039def2fd4ca18d..87a728088ee6e4fcf4e18d0d231f59b5011970b9 100644
--- a/clang/test/CXX/drs/cwg25xx.cpp
+++ b/clang/test/CXX/drs/cwg25xx.cpp
@@ -88,6 +88,9 @@ operator""  _div();
 using ::cwg2521::operator"" _\u03C0___;
 using ::cwg2521::operator""_div;
 // since-cxx11-warning@-2 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}}
+
+long double operator"" _RESERVED(long double);
+// since-cxx11-warning@-1 {{identifier '_RESERVED' preceded by whitespace in a literal operator declaration is deprecated}}
 #pragma clang diagnostic pop
 #endif
 } // namespace cwg2521
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/blocks.mm b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/blocks.mm
index e93c37f3b9ae128531b4e973a76628949d0a1c96..b644e0fdfe3c0f87dd4c2b7a23417d205e9856a2 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/blocks.mm
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/blocks.mm
@@ -119,17 +119,16 @@ namespace PR13117 {
 
     template<typename ... Args> static void f1()
     {
-      (void)^(Args args) { // expected-error{{block contains unexpanded parameter pack 'Args'}}
+      (void)^(Args args) { // expected-error{{expression contains unexpanded parameter pack 'Args'}}
       };
     }
 
     template<typename ... Args> static void f2()
     {
-      // FIXME: Allow this.
       f(
-        ^(Args args) // expected-error{{block contains unexpanded parameter pack 'Args'}}
+        ^(Args args)
         { }
-        ... // expected-error{{pack expansion does not contain any unexpanded parameter packs}}
+        ...
       );
     }
 
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p1.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p1.cpp
index 1c227a1b10d385ab5a1019b55d5625cb9a728835..ec478fbba60a18ec9a9059ab865521407a47c8fc 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p1.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p1.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++11 -Wno-deprecated-literal-operator -verify %s
 
 void operator "" p31(long double); // expected-warning{{user-defined literal suffixes not starting with '_' are reserved}}
 void operator "" _p31(long double);
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p10.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p10.cpp
index 1b5d3880cb61299cfb7b7a69d7bb2c0bd6ca442b..6a9d713ca72d2e3cb822115b97a77adc6ab76e7b 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p10.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p10.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++11 -verify %s
+// RUN: %clang_cc1 -std=c++11 -Wno-deprecated-literal-operator -verify %s
 
 using size_t = decltype(sizeof(int));
 void operator "" wibble(const char *); // expected-warning {{user-defined literal suffixes not starting with '_' are reserved; no literal will invoke this operator}}
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p11.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p11.cpp
index 8b5fcf4b609b65395c074c4468067af88005605c..d69a58a7dfad206443d1567d9ff9fa2a0cd99c79 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p11.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p11.cpp
@@ -6,16 +6,16 @@ template<typename T, typename U> struct same_type;
 template<typename T> struct same_type<T, T> {};
 template<typename T> using X = T;
 template<typename CharT, X<CharT>...>
-int operator "" _x(); // expected-warning {{string literal operator templates are a GNU extension}}
+int operator ""_x(); // expected-warning {{string literal operator templates are a GNU extension}}
 template<char...>
-double operator "" _x();
+double operator ""_x();
 
 auto a="string"_x;
 auto b=42_x;
 same_type<decltype(a), int> test_a;
 same_type<decltype(b), double> test_b;
 
-char operator "" _x(const char *begin, size_t size);
+char operator ""_x(const char *begin, size_t size);
 auto c="string"_x;
 auto d=L"string"_x;
 same_type<decltype(c), char> test_c;
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p3.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p3.cpp
index d764989312c246a33270d61b9d6c5fce71b20861..e5ab09c628bcfab14d8097b352f35b0cd8e82ba2 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p3.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p3.cpp
@@ -1,18 +1,18 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify %s
 
-int &operator "" _x1 (unsigned long long);
+int &operator ""_x1 (unsigned long long);
 int &i1 = 0x123_x1;
 
-double &operator "" _x1 (const char *);
+double &operator ""_x1 (const char *);
 int &i2 = 45_x1;
 
-template<char...> char &operator "" _x1 ();
+template<char...> char &operator ""_x1 ();
 int &i3 = 0377_x1;
 
 int &i4 = 90000000000000000000000000000000000000000000000_x1; // expected-error {{integer literal is too large to be represented in any integer type}}
 
-double &operator "" _x2 (const char *);
+double &operator ""_x2 (const char *);
 double &i5 = 123123123123123123123123123123123123123123123_x2;
 
-template<char...Cs> constexpr int operator "" _x3() { return sizeof...(Cs); }
+template<char...Cs> constexpr int operator ""_x3() { return sizeof...(Cs); }
 static_assert(123456789012345678901234567890123456789012345678901234567890_x3 == 60, "");
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p4.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p4.cpp
index 011e832c69d72988d2be262fac01a694d82d9589..7dbe70ce084e7613eff1adb16c4367a2a8065074 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p4.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p4.cpp
@@ -1,18 +1,18 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify %s
 
-int &operator "" _x1 (long double);
+int &operator ""_x1 (long double);
 int &i1 = 0.123_x1;
 
-double &operator "" _x1 (const char *);
+double &operator ""_x1 (const char *);
 int &i2 = 45._x1;
 
-template<char...> char &operator "" _x1 ();
+template<char...> char &operator ""_x1 ();
 int &i3 = 0377e-1_x1;
 
 int &i4 = 1e1000000_x1; // expected-warning {{too large for type 'long double'}}
 
-double &operator "" _x2 (const char *);
+double &operator ""_x2 (const char *);
 double &i5 = 1e1000000_x2;
 
-template<char...Cs> constexpr int operator "" _x3() { return sizeof...(Cs); }
+template<char...Cs> constexpr int operator ""_x3() { return sizeof...(Cs); }
 static_assert(1e1000000_x3 == 9, "");
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p5.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p5.cpp
index aee20545ececfcea60daf6db18ae9b3e5d9df6f4..afadba282e626c9492bac13d81f563cdce63bd59 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p5.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p5.cpp
@@ -3,19 +3,19 @@
 
 using size_t = decltype(sizeof(int));
 
-int &operator "" _x1 (const char *);
-double &operator "" _x1 (const char *, size_t);
+int &operator ""_x1 (const char *);
+double &operator ""_x1 (const char *, size_t);
 double &i1 = "foo"_x1;
 #if __cplusplus >= 202002L
 using char8 = float;
-float &operator "" _x1 (const char8_t *, size_t);
+float &operator ""_x1 (const char8_t *, size_t);
 #else
 using char8 = double;
 #endif
 char8 &i2 = u8"foo"_x1;
 double &i3 = L"foo"_x1; // expected-error {{no matching literal operator for call to 'operator""_x1' with arguments of types 'const wchar_t *' and 'unsigned long'}}
 
-char &operator "" _x1(const wchar_t *, size_t);
+char &operator ""_x1(const wchar_t *, size_t);
 char &i4 = L"foo"_x1; // ok
 double &i5 = R"(foo)"_x1; // ok
 char8 &i6 = u\
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p6.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p6.cpp
index 23cd7081d5e3ee91e2c86b96e97256c9a26e555e..b1df641f2dc43c87e2c6ae8a7fcaa35c079de461 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p6.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p6.cpp
@@ -2,13 +2,13 @@
 
 using size_t = decltype(sizeof(int));
 
-int &operator "" _x1 (const char *);
+int &operator ""_x1 (const char *);
 double &i1 = 'a'_x1; // expected-error {{no matching literal operator}}
-double &operator "" _x1 (wchar_t);
+double &operator ""_x1 (wchar_t);
 double &i2 = L'a'_x1;
 double &i3 = 'a'_x1; // expected-error {{no matching literal operator}}
-double &i4 = operator"" _x1('a'); // ok
+double &i4 = operator""_x1('a'); // ok
 
-char &operator "" _x1(char16_t);
+char &operator ""_x1(char16_t);
 char &i5 = u'a'_x1; // ok
 double &i6 = L'a'_x1; // ok
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p7.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p7.cpp
index 0b40ecdc143fcb067e8a4219a623091524d80531..d571fcb8697eb01e1d844c396f34a785ac99148b 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p7.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p7.cpp
@@ -10,9 +10,9 @@ template<typename T> struct same_type<T, T> {};
 
 namespace std_example {
 
-long double operator "" _w(long double);
-std::string operator "" _w(const char16_t*, size_t);
-unsigned operator "" _w(const char*);
+long double operator ""_w(long double);
+std::string operator ""_w(const char16_t*, size_t);
+unsigned operator ""_w(const char*);
 int main() {
   auto v1 = 1.2_w;    // calls operator""_w(1.2L)
   auto v2 = u"one"_w; // calls operator""_w(u"one", 3)
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p8.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p8.cpp
index d9078221ff5e3a82bd5daa35a65e4d7f21f44666..67d976263e0167418295a56eaff0171e9e265ac6 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p8.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p8.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -std=c++11 -verify %s
 
 using size_t = decltype(sizeof(int));
-constexpr const char *operator "" _id(const char *p, size_t) { return p; }
+constexpr const char *operator ""_id(const char *p, size_t) { return p; }
 constexpr const char *s = "foo"_id "bar" "baz"_id "quux";
 
 constexpr bool streq(const char *p, const char *q) {
@@ -9,8 +9,8 @@ constexpr bool streq(const char *p, const char *q) {
 }
 static_assert(streq(s, "foobarbazquux"), "");
 
-constexpr const char *operator "" _trim(const char *p, size_t n) {
-  return *p == ' ' ? operator "" _trim(p + 1, n - 1) : p;
+constexpr const char *operator ""_trim(const char *p, size_t n) {
+  return *p == ' ' ? operator ""_trim(p + 1, n - 1) : p;
 }
 constexpr const char *t = "   " " "_trim "  foo";
 static_assert(streq(t, "foo"), "");
diff --git a/clang/test/CXX/lex/lex.literal/lex.ext/p9.cpp b/clang/test/CXX/lex/lex.literal/lex.ext/p9.cpp
index 65e27b41b066804af7e6bdfb7d4bcfb8ce64feb6..fbdedd119d3d68e9364d87f941284dcf0dfd4a93 100644
--- a/clang/test/CXX/lex/lex.literal/lex.ext/p9.cpp
+++ b/clang/test/CXX/lex/lex.literal/lex.ext/p9.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s
 
 using size_t = decltype(sizeof(int));
-void operator "" _x(const wchar_t *, size_t);
+void operator ""_x(const wchar_t *, size_t);
 
 namespace std_example {
 
diff --git a/clang/test/CXX/over/over.oper/over.literal/p2.cpp b/clang/test/CXX/over/over.oper/over.literal/p2.cpp
index f3ebadd2b8b9bc188a010186e2d31ea1c1d887b9..cf26806e9e7d757319d5618f7d16496fa14b68d7 100644
--- a/clang/test/CXX/over/over.oper/over.literal/p2.cpp
+++ b/clang/test/CXX/over/over.oper/over.literal/p2.cpp
@@ -1,43 +1,43 @@
 // RUN: %clang_cc1 -std=c++11 %s -verify
 
-void operator "" _a(const char *);
+void operator ""_a(const char *);
 
 namespace N {
-  using ::operator "" _a;
+  using ::operator ""_a;
 
-  void operator "" _b(const char *);
+  void operator ""_b(const char *);
 }
 
-using N::operator "" _b;
+using N::operator ""_b;
 
 class C {
-  void operator "" _c(const char *); // expected-error {{must be in a namespace or global scope}}
+  void operator ""_c(const char *); // expected-error {{must be in a namespace or global scope}}
 
-  static void operator "" _c(unsigned long long); // expected-error {{must be in a namespace or global scope}}
+  static void operator ""_c(unsigned long long); // expected-error {{must be in a namespace or global scope}}
 
-  friend void operator "" _d(const char *);
+  friend void operator ""_d(const char *);
 };
 
-int operator "" _e; // expected-error {{cannot be the name of a variable}}
+int operator ""_e; // expected-error {{cannot be the name of a variable}}
 
 void f() {
-  int operator "" _f; // expected-error {{cannot be the name of a variable}}
+  int operator ""_f; // expected-error {{cannot be the name of a variable}}
 }
 
 extern "C++" {
-  void operator "" _g(const char *);
+  void operator ""_g(const char *);
 }
 
-template<char...> void operator "" _h() {}
+template<char...> void operator ""_h() {}
 
-template<> void operator "" _h<'a', 'b', 'c'>() {}
+template<> void operator ""_h<'a', 'b', 'c'>() {}
 
-template void operator "" _h<'a', 'b', 'c', 'd'>();
+template void operator ""_h<'a', 'b', 'c', 'd'>();
 
 namespace rdar13605348 {
 
 class C {
-  double operator"" _x(long double value) { return double(value); } // expected-error{{literal operator 'operator""_x' must be in a namespace or global scope}}
+  double operator""_x(long double value) { return double(value); } // expected-error{{literal operator 'operator""_x' must be in a namespace or global scope}}
   double value() { return 3.2_x; } // expected-error{{no matching literal operator for call to}}
 };
 
diff --git a/clang/test/CXX/over/over.oper/over.literal/p3.cpp b/clang/test/CXX/over/over.oper/over.literal/p3.cpp
index 674ace9aee192917e3b1146fc7986b8dd1e659dd..53ebe102630843bdaf1180dea61891cf2b12a4a8 100644
--- a/clang/test/CXX/over/over.oper/over.literal/p3.cpp
+++ b/clang/test/CXX/over/over.oper/over.literal/p3.cpp
@@ -3,38 +3,38 @@
 using size_t = decltype(sizeof(int));
 
 // Acceptable parameter declarations
-char operator "" _a(const char *);
-char operator "" _a(const char []);
-char operator "" _a(unsigned long long);
-char operator "" _a(long double);
-char operator "" _a(char);
-char operator "" _a(const volatile char);
-char operator "" _a(wchar_t);
-char operator "" _a(char16_t);
-char operator "" _a(char32_t);
-char operator "" _a(const char *, size_t);
-char operator "" _a(const wchar_t *, size_t);
-char operator "" _a(const char16_t *, size_t);
-char operator "" _a(const char32_t *, size_t);
-char operator "" _a(const char [32], size_t);
+char operator ""_a(const char *);
+char operator ""_a(const char []);
+char operator ""_a(unsigned long long);
+char operator ""_a(long double);
+char operator ""_a(char);
+char operator ""_a(const volatile char);
+char operator ""_a(wchar_t);
+char operator ""_a(char16_t);
+char operator ""_a(char32_t);
+char operator ""_a(const char *, size_t);
+char operator ""_a(const wchar_t *, size_t);
+char operator ""_a(const char16_t *, size_t);
+char operator ""_a(const char32_t *, size_t);
+char operator ""_a(const char [32], size_t);
 
 // Unacceptable parameter declarations
-char operator "" _b(); // expected-error {{parameter}}
-char operator "" _b(const wchar_t *); // expected-error {{parameter}}
-char operator "" _b(long long); // expected-error {{parameter}}
-char operator "" _b(double); // expected-error {{parameter}}
-char operator "" _b(short); // expected-error {{parameter}}
-char operator "" _a(char, int = 0); // expected-error {{parameter}}
-char operator "" _b(unsigned short); // expected-error {{parameter}}
-char operator "" _b(signed char); // expected-error {{parameter}}
-char operator "" _b(unsigned char); // expected-error {{parameter}}
-char operator "" _b(const short *, size_t); // expected-error {{parameter}}
-char operator "" _b(const unsigned short *, size_t); // expected-error {{parameter}}
-char operator "" _b(const signed char *, size_t); // expected-error {{parameter}}
-char operator "" _b(const unsigned char *, size_t); // expected-error {{parameter}}
-char operator "" _a(const volatile char *, size_t); // expected-error {{parameter}}
-char operator "" _a(volatile wchar_t *, size_t); // expected-error {{parameter}}
-char operator "" _a(char16_t *, size_t); // expected-error {{parameter}}
-char operator "" _a(const char32_t *, size_t, bool = false); // expected-error {{parameter}}
-char operator "" _a(const char *, signed long); // expected-error {{parameter}}
-char operator "" _a(const char *, size_t = 0); // expected-error {{default argument}}
+char operator ""_b(); // expected-error {{parameter}}
+char operator ""_b(const wchar_t *); // expected-error {{parameter}}
+char operator ""_b(long long); // expected-error {{parameter}}
+char operator ""_b(double); // expected-error {{parameter}}
+char operator ""_b(short); // expected-error {{parameter}}
+char operator ""_a(char, int = 0); // expected-error {{parameter}}
+char operator ""_b(unsigned short); // expected-error {{parameter}}
+char operator ""_b(signed char); // expected-error {{parameter}}
+char operator ""_b(unsigned char); // expected-error {{parameter}}
+char operator ""_b(const short *, size_t); // expected-error {{parameter}}
+char operator ""_b(const unsigned short *, size_t); // expected-error {{parameter}}
+char operator ""_b(const signed char *, size_t); // expected-error {{parameter}}
+char operator ""_b(const unsigned char *, size_t); // expected-error {{parameter}}
+char operator ""_a(const volatile char *, size_t); // expected-error {{parameter}}
+char operator ""_a(volatile wchar_t *, size_t); // expected-error {{parameter}}
+char operator ""_a(char16_t *, size_t); // expected-error {{parameter}}
+char operator ""_a(const char32_t *, size_t, bool = false); // expected-error {{parameter}}
+char operator ""_a(const char *, signed long); // expected-error {{parameter}}
+char operator ""_a(const char *, size_t = 0); // expected-error {{default argument}}
diff --git a/clang/test/CXX/over/over.oper/over.literal/p5.cpp b/clang/test/CXX/over/over.oper/over.literal/p5.cpp
index bfad5f00cf6c75cf020dcb74649afb629cb488e7..593aa57b76a81cddd0f4d1912400e2dcf1877f26 100644
--- a/clang/test/CXX/over/over.oper/over.literal/p5.cpp
+++ b/clang/test/CXX/over/over.oper/over.literal/p5.cpp
@@ -3,20 +3,20 @@
 using size_t = decltype(sizeof(int));
 template<char...> struct S {};
 
-template<char...> void operator "" _a();
-template<char... C> S<C...> operator "" _a();
+template<char...> void operator ""_a();
+template<char... C> S<C...> operator ""_a();
 
 template<typename T> struct U {
-  friend int operator "" _a(const char *, size_t);
+  friend int operator ""_a(const char *, size_t);
   // FIXME: It's not entirely clear whether this is intended to be legal.
-  friend U operator "" _a(const T *, size_t); // expected-error {{parameter}}
+  friend U operator ""_a(const T *, size_t); // expected-error {{parameter}}
 };
 template<char...> struct V {
-  friend void operator "" _b(); // expected-error {{parameters}}
+  friend void operator ""_b(); // expected-error {{parameters}}
 };
 
-template<char... C, int N = 0> void operator "" _b(); // expected-error {{template}}
-template<char... C> void operator "" _b(int N = 0); // expected-error {{template}}
-template<char, char...> void operator "" _b(); // expected-error {{template}}
-template<typename T> T operator "" _b(const char *); // expected-error {{template}}
-template<typename T> int operator "" _b(const T *, size_t); // expected-error {{template}}
+template<char... C, int N = 0> void operator ""_b(); // expected-error {{template}}
+template<char... C> void operator ""_b(int N = 0); // expected-error {{template}}
+template<char, char...> void operator ""_b(); // expected-error {{template}}
+template<typename T> T operator ""_b(const char *); // expected-error {{template}}
+template<typename T> int operator ""_b(const T *, size_t); // expected-error {{template}}
diff --git a/clang/test/CXX/over/over.oper/over.literal/p6.cpp b/clang/test/CXX/over/over.oper/over.literal/p6.cpp
index 9ecf9ccccb14c6bc191c7038d7441221caed050e..265050e0a00967506fd178c988a8253708a20699 100644
--- a/clang/test/CXX/over/over.oper/over.literal/p6.cpp
+++ b/clang/test/CXX/over/over.oper/over.literal/p6.cpp
@@ -1,15 +1,15 @@
 // RUN: %clang_cc1 -std=c++11 %s -verify
 
 // expected-note@+1 {{extern "C" language linkage specification begins here}}
-extern "C" void operator "" _a(const char *); // expected-error {{must have C++ linkage}}
-extern "C" template<char...> void operator "" _b(); // expected-error {{must have C++ linkage}}
+extern "C" void operator ""_a(const char *); // expected-error {{must have C++ linkage}}
+extern "C" template<char...> void operator ""_b(); // expected-error {{must have C++ linkage}}
 // expected-note@-1 {{extern "C" language linkage specification begins here}}
 
 extern "C" { // expected-note 4 {{extern "C" language linkage specification begins here}}
-  void operator "" _c(const char *); // expected-error {{must have C++ linkage}}
-  template<char...> void operator "" _d(); // expected-error {{must have C++ linkage}}
+  void operator ""_c(const char *); // expected-error {{must have C++ linkage}}
+  template<char...> void operator ""_d(); // expected-error {{must have C++ linkage}}
   namespace N {
-    void operator "" _e(const char *); // expected-error {{must have C++ linkage}}
-    template<char...> void operator "" _f(); // expected-error {{must have C++ linkage}}
+    void operator ""_e(const char *); // expected-error {{must have C++ linkage}}
+    template<char...> void operator ""_f(); // expected-error {{must have C++ linkage}}
   }
 }
diff --git a/clang/test/CXX/over/over.oper/over.literal/p7.cpp b/clang/test/CXX/over/over.oper/over.literal/p7.cpp
index 74e9457bb55c38ce33a1b319ee1455e671684ac2..244f043d4412e73bc40b09aece72265e67923279 100644
--- a/clang/test/CXX/over/over.oper/over.literal/p7.cpp
+++ b/clang/test/CXX/over/over.oper/over.literal/p7.cpp
@@ -1,17 +1,17 @@
 // RUN: %clang_cc1 -std=c++11 %s -verify
 // expected-no-diagnostics
 
-constexpr int operator "" _a(const char *c) {
+constexpr int operator ""_a(const char *c) {
   return c[0];
 }
 
-static_assert(operator "" _a("foo") == 'f', "");
+static_assert(operator ""_a("foo") == 'f', "");
 
 void puts(const char *);
-static inline void operator "" _puts(const char *c) {
+static inline void operator ""_puts(const char *c) {
   puts(c);
 }
 void f() {
-  operator "" _puts("foo");
-  operator "" _puts("bar");
+  operator ""_puts("foo");
+  operator ""_puts("bar");
 }
diff --git a/clang/test/CXX/over/over.oper/over.literal/p8.cpp b/clang/test/CXX/over/over.oper/over.literal/p8.cpp
index 6644bae7e610d1fb08d5a1a9348a1743bc2c0b66..8aa31e1acd6e15724263056d0cb4f09e51fac8d7 100644
--- a/clang/test/CXX/over/over.oper/over.literal/p8.cpp
+++ b/clang/test/CXX/over/over.oper/over.literal/p8.cpp
@@ -5,13 +5,13 @@ namespace std {
   using size_t = decltype(sizeof(int));
 }
 
-void operator "" _km(long double); // ok
-string operator "" _i18n(const char*, std::size_t); // ok
-template<char...> int operator "" \u03C0(); // ok, UCN for lowercase pi // expected-warning {{reserved}}
+void operator ""_km(long double); // ok
+string operator ""_i18n(const char*, std::size_t); // ok
+template<char...> int operator ""\u03C0(); // ok, UCN for lowercase pi // expected-warning {{reserved}}
 float operator ""E(const char *); // expected-error {{invalid suffix on literal}} expected-warning {{reserved}}
 float operator " " B(const char *); // expected-error {{must be '""'}} expected-warning {{reserved}}
-string operator "" 5X(const char *, std::size_t); // expected-error {{expected identifier}}
-double operator "" _miles(double); // expected-error {{parameter}}
+string operator ""5X(const char *, std::size_t); // expected-error {{expected identifier}}
+double operator ""_miles(double); // expected-error {{parameter}}
 template<char...> int operator "" j(const char*); // expected-error {{template}}
 
 float operator ""_E(const char *);
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
index 1bbbd1d3429ddd4e92ad34d51537e7a5ea799245..19793fe826372659d1a462e4b6250d6d97c7e8ae 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
@@ -2,13 +2,13 @@
 
 template <class T> struct eval; // expected-note 3{{template is declared here}}
 
-template <template <class, class...> class TT, class T1, class... Rest>
+template <template <class, class...> class TT, class T1, class... Rest> 
 struct eval<TT<T1, Rest...>> { };
 
-template <class T1> struct A;
-template <class T1, class T2> struct B;
-template <int N> struct C;
-template <class T1, int N> struct D;
+template <class T1> struct A; 
+template <class T1, class T2> struct B; 
+template <int N> struct C; 
+template <class T1, int N> struct D; 
 template <class T1, class T2, int N = 17> struct E;
 
 eval<A<int>> eA;
@@ -17,32 +17,27 @@ eval<C<17>> eC; // expected-error{{implicit instantiation of undefined template
 eval<D<int, 17>> eD; // expected-error{{implicit instantiation of undefined template 'eval<D<int, 17>>'}}
 eval<E<int, float>> eE; // expected-error{{implicit instantiation of undefined template 'eval<E<int, float>>}}
 
-template<
-  template <int ...N> // expected-error{{deduced non-type template argument does not have the same type as the corresponding template parameter ('int' vs 'long')}}
-  class TT // expected-note {{previous template template parameter is here}}
-> struct X0 { };
-
+template<template <int ...N> class TT> struct X0 { }; // expected-note{{previous non-type template parameter with type 'int' is here}}
 template<int I, int J, int ...Rest> struct X0a;
 template<int ...Rest> struct X0b;
-template<int I, long J> struct X0c; // expected-note{{template parameter is declared here}}
+template<int I, long J> struct X0c; // expected-note{{template non-type parameter has a different type 'long' in template argument}}
 
 X0<X0a> inst_x0a;
 X0<X0b> inst_x0b;
-X0<X0c> inst_x0c; // expected-note{{template template argument has different template parameters than its corresponding template template parameter}}
+X0<X0c> inst_x0c; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
 
-template<typename T,
-         template <T ...N> // expected-error{{deduced non-type template argument does not have the same type as the corresponding template parameter ('short' vs 'long')}}
-         class TT // expected-note {{previous template template parameter is here}}
-> struct X1 { };
+template<typename T, 
+         template <T ...N> class TT>  // expected-note{{previous non-type template parameter with type 'short' is here}}
+struct X1 { };
 template<int I, int J, int ...Rest> struct X1a;
 template<long I, long ...Rest> struct X1b;
 template<short I, short J> struct X1c;
-template<short I, long J> struct X1d; // expected-note{{template parameter is declared here}}
+template<short I, long J> struct X1d; // expected-note{{template non-type parameter has a different type 'long' in template argument}}
 
 X1<int, X1a> inst_x1a;
 X1<long, X1b> inst_x1b;
 X1<short, X1c> inst_x1c;
-X1<short, X1d> inst_x1d; // expected-note{{template template argument has different template parameters than its corresponding template template parameter}}
+X1<short, X1d> inst_x1d; // expected-error{{template template argument has different template parameters than its corresponding template template paramete}}
 
 template <int> class X2; // expected-note{{template is declared here}} \
                          // expected-note{{template is declared here}}
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.decl/p4.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.decl/p4.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..70064f867e18e351c27f89e8eff89dd96c1bd635
--- /dev/null
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.decl/p4.cpp
@@ -0,0 +1,175 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+// expected-no-diagnostics
+
+template<typename T>
+concept D = true;
+
+template<typename T>
+struct A {
+  template<typename U, bool V>
+  void f() requires V;
+
+  template<>
+  void f<short, true>();
+
+  template<D U>
+  void g();
+
+  template<typename U, bool V> requires V
+  struct B;
+
+  template<typename U, bool V> requires V
+  struct B<U*, V>;
+
+  template<>
+  struct B<short, true>;
+
+  template<D U>
+  struct C;
+
+  template<D U>
+  struct C<U*>;
+
+  template<typename U, bool V> requires V
+  static int x;
+
+  template<typename U, bool V> requires V
+  static int x<U*, V>;
+
+  template<>
+  int x<short, true>;
+
+  template<D U>
+  static int y;
+
+  template<D U>
+  static int y<U*>;
+};
+
+template<typename T>
+template<typename U, bool V>
+void A<T>::f() requires V { }
+
+template<typename T>
+template<D U>
+void A<T>::g() { }
+
+template<typename T>
+template<typename U, bool V> requires V
+struct A<T>::B { };
+
+template<typename T>
+template<typename U, bool V> requires V
+struct A<T>::B<U*, V> { };
+
+template<typename T>
+template<typename U, bool V> requires V
+struct A<T>::B<U&, V> { };
+
+template<typename T>
+template<D U>
+struct A<T>::C { };
+
+template<typename T>
+template<D U>
+struct A<T>::C<U*> { };
+
+template<typename T>
+template<typename U, bool V> requires V
+int A<T>::x = 0;
+
+template<typename T>
+template<typename U, bool V> requires V
+int A<T>::x<U*, V> = 0;
+
+template<typename T>
+template<typename U, bool V> requires V
+int A<T>::x<U&, V> = 0;
+
+template<typename T>
+template<D U>
+int A<T>::y = 0;
+
+template<typename T>
+template<D U>
+int A<T>::y<U*> = 0;
+
+template<>
+template<typename U, bool V>
+void A<short>::f() requires V;
+
+template<>
+template<>
+void A<short>::f<int, true>();
+
+template<>
+template<>
+void A<void>::f<int, true>();
+
+template<>
+template<D U>
+void A<short>::g();
+
+template<>
+template<typename U, bool V> requires V
+struct A<int>::B;
+
+template<>
+template<>
+struct A<int>::B<int, true>;
+
+template<>
+template<>
+struct A<void>::B<int, true>;
+
+template<>
+template<typename U, bool V> requires V
+struct A<int>::B<U*, V>;
+
+template<>
+template<typename U, bool V> requires V
+struct A<int>::B<U&, V>;
+
+template<>
+template<D U>
+struct A<int>::C;
+
+template<>
+template<D U>
+struct A<int>::C<U*>;
+
+template<>
+template<D U>
+struct A<int>::C<U&>;
+
+template<>
+template<typename U, bool V> requires V
+int A<long>::x;
+
+template<>
+template<>
+int A<long>::x<int, true>;
+
+template<>
+template<>
+int A<void>::x<int, true>;
+
+template<>
+template<typename U, bool V> requires V
+int A<long>::x<U*, V>;
+
+template<>
+template<typename U, bool V> requires V
+int A<long>::x<U&, V>;
+
+template<>
+template<D U>
+int A<long>::y;
+
+template<>
+template<D U>
+int A<long>::y<U*>;
+
+template<>
+template<D U>
+int A<long>::y<U&>;
diff --git a/clang/test/CXX/temp/temp.param/p12.cpp b/clang/test/CXX/temp/temp.param/p12.cpp
index 8317e7f24152ccee645de050c4da872f10112477..7be38790905fa398b60c76025b8422a61fd9523c 100644
--- a/clang/test/CXX/temp/temp.param/p12.cpp
+++ b/clang/test/CXX/temp/temp.param/p12.cpp
@@ -1,40 +1,39 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
-template<typename> struct Y1; // expected-note{{template is declared here}}
+// RUN: %clang_cc1 -fsyntax-only -verify %s 
+template<typename> struct Y1; // expected-note{{too few template parameters in template template argument}}
 template<typename, int> struct Y2;
 
 // C++ [temp.param]p12:
-template<class T1,
+template<class T1, 
          class T2 = int> // expected-note{{previous default template argument defined here}}
   class B3;
 template<class T1, typename T2> class B3;
-template<class T1,
+template<class T1, 
          typename T2 = float> // expected-error{{template parameter redefines default argument}}
   class B3;
 
-template<template<class, int> class,
+template<template<class, int> class, 
          template<class> class = Y1> // expected-note{{previous default template argument defined here}}
   class B3t;
 
 template<template<class, int> class, template<class> class> class B3t;
 
-template<template<class, int> class,
+template<template<class, int> class, 
          template<class> class = Y1> // expected-error{{template parameter redefines default argument}}
   class B3t;
 
-template<int N,
+template<int N, 
          int M = 5> // expected-note{{previous default template argument defined here}}
   class B3n;
 
 template<int N, int M> class B3n;
 
-template<int N,
+template<int N, 
          int M = 7>  // expected-error{{template parameter redefines default argument}}
   class B3n;
 
 // Check validity of default arguments
-template<template<class, int> class =// expected-note {{previous template template parameter is here}}
-           Y1> // expected-error{{too many template arguments for class template 'Y1'}}
-               // expected-note@-1 {{template template argument has different template parameters than its corresponding template template parameter}}
+template<template<class, int> class // expected-note{{previous template template parameter is here}}
+           = Y1> // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
   class C1 {};
 
 C1<> c1; // expected-note{{while checking a default template argument}}
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..87127366eb58a50b3b3291e33a86945dab05eaec
--- /dev/null
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p7.cpp
@@ -0,0 +1,228 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s
+
+namespace Undefined {
+  template<typename T>
+  struct A {
+    template<typename U>
+    static constexpr int f(); // expected-note {{declared here}}
+
+    template<typename U>
+    static const int x; // expected-note {{declared here}}
+
+    template<typename U>
+    static const int x<U*>; // expected-note {{declared here}}
+
+    template<typename U>
+    struct B; // expected-note {{template is declared here}}
+
+    template<typename U>
+    struct B<U*>; // expected-note {{template is declared here}}
+  };
+
+  template<>
+  template<typename U>
+  constexpr int A<short>::f() {
+    return A<long>::f<U>();
+  }
+
+  template<>
+  template<typename U>
+  constexpr int A<short>::x = A<long>::x<U>;
+
+  template<>
+  template<typename U>
+  constexpr int A<short>::x<U*> = A<long>::x<U*>;
+
+  template<>
+  template<typename U>
+  struct A<short>::B<U*> {
+    static constexpr int y = A<long>::B<U*>::y;
+  };
+
+  template<>
+  template<typename U>
+  struct A<short>::B {
+    static constexpr int y = A<long>::B<U>::y;
+  };
+
+  template<>
+  template<typename U>
+  constexpr int A<long>::f() {
+    return 1;
+  }
+
+  template<>
+  template<typename U>
+  constexpr int A<long>::x = 1;
+
+  template<>
+  template<typename U>
+  constexpr int A<long>::x<U*> = 2;
+
+  template<>
+  template<typename U>
+  struct A<long>::B {
+    static constexpr int y = 1;
+  };
+
+  template<>
+  template<typename U>
+  struct A<long>::B<U*> {
+    static constexpr int y = 2;
+  };
+
+  static_assert(A<int>::f<int>() == 0); // expected-error {{static assertion expression is not an integral constant expression}}
+                                        // expected-note@-1 {{undefined function 'f<int>' cannot be used in a constant expression}}
+  static_assert(A<int>::x<int> == 0); // expected-error {{static assertion expression is not an integral constant expression}}
+                                      // expected-note@-1 {{initializer of 'x<int>' is unknown}}
+  static_assert(A<int>::x<int*> == 0); // expected-error {{static assertion expression is not an integral constant expression}}
+                                       // expected-note@-1 {{initializer of 'x<int *>' is unknown}}
+  static_assert(A<int>::B<int>::y == 0); // expected-error {{implicit instantiation of undefined template 'Undefined::A<int>::B<int>'}}
+  static_assert(A<int>::B<int*>::y == 0); // expected-error {{implicit instantiation of undefined template 'Undefined::A<int>::B<int *>'}}
+
+  static_assert(A<short>::f<int>() == 1);
+  static_assert(A<short>::x<int> == 1);
+  static_assert(A<short>::x<int*> == 2);
+  static_assert(A<short>::B<int>::y == 1);
+  static_assert(A<short>::B<int*>::y == 2);
+} // namespace Undefined
+
+namespace Defined {
+  template<typename T>
+  struct A {
+    template<typename U>
+    static constexpr int f() {
+      return 0;
+    };
+
+    template<typename U>
+    static const int x = 0;
+
+    template<typename U>
+    static const int x<U*> = 0;
+
+    template<typename U>
+    struct B {
+      static constexpr int y = 0;
+    };
+
+    template<typename U>
+    struct B<U*> {
+      static constexpr int y = 0;
+    };
+  };
+
+  template<>
+  template<typename U>
+  constexpr int A<short>::f() {
+    return A<long>::f<U>();
+  }
+
+  template<>
+  template<typename U>
+  constexpr int A<short>::x = A<long>::x<U>;
+
+  template<>
+  template<typename U>
+  constexpr int A<short>::x<U*> = A<long>::x<U*>;
+
+  template<>
+  template<typename U>
+  struct A<short>::B<U*> {
+    static constexpr int y = A<long>::B<U*>::y;
+  };
+
+  template<>
+  template<typename U>
+  struct A<short>::B {
+    static constexpr int y = A<long>::B<U>::y;
+  };
+
+  template<>
+  template<typename U>
+  constexpr int A<long>::f() {
+    return 1;
+  }
+
+  template<>
+  template<typename U>
+  constexpr int A<long>::x = 1;
+
+  template<>
+  template<typename U>
+  constexpr int A<long>::x<U*> = 2;
+
+  template<>
+  template<typename U>
+  struct A<long>::B {
+    static constexpr int y = 1;
+  };
+
+  template<>
+  template<typename U>
+  struct A<long>::B<U*> {
+    static constexpr int y = 2;
+  };
+
+  static_assert(A<int>::f<int>() == 0);
+  static_assert(A<int>::x<int> == 0);
+  static_assert(A<int>::x<int*> == 0);
+  static_assert(A<int>::B<int>::y == 0);
+  static_assert(A<int>::B<int*>::y == 0);
+
+  static_assert(A<short>::f<int>() == 1);
+  static_assert(A<short>::x<int> == 1);
+  static_assert(A<short>::x<int*> == 2);
+  static_assert(A<short>::B<int>::y == 1);
+  static_assert(A<short>::B<int*>::y == 2);
+} // namespace Defined
+
+namespace Dependent {
+  template<int I>
+  struct A {
+    template<int J>
+    static constexpr int f();
+
+    template<int J>
+    static const int x;
+
+    template<int J>
+    struct B;
+  };
+
+  template<>
+  template<int J>
+  constexpr int A<0>::f() {
+    return A<1>::f<J>();
+  }
+
+  template<>
+  template<int J>
+  constexpr int A<1>::f() {
+    return J;
+  }
+
+  template<>
+  template<int J>
+  constexpr int A<0>::x = A<1>::x<J>;
+
+  template<>
+  template<int J>
+  constexpr int A<1>::x = J;
+
+  template<>
+  template<int J>
+  struct A<0>::B {
+    static constexpr int y = A<1>::B<J>::y;
+  };
+
+  template<>
+  template<int J>
+  struct A<1>::B {
+    static constexpr int y = J;
+  };
+
+  static_assert(A<0>::f<2>() == 2);
+  static_assert(A<0>::x<2> == 2);
+  static_assert(A<0>::B<2>::y == 2);
+} // namespace Dependent
diff --git a/clang/test/CodeGen/2008-07-22-bitfield-init-after-zero-len-array.c b/clang/test/CodeGen/2008-07-22-bitfield-init-after-zero-len-array.c
index b72d689659e602d43dac340725d3eee8c98c484f..b639734ef5d4b7ae8840d73a6a8dac6aeb0d6313 100644
--- a/clang/test/CodeGen/2008-07-22-bitfield-init-after-zero-len-array.c
+++ b/clang/test/CodeGen/2008-07-22-bitfield-init-after-zero-len-array.c
@@ -8,4 +8,4 @@ struct et7 {
   52, 
 };
 
-// CHECK: @yv7 ={{.*}} global %struct.et7 { [0 x float] zeroinitializer, i8 52 }
+// CHECK: @yv7 ={{.*}} global { [0 x float], i8, [3 x i8] } { [0 x float] zeroinitializer, i8 52, [3 x i8] zeroinitializer }
diff --git a/clang/test/CodeGen/2008-08-07-AlignPadding1.c b/clang/test/CodeGen/2008-08-07-AlignPadding1.c
index b51bcbc0244da8ac6938a0813569a3aacab1c4f1..0a37ce9fb653ac39d41c0668de1070f864a73432 100644
--- a/clang/test/CodeGen/2008-08-07-AlignPadding1.c
+++ b/clang/test/CodeGen/2008-08-07-AlignPadding1.c
@@ -21,9 +21,9 @@ struct gc_generation {
 
 #define GEN_HEAD(n) (&generations[n].head)
 
-// The idea is that there are 6 undefs in this structure initializer to cover
+// The idea is that there are 6 zeroinitializers in this structure initializer to cover
 // the padding between elements.
-// CHECK: @generations ={{.*}} global [3 x %struct.gc_generation] [%struct.gc_generation { %union._gc_head { %struct.anon { ptr @generations, ptr @generations, i64 0 }, [8 x i8] undef }, i32 700, i32 0, [8 x i8] undef }, %struct.gc_generation { %union._gc_head { %struct.anon { ptr getelementptr (i8, ptr @generations, i64 48), ptr getelementptr (i8, ptr @generations, i64 48), i64 0 }, [8 x i8] undef }, i32 10, i32 0, [8 x i8] undef }, %struct.gc_generation { %union._gc_head { %struct.anon { ptr getelementptr (i8, ptr @generations, i64 96), ptr getelementptr (i8, ptr @generations, i64 96), i64 0 }, [8 x i8] undef }, i32 10, i32 0, [8 x i8] undef }]
+// CHECK: @generations ={{.*}} global [3 x %struct.gc_generation] [%struct.gc_generation { %union._gc_head { %struct.anon { ptr @generations, ptr @generations, i64 0 }, [8 x i8] zeroinitializer }, i32 700, i32 0, [8 x i8] zeroinitializer }, %struct.gc_generation { %union._gc_head { %struct.anon { ptr getelementptr (i8, ptr @generations, i64 48), ptr getelementptr (i8, ptr @generations, i64 48), i64 0 }, [8 x i8] zeroinitializer }, i32 10, i32 0, [8 x i8] zeroinitializer }, %struct.gc_generation { %union._gc_head { %struct.anon { ptr getelementptr (i8, ptr @generations, i64 96), ptr getelementptr (i8, ptr @generations, i64 96), i64 0 }, [8 x i8] zeroinitializer }, i32 10, i32 0, [8 x i8] zeroinitializer }]
 /* linked lists of container objects */
 struct gc_generation generations[3] = {
         /* PyGC_Head,                           threshold,      count */
diff --git a/clang/test/CodeGen/2009-06-14-anonymous-union-init.c b/clang/test/CodeGen/2009-06-14-anonymous-union-init.c
index 13f6357f7966d93c23419a1232893a43c1bce67f..a4375d7868f01d909a0c4b1af153343a123adc09 100644
--- a/clang/test/CodeGen/2009-06-14-anonymous-union-init.c
+++ b/clang/test/CodeGen/2009-06-14-anonymous-union-init.c
@@ -7,7 +7,7 @@ struct sysfs_dirent {
 };
 struct sysfs_dirent sysfs_root = { {}, 16877 };
 
-// CHECK: @sysfs_root = {{.*}}global %struct.sysfs_dirent { %union.anon zeroinitializer, i16 16877 }
+// CHECK: @sysfs_root = {{.*}}global { %union.anon, i16, [2 x i8] } { %union.anon zeroinitializer, i16 16877, [2 x i8] zeroinitializer }
 
 struct Foo {
  union { struct empty {} x; };
@@ -16,4 +16,4 @@ struct Foo {
 struct Foo foo = { {}, 16877 };
 
 // EMPTY:      @foo = {{.*}}global %struct.Foo { i16 16877 }
-// EMPTY-MSVC: @foo = {{.*}}global %struct.Foo { [4 x i8] undef, i16 16877 }
+// EMPTY-MSVC: @foo = {{.*}}global %struct.Foo { [4 x i8] zeroinitializer, i16 16877 }
diff --git a/clang/test/CodeGen/64bit-swiftcall.c b/clang/test/CodeGen/64bit-swiftcall.c
index 7af65ccf556a81343d41f8f5aff537f00f0c61f7..7f8aa02d97ce1f474f0d042e6d00bc8b26154ee3 100644
--- a/clang/test/CodeGen/64bit-swiftcall.c
+++ b/clang/test/CodeGen/64bit-swiftcall.c
@@ -14,8 +14,6 @@
 
 // CHECK-DAG: %struct.atomic_padded = type { { %struct.packed, [7 x i8] } }
 // CHECK-DAG: %struct.packed = type <{ i64, i8 }>
-//
-// CHECK: [[STRUCT2_RESULT:@.*]] = private {{.*}} constant [[STRUCT2_TYPE:%.*]] { i32 0, i8 0, i8 undef, i8 0, i32 0, i32 0 }
 
 /*****************************************************************************/
 /****************************** PARAMETER ABIS *******************************/
@@ -162,8 +160,8 @@ typedef struct {
 } struct_2;
 TEST(struct_2);
 // CHECK-LABEL: define{{.*}} swiftcc { i64, i64 } @return_struct_2() {{.*}}{
-// CHECK:   [[RET:%.*]] = alloca [[STRUCT2_TYPE]], align 4
-// CHECK:   call void @llvm.memcpy{{.*}}({{.*}}[[RET]], {{.*}}[[STRUCT2_RESULT]]
+// CHECK:   [[RET:%.*]] = alloca [[STRUCT2:%.*]], align 4
+// CHECK:   call void @llvm.memset
 // CHECK:   [[GEP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RET]], i32 0, i32 0
 // CHECK:   [[T0:%.*]] = load i64, ptr [[GEP0]], align 4
 // CHECK:   [[GEP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RET]], i32 0, i32 1
@@ -173,7 +171,7 @@ TEST(struct_2);
 // CHECK:   ret { i64, i64 } [[R1]]
 // CHECK: }
 // CHECK-LABEL: define{{.*}} swiftcc void @take_struct_2(i64 %0, i64 %1) {{.*}}{
-// CHECK:   [[V:%.*]] = alloca [[STRUCT:%.*]], align 4
+// CHECK:   [[V:%.*]] = alloca [[STRUCT2]], align 4
 // CHECK:   [[GEP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[V]], i32 0, i32 0
 // CHECK:   store i64 %0, ptr [[GEP0]], align 4
 // CHECK:   [[GEP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[V]], i32 0, i32 1
@@ -181,7 +179,7 @@ TEST(struct_2);
 // CHECK:   ret void
 // CHECK: }
 // CHECK-LABEL: define{{.*}} void @test_struct_2() {{.*}} {
-// CHECK:   [[TMP:%.*]] = alloca [[STRUCT2_TYPE]], align 4
+// CHECK:   [[TMP:%.*]] = alloca [[STRUCT2]], align 4
 // CHECK:   [[CALL:%.*]] = call swiftcc { i64, i64 } @return_struct_2()
 // CHECK:   [[GEP:%.*]] = getelementptr inbounds nuw {{.*}} [[TMP]], i32 0, i32 0
 // CHECK:   [[T0:%.*]] = extractvalue { i64, i64 } [[CALL]], 0
@@ -254,7 +252,7 @@ typedef union {
 TEST(union_het_fp)
 // CHECK-LABEL: define{{.*}} swiftcc i64 @return_union_het_fp()
 // CHECK:  [[RET:%.*]] = alloca [[UNION:%.*]], align 8
-// CHECK:  call void @llvm.memcpy{{.*}}(ptr align 8 [[RET]]
+// CHECK:  call void @llvm.memset{{.*}}(ptr align 8 [[RET]]
 // CHECK:  [[GEP:%.*]] = getelementptr inbounds nuw { i64 }, ptr [[RET]], i32 0, i32 0
 // CHECK:  [[R0:%.*]] = load i64, ptr [[GEP]], align 8
 // CHECK:  ret i64 [[R0]]
diff --git a/clang/test/CodeGen/RISCV/riscv-inline-asm-rvv.c b/clang/test/CodeGen/RISCV/riscv-inline-asm-rvv.c
index 59adc5c45521abb933a11bc939f060de712eb70c..879fb1238d83abdbb373ac937b3b6251ad83af0e 100644
--- a/clang/test/CodeGen/RISCV/riscv-inline-asm-rvv.c
+++ b/clang/test/CodeGen/RISCV/riscv-inline-asm-rvv.c
@@ -29,6 +29,14 @@ vint32m1_t test_vr(vint32m1_t a, vint32m1_t b) {
   return ret;
 }
 
+vint32m1_t test_vd(vint32m1_t a, vint32m1_t b) {
+// CHECK-LABEL: define{{.*}} @test_vd
+// CHECK: %0 = tail call <vscale x 2 x i32> asm sideeffect "vadd.vv $0, $1, $2", "=^vd,^vd,^vd"(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b)
+  vint32m1_t ret;
+  asm volatile ("vadd.vv %0, %1, %2" : "=vd"(ret) : "vd"(a), "vd"(b));
+  return ret;
+}
+
 vbool1_t test_vm(vbool1_t a, vbool1_t b) {
 // CHECK-LABEL: define{{.*}} @test_vm
 // CHECK: %0 = tail call <vscale x 64 x i1> asm sideeffect "vmand.mm $0, $1, $2", "=^vm,^vm,^vm"(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b)
diff --git a/clang/test/CodeGen/arm-swiftcall.c b/clang/test/CodeGen/arm-swiftcall.c
index ec0e3867909a8697f3e382b649ec870f40adfccb..677b878c6765da3ac25d27c27a8b2904a6f165ae 100644
--- a/clang/test/CodeGen/arm-swiftcall.c
+++ b/clang/test/CodeGen/arm-swiftcall.c
@@ -172,7 +172,7 @@ typedef struct {
 TEST(struct_2);
 // CHECK-LABEL: define{{.*}} @return_struct_2()
 // CHECK:   [[RET:%.*]] = alloca [[REC:%.*]], align 4
-// CHECK:   @llvm.memcpy
+// CHECK:   @llvm.memset
 // CHECK:   [[T0:%.*]] = getelementptr inbounds nuw [[AGG:{ i32, i32, float, float }]], ptr [[RET]], i32 0, i32 0
 // CHECK:   [[FIRST:%.*]] = load i32, ptr [[T0]], align 4
 // CHECK:   [[T0:%.*]] = getelementptr inbounds nuw [[AGG]], ptr [[RET]], i32 0, i32 1
@@ -274,7 +274,7 @@ typedef union {
 TEST(union_het_fp)
 // CHECK-LABEL: define{{.*}} @return_union_het_fp()
 // CHECK:   [[RET:%.*]] = alloca [[REC:%.*]], align {{(4|8)}}
-// CHECK:   @llvm.memcpy
+// CHECK:   @llvm.memset
 // CHECK:   [[T0:%.*]] = getelementptr inbounds nuw [[AGG:{ i32, i32 }]], ptr [[RET]], i32 0, i32 0
 // CHECK:   [[FIRST:%.*]] = load i32, ptr [[T0]], align {{(4|8)}}
 // CHECK:   [[T0:%.*]] = getelementptr inbounds nuw [[AGG]], ptr [[RET]], i32 0, i32 1
diff --git a/clang/test/CodeGen/builtins.c b/clang/test/CodeGen/builtins.c
index 6383a3c65e3750ac1a01973e503ddff88e8da95f..eda6c67fdad00c1614120112de2fcaf7fcb90a4a 100644
--- a/clang/test/CodeGen/builtins.c
+++ b/clang/test/CodeGen/builtins.c
@@ -69,6 +69,13 @@ int main(void) {
   P(issignaling, (1.));
   P(isfpclass, (1., 1));
 
+  Q(fmaximum_num, (1.0, 2.0));
+  Q(fmaximum_numf, (1.0, 2.0));
+  Q(fmaximum_numl, (1.0, 2.0));
+  Q(fminimum_num, (1.0, 2.0));
+  Q(fminimum_numf, (1.0, 2.0));
+  Q(fminimum_numl, (1.0, 2.0));
+
   // Bitwise & Numeric Functions
 
   P(abs, (N));
@@ -305,7 +312,7 @@ void test_float_builtins(__fp16 *H, float F, double D, long double LD) {
 }
 
 // CHECK-LABEL: define{{.*}} void @test_float_builtin_ops
-void test_float_builtin_ops(float F, double D, long double LD) {
+void test_float_builtin_ops(float F, double D, long double LD, int I) {
   volatile float resf;
   volatile double resd;
   volatile long double resld;
@@ -353,6 +360,58 @@ void test_float_builtin_ops(float F, double D, long double LD) {
   resld = __builtin_fmaxl(LD, LD);
   // CHECK: call x86_fp80 @llvm.maxnum.f80
 
+  resf = __builtin_fminimum_numf(F, F);
+  // CHECK: call float @llvm.minimumnum.f32
+
+  resf = __builtin_fminimum_numf(I, I);
+  // CHECK: sitofp i32 {{%[0-9]+}} to float
+  // CHECK: sitofp i32 {{%[0-9]+}} to float
+  // CHECK: call float @llvm.minimumnum.f32
+
+  resf = __builtin_fminimum_numf(1.0, 2.0);
+  // CHECK: store volatile float 1.000000e+00, ptr %resf
+
+  resd = __builtin_fminimum_num(D, D);
+  // CHECK: call double @llvm.minimumnum.f64
+
+  resd = __builtin_fminimum_num(I, I);
+  // CHECK: sitofp i32 {{%[0-9]+}} to double
+  // CHECK: sitofp i32 {{%[0-9]+}} to double
+  // CHECK: call double @llvm.minimumnum.f64
+
+  resd = __builtin_fminimum_num(1.0, 2.0);
+  // CHECK: store volatile double 1.000000e+00, ptr %resd
+
+  //FIXME: __builtin_fminimum_numl is not supported well yet.
+  resld = __builtin_fminimum_numl(1.0, 2.0);
+  // CHECK: store volatile x86_fp80 0xK3FFF8000000000000000, ptr %resld, align 16
+
+  resf = __builtin_fmaximum_numf(F, F);
+  // CHECK: call float @llvm.maximumnum.f32
+
+  resf = __builtin_fmaximum_numf(I, I);
+  // CHECK: sitofp i32 {{%[0-9]+}} to float
+  // CHECK: sitofp i32 {{%[0-9]+}} to float
+  // CHECK: call float @llvm.maximumnum.f32
+
+  resf = __builtin_fmaximum_numf(1.0, 2.0);
+  // CHECK: store volatile float 2.000000e+00, ptr %resf
+
+  resd = __builtin_fmaximum_num(D, D);
+  // CHECK: call double @llvm.maximumnum.f64
+
+  resd = __builtin_fmaximum_num(I, I);
+  // CHECK: sitofp i32 {{%[0-9]+}} to double
+  // CHECK: sitofp i32 {{%[0-9]+}} to double
+  // CHECK: call double @llvm.maximumnum.f64
+
+  resd = __builtin_fmaximum_num(1.0, 2.0);
+  // CHECK: store volatile double 2.000000e+00, ptr %resd
+
+  //FIXME: __builtin_fmaximum_numl is not supported well yet.
+  resld = __builtin_fmaximum_numl(1.0, 2.0);
+  // CHECK: store volatile x86_fp80 0xK40008000000000000000, ptr %resld, align 16
+
   resf = __builtin_fabsf(F);
   // CHECK: call float @llvm.fabs.f32
 
diff --git a/clang/test/CodeGen/const-init.c b/clang/test/CodeGen/const-init.c
index ad3e9551199ac25622463d335ef4964db2b25951..fc973cb983a80a8e67ce01d5d8fe8de350b3a804 100644
--- a/clang/test/CodeGen/const-init.c
+++ b/clang/test/CodeGen/const-init.c
@@ -170,7 +170,7 @@ void g30(void) {
     int : 1;
     int x;
   } a = {};
-  // CHECK: @g30.a = internal global %struct.anon.1 <{ i8 undef, i32 0 }>, align 1
+  // CHECK: @g30.a = internal global %struct.anon.1 zeroinitializer, align 1
 #pragma pack()
 }
 
@@ -182,7 +182,7 @@ void g31(void) {
     short z;
   } a = {23122, -12312731, -312};
 #pragma pack()
-  // CHECK: @g31.a = internal global %struct.anon.2 { i16 23122, i32 -12312731, i16 -312 }, align 4
+  // CHECK: @g31.a = internal global { i16, [2 x i8], i32, i16, [2 x i8] } { i16 23122, [2 x i8] zeroinitializer, i32 -12312731, i16 -312, [2 x i8] zeroinitializer }, align 4
 }
 
 // Clang should evaluate this in constant context, so floating point mode should
diff --git a/clang/test/CodeGen/decl.c b/clang/test/CodeGen/decl.c
index a63846b3223da474c06816fb570259b03aa05197..97446781fdbd2bd31e85086d65ab30b5185c7d54 100644
--- a/clang/test/CodeGen/decl.c
+++ b/clang/test/CodeGen/decl.c
@@ -2,10 +2,10 @@
 
 // CHECK: @test1.x = internal constant [12 x i32] [i32 1
 // CHECK: @__const.test2.x = private unnamed_addr constant [13 x i32] [i32 1,
-// CHECK: @test5w = {{(dso_local )?}}global { i32, [4 x i8] } { i32 2, [4 x i8] undef }
+// CHECK: @test5w = {{(dso_local )?}}global { i32, [4 x i8] } { i32 2, [4 x i8] zeroinitializer }
 // CHECK: @test5y = {{(dso_local )?}}global { double } { double 7.300000e+0{{[0]*}}1 }
 
-// CHECK: @__const.test6.x = private unnamed_addr constant %struct.SelectDest { i8 1, i8 2, i32 3, i32 0 }
+// CHECK: @__const.test6.x = private unnamed_addr constant { i8, i8, [2 x i8], i32, i32 } { i8 1, i8 2, [2 x i8] zeroinitializer, i32 3, i32 0 }
 
 // CHECK: @test7 = {{(dso_local )?}}global [2 x %struct.test7s] [%struct.test7s { i32 1, i32 2 }, %struct.test7s { i32 4, i32 0 }]
 
diff --git a/clang/test/CodeGen/designated-initializers.c b/clang/test/CodeGen/designated-initializers.c
index 620b1b90d257584b1bbbbf1060c16cd03091d671..ac7860db43be7775d9a3eb4098dfac1322c79dbc 100644
--- a/clang/test/CodeGen/designated-initializers.c
+++ b/clang/test/CodeGen/designated-initializers.c
@@ -8,7 +8,7 @@ struct foo {
 // CHECK: @u ={{.*}} global %union.anon zeroinitializer
 union { int i; float f; } u = { };
 
-// CHECK: @u2 ={{.*}} global { i32, [4 x i8] } { i32 0, [4 x i8] undef }
+// CHECK: @u2 ={{.*}} global { i32, [4 x i8] } zeroinitializer
 union { int i; double f; } u2 = { };
 
 // CHECK: @u3 ={{.*}} global  %union.anon.1 zeroinitializer
@@ -62,22 +62,22 @@ struct overwrite_string_struct2 {
   char L[6];
   int M;
 } overwrite_string2[] = { { { "foo" }, 1 }, [0].L[2] = 'x'};
-// CHECK: [6 x i8] c"fox\00\00\00", i32 1
+// CHECK: [6 x i8] c"fox\00\00\00", [2 x i8] zeroinitializer, i32 1
 struct overwrite_string_struct3 {
   char L[3];
   int M;
 } overwrite_string3[] = { { { "foo" }, 1 }, [0].L[2] = 'x'};
-// CHECK: [3 x i8] c"fox", i32 1
+// CHECK: [3 x i8] c"fox", i8 0, i32 1
 struct overwrite_string_struct4 {
   char L[3];
   int M;
 } overwrite_string4[] = { { { "foobar" }, 1 }, [0].L[2] = 'x'};
-// CHECK: [3 x i8] c"fox", i32 1
+// CHECK: [3 x i8] c"fox", i8 0, i32 1
 struct overwrite_string_struct5 {
   char L[6];
   int M;
 } overwrite_string5[] = { { { "foo" }, 1 }, [0].L[4] = 'y'};
-// CHECK: [6 x i8] c"foo\00y\00", i32 1
+// CHECK: [6 x i8] c"foo\00y\00", [2 x i8] zeroinitializer, i32 1
 
 
 // CHECK: @u1 = {{.*}} { i32 65535 }
@@ -138,7 +138,7 @@ union_16644_t union_16644_instance_4[2] =
   [1].b[1] = 4
 };
 
-// CHECK: @lab ={{.*}} global { [4 x i8], i32 } { [4 x i8] undef, i32 123 }
+// CHECK: @lab ={{.*}} global { [4 x i8], i32 } { [4 x i8] zeroinitializer, i32 123 }
 struct leading_anon_bitfield { int : 32; int n; } lab = { .n = 123 };
 
 struct Base {
diff --git a/clang/test/CodeGen/ext-int.c b/clang/test/CodeGen/ext-int.c
index e3d609a4ba4a2e4208a51ae4d4221d7f079e9347..aebacd6f22ffc4a524d6b4e11a54b7afc12937c3 100644
--- a/clang/test/CodeGen/ext-int.c
+++ b/clang/test/CodeGen/ext-int.c
@@ -16,7 +16,7 @@
 unsigned _BitInt(1) GlobSize1 = 0;
 // CHECK: @GlobSize1 = {{.*}}global i8 0
 
-// CHECK64: @__const.foo.A = private unnamed_addr constant { i32, [4 x i8], <{ i8, [23 x i8] }> } { i32 1, [4 x i8] undef, <{ i8, [23 x i8] }> <{ i8 -86, [23 x i8] zeroinitializer }> }, align 8
+// CHECK64: @__const.foo.A = private unnamed_addr constant { i32, [4 x i8], <{ i8, [23 x i8] }> } { i32 1, [4 x i8] zeroinitializer, <{ i8, [23 x i8] }> <{ i8 -86, [23 x i8] zeroinitializer }> }, align 8
 // @BigGlob = global [40 x i8] c"\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF\FF", align 8
 // CHECK64: @f.p = internal global <{ i8, i8, [22 x i8] }> <{ i8 16, i8 39, [22 x i8] zeroinitializer }>, align 8
 
@@ -91,8 +91,8 @@ int foo(int a) {
   // CHECK64: %B2 = getelementptr inbounds nuw %struct.S1, ptr %B, i32 0, i32 2
   // WIN32: %B2 = getelementptr inbounds nuw %struct.S1, ptr %B, i32 0, i32 2
   // LIN32: %B2 = getelementptr inbounds nuw %struct.S1, ptr %B, i32 0, i32 1
-  // CHECK: %0 = load i32, ptr %a.addr, align 4
-  // CHECK: %conv = sext i32 %0 to i129
+  // CHECK: %[[V1:.+]] = load i32, ptr %a.addr, align 4
+  // CHECK: %conv = sext i32 %[[V1]] to i129
   // CHECK64: storedv = sext i129 %conv to i192
   // WIN32: storedv = sext i129 %conv to i192
   // LIN32: storedv = sext i129 %conv to i160
@@ -102,12 +102,12 @@ int foo(int a) {
   // CHECK64: %B3 = getelementptr inbounds nuw %struct.S1, ptr %A, i32 0, i32 2
   // WIN32: %B3 = getelementptr inbounds nuw %struct.S1, ptr %A, i32 0, i32 2
   // LIN32: %B3 = getelementptr inbounds nuw %struct.S1, ptr %A, i32 0, i32 1
-  // CHECK64: %1 = load i192, ptr %B3, align 8
-  // WIN32: %1 = load i192, ptr %B3, align 8
-  // LIN32: %1 = load i160, ptr %B3, align 4
-  // CHECK64: %loadedv = trunc i192 %1 to i129
-  // WIN32: %loadedv = trunc i192 %1 to i129
-  // LIN32: %loadedv = trunc i160 %1 to i129
+  // CHECK64: %[[V2:.+]] = load i192, ptr %B3, align 8
+  // WIN32: %[[V2:.+]] = load i192, ptr %B3, align 8
+  // LIN32: %[[V2:.+]] = load i160, ptr %B3, align 4
+  // CHECK64: %loadedv = trunc i192 %[[V2]] to i129
+  // WIN32: %loadedv = trunc i192 %[[V2]] to i129
+  // LIN32: %loadedv = trunc i160 %[[V2]] to i129
   // CHECK: %conv4 = trunc i129 %loadedv to i32
   struct S1 A = {1, 170};
   struct S1 B = {1, a};
diff --git a/clang/test/CodeGen/flexible-array-init.c b/clang/test/CodeGen/flexible-array-init.c
index 15a30c15ac966effae52a0716a5ca4f8e2ccb5b1..17b520fe83094298b0075e555b61d96b2efe6b19 100644
--- a/clang/test/CodeGen/flexible-array-init.c
+++ b/clang/test/CodeGen/flexible-array-init.c
@@ -14,11 +14,11 @@ struct { int y[]; } b1 = { { 14, 16 } };
 
 // sizeof(c) == 8, so this global should be at least 8 bytes.
 struct { int x; char c; char y[]; } c = { 1, 2, { 13, 15 } };
-// CHECK: @c ={{.*}} global { i32, i8, [2 x i8] } { i32 1, i8 2, [2 x i8] c"\0D\0F" }
+// CHECK: @c ={{.*}} global { i32, i8, [2 x i8], i8 } { i32 1, i8 2, [2 x i8] c"\0D\0F", i8 0 }
 
 // sizeof(d) == 8, so this global should be at least 8 bytes.
 struct __attribute((packed, aligned(4))) { char a; int x; char z[]; } d = { 1, 2, { 13, 15 } };
-// CHECK: @d ={{.*}} <{ i8, i32, [2 x i8], i8 }> <{ i8 1, i32 2, [2 x i8] c"\0D\0F", i8 undef }>,
+// CHECK: @d ={{.*}} <{ i8, i32, [2 x i8], i8 }> <{ i8 1, i32 2, [2 x i8] c"\0D\0F", i8 0 }>,
 
 // This global needs 9 bytes to hold all the flexible array members.
 struct __attribute((packed, aligned(4))) { char a; int x; char z[]; } e = { 1, 2, { 13, 15, 17, 19 } };
@@ -55,21 +55,21 @@ struct { int a; union { int b; short x[]; }; int c; int d; } hf = {1, 2, {}, 3};
 
 // First member is the potential flexible array, initialization requires braces.
 struct { int a; union { short x; int b; }; int c; int d; } i = {1, 2, {}, 3};
-// CHECK: @i = global { i32, { i16, [2 x i8] }, i32, i32 } { i32 1, { i16, [2 x i8] } { i16 2, [2 x i8] undef }, i32 0, i32 3 }
+// CHECK: @i = global { i32, { i16, [2 x i8] }, i32, i32 } { i32 1, { i16, [2 x i8] } { i16 2, [2 x i8] zeroinitializer }, i32 0, i32 3 }
 struct { int a; union { short x[0]; int b; }; int c; int d; } i0 = {1, {}, 2, 3};
-// CHECK: @i0 = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 1, { [0 x i16], [4 x i8] } { [0 x i16] zeroinitializer, [4 x i8] undef }, i32 2, i32 3 }
+// CHECK: @i0 = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 1, { [0 x i16], [4 x i8] } zeroinitializer, i32 2, i32 3 }
 struct { int a; union { short x[1]; int b; }; int c; int d; } i1 = {1, {2}, {}, 3};
-// CHECK: @i1 = global { i32, { [1 x i16], [2 x i8] }, i32, i32 } { i32 1, { [1 x i16], [2 x i8] } { [1 x i16] [i16 2], [2 x i8] undef }, i32 0, i32 3 }
+// CHECK: @i1 = global { i32, { [1 x i16], [2 x i8] }, i32, i32 } { i32 1, { [1 x i16], [2 x i8] } { [1 x i16] [i16 2], [2 x i8] zeroinitializer }, i32 0, i32 3 }
 struct { int a; union { short x[]; int b; }; int c; int d; } i_f = {4, {}, {}, 6};
-// CHECK: @i_f = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 4, { [0 x i16], [4 x i8] } { [0 x i16] zeroinitializer, [4 x i8] undef }, i32 0, i32 6 }
+// CHECK: @i_f = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 4, { [0 x i16], [4 x i8] } zeroinitializer, i32 0, i32 6 }
 
 // Named initializers; order doesn't matter.
 struct { int a; union { int b; short x; }; int c; int d; } hn = {.a = 1, .x = 2, .c = 3};
-// CHECK: @hn = global { i32, { i16, [2 x i8] }, i32, i32 } { i32 1, { i16, [2 x i8] } { i16 2, [2 x i8] undef }, i32 3, i32 0 }
+// CHECK: @hn = global { i32, { i16, [2 x i8] }, i32, i32 } { i32 1, { i16, [2 x i8] } { i16 2, [2 x i8] zeroinitializer }, i32 3, i32 0 }
 struct { int a; union { int b; short x[0]; }; int c; int d; } hn0 = {.a = 1, .x = {2}, .c = 3};
-// CHECK: @hn0 = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 1, { [0 x i16], [4 x i8] } { [0 x i16] zeroinitializer, [4 x i8] undef }, i32 3, i32 0 }
+// CHECK: @hn0 = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 1, { [0 x i16], [4 x i8] } zeroinitializer, i32 3, i32 0 }
 struct { int a; union { int b; short x[1]; }; int c; int d; } hn1 = {.a = 1, .x = {2}, .c = 3};
-// CHECK: @hn1 = global { i32, { [1 x i16], [2 x i8] }, i32, i32 } { i32 1, { [1 x i16], [2 x i8] } { [1 x i16] [i16 2], [2 x i8] undef }, i32 3, i32 0 }
+// CHECK: @hn1 = global { i32, { [1 x i16], [2 x i8] }, i32, i32 } { i32 1, { [1 x i16], [2 x i8] } { [1 x i16] [i16 2], [2 x i8] zeroinitializer }, i32 3, i32 0 }
 
 struct { char a[]; } empty_struct = {};
 // CHECK: @empty_struct ={{.*}} global %struct.anon{{.*}} zeroinitializer, align 1
@@ -96,10 +96,10 @@ union { char a[]; } only_in_union0 = {0};
 // CHECK: @only_in_union0 = global { [1 x i8] } zeroinitializer, align 1
 
 union { char a[]; int b; } first_in_union = {};
-// CHECK: @first_in_union = global { [0 x i8], [4 x i8] } { [0 x i8] zeroinitializer, [4 x i8] undef }, align 4
+// CHECK: @first_in_union = global { [0 x i8], [4 x i8] } zeroinitializer, align 4
 
 union { char a[]; int b; } first_in_union0 = {0};
-// CHECK: @first_in_union0 = global { [1 x i8], [3 x i8] } { [1 x i8] zeroinitializer, [3 x i8] undef }, align 4
+// CHECK: @first_in_union0 = global { [1 x i8], [3 x i8] } zeroinitializer, align 4
 
 union { char a[]; int b; } first_in_union123 = { {1, 2, 3} };
-// CHECK: @first_in_union123 = global { [3 x i8], i8 } { [3 x i8] c"\01\02\03", i8 undef }, align 4
+// CHECK: @first_in_union123 = global { [3 x i8], i8 } { [3 x i8] c"\01\02\03", i8 0 }, align 4
diff --git a/clang/test/CodeGen/fmaxfmin-invalid-arguments-type.c b/clang/test/CodeGen/fmaxfmin-invalid-arguments-type.c
new file mode 100644
index 0000000000000000000000000000000000000000..3da41b40b4069780ccad31e1a50ed0e36d3e2b1b
--- /dev/null
+++ b/clang/test/CodeGen/fmaxfmin-invalid-arguments-type.c
@@ -0,0 +1,33 @@
+// RUN: not %clang_cc1 -triple x86_64 %s -fsyntax-only -verify 2>&1 | FileCheck %s --check-prefix=CHECK-ERR
+
+float fminimum_numf (float, float);
+double fminimum_num (double, double);
+long double fminimum_numl (long double, long double);
+float fmaximum_numf (float, float);
+double fmaximum_num (double, double);
+long double fmaximum_numl (long double, long double);
+
+// CHECK-ERR: passing 'char *' to parameter of incompatible type 'float'
+float fmin1(char *a, char *b) {
+        return fminimum_numf(a, b);
+}
+// CHECK-ERR: passing 'char *' to parameter of incompatible type 'double'
+float fmin2(char *a, char *b) {
+        return fminimum_num(a, b);
+}
+// CHECK-ERR: passing 'char *' to parameter of incompatible type 'long double'
+float fmin3(char *a, char *b) {
+        return fminimum_numl(a, b);
+}
+// CHECK-ERR: passing 'char *' to parameter of incompatible type 'float'
+float fmax1(char *a, char *b) {
+        return fmaximum_numf(a, b);
+}
+// CHECK-ERR: passing 'char *' to parameter of incompatible type 'double'
+float fmax2(char *a, char *b) {
+        return fmaximum_num(a, b);
+}
+// CHECK-ERR: passing 'char *' to parameter of incompatible type 'long double'
+float fmax3(char *a, char *b) {
+        return fmaximum_numl(a, b);
+}
diff --git a/clang/test/CodeGen/global-init.c b/clang/test/CodeGen/global-init.c
index 7f1d675b97c09e52660849a432d6b563ca28ab5d..b156466dbaaffc56b57997414450a4acabbf75c2 100644
--- a/clang/test/CodeGen/global-init.c
+++ b/clang/test/CodeGen/global-init.c
@@ -33,7 +33,7 @@ struct ManyFields {
   int f;
 };
 
-// CHECK: global %struct.ManyFields { i32 1, i32 2, i32 0, i8 0, i32 0, i32 0 }
+// CHECK: global { i32, i32, i32, i8, [3 x i8], i32, i32 } { i32 1, i32 2, i32 0, i8 0, [3 x i8] zeroinitializer, i32 0, i32 0 }
 struct ManyFields FewInits = {1, 2};
 
 
diff --git a/clang/test/CodeGen/init.c b/clang/test/CodeGen/init.c
index cbf615bb9ddfea111ee9fab3aa5d51f6fa0b815d..27f427dff8f79ef1987ec1304fc52ba0bc9a348b 100644
--- a/clang/test/CodeGen/init.c
+++ b/clang/test/CodeGen/init.c
@@ -187,25 +187,6 @@ void nonzeroMemsetf64(void) {
   // CHECK: call void @llvm.memset.p0.i32(ptr {{.*}}, i8 68, i32 56, i1 false)
 }
 
-void nonzeroPaddedUnionMemset(void) {
-  union U { char c; int i; };
-  union U arr[9] = { 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, 0xF0, };
-  // CHECK-LABEL: @nonzeroPaddedUnionMemset(
-  // CHECK-NOT: store
-  // CHECK-NOT: memcpy
-  // CHECK: call void @llvm.memset.p0.i32(ptr {{.*}}, i8 -16, i32 36, i1 false)
-}
-
-void nonzeroNestedMemset(void) {
-  union U { char c; int i; };
-  struct S { union U u; short i; };
-  struct S arr[5] = { { {0xF0}, 0xF0F0 }, { {0xF0}, 0xF0F0 }, { {0xF0}, 0xF0F0 }, { {0xF0}, 0xF0F0 }, { {0xF0}, 0xF0F0 }, };
-  // CHECK-LABEL: @nonzeroNestedMemset(
-  // CHECK-NOT: store
-  // CHECK-NOT: memcpy
-  // CHECK: call void @llvm.memset.p0.i32(ptr {{.*}}, i8 -16, i32 40, i1 false)
-}
-
 // PR9257
 struct test11S {
   int A[10];
diff --git a/clang/test/CodeGen/linux-kernel-struct-union-initializer.c b/clang/test/CodeGen/linux-kernel-struct-union-initializer.c
new file mode 100644
index 0000000000000000000000000000000000000000..8c0eb3702cf0b598c88a7ea1a32767b113f9749e
--- /dev/null
+++ b/clang/test/CodeGen/linux-kernel-struct-union-initializer.c
@@ -0,0 +1,275 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --version 5
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=gnu11 -verify -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+union U1 {
+  int x;
+  char y[16];
+};
+
+struct S1 {
+  int x;
+  union U1 y;
+};
+
+union U2 {
+  int x;
+  char y[16];
+} __attribute__((__aligned__(32)));
+
+struct S2 {
+  int x;
+  long long y;
+  char z[8];
+} __attribute__((__aligned__(32)));
+
+struct S3 {
+  char x;
+  unsigned char y : 4;
+  unsigned char z : 7;
+} __attribute__((packed));
+
+union U1 global_u1 = {};
+
+union U1 global_u2 = {3};
+
+union U1 global_u2_from_cast = (union U1)3;
+
+struct S1 global_s1 = {};
+
+struct S1 global_s2 = {
+    .x = 3,
+};
+
+struct S1 global_s3 = {.x = 3, .y = {.x = 6}};
+
+const union U1 global_const_u1 = {4};
+struct S1 global_s3_from_const_u1 = {.y = global_const_u1};
+
+union U2 global_u3 = {};
+
+struct S2 global_s4 = {};
+
+struct S2 global_s5 = {.x = 1};
+
+struct S3 global_s6 = {101,  15, 123};
+
+// Test empty initializer for union.
+//.
+// CHECK: @global_u1 = global %union.U1 zeroinitializer, align 4
+// CHECK: @global_u2 = global %union.U1 { i32 3, [12 x i8] zeroinitializer }, align 4
+// CHECK: @global_u2_from_cast = global { i32, [12 x i8] } { i32 3, [12 x i8] zeroinitializer }, align 4
+// CHECK: @global_s1 = global %struct.S1 zeroinitializer, align 4
+// CHECK: @global_s2 = global %struct.S1 { i32 3, %union.U1 zeroinitializer }, align 4
+// CHECK: @global_s3 = global %struct.S1 { i32 3, %union.U1 { i32 6, [12 x i8] zeroinitializer } }, align 4
+// CHECK: @global_const_u1 = constant %union.U1 { i32 4, [12 x i8] zeroinitializer }, align 4
+// CHECK: @global_s3_from_const_u1 = global %struct.S1 { i32 0, %union.U1 { i32 4, [12 x i8] zeroinitializer } }, align 4
+// CHECK: @global_u3 = global %union.U2 zeroinitializer, align 32
+// CHECK: @global_s4 = global { i32, [4 x i8], i64, [8 x i8], [8 x i8] } zeroinitializer, align 32
+// CHECK: @global_s5 = global { i32, [4 x i8], i64, [8 x i8], [8 x i8] } { i32 1, [4 x i8] zeroinitializer, i64 0, [8 x i8] zeroinitializer, [8 x i8] zeroinitializer }, align 32
+// CHECK: @global_s6 = global { i8, i8, i8 } { i8 101, i8 -65, i8 7 }, align 1
+// CHECK: @test2.a = internal global %union.U1 zeroinitializer, align 4
+// CHECK: @__const.test3.a = private unnamed_addr constant %union.U1 { i32 3, [12 x i8] zeroinitializer }, align 4
+// CHECK: @test4.a = internal global %union.U1 { i32 3, [12 x i8] zeroinitializer }, align 4
+// CHECK: @test6.s = internal global %struct.S1 zeroinitializer, align 4
+// CHECK: @__const.test7.s = private unnamed_addr constant %struct.S1 { i32 3, %union.U1 zeroinitializer }, align 4
+// CHECK: @test8.s = internal global %struct.S1 { i32 3, %union.U1 zeroinitializer }, align 4
+// CHECK: @__const.test9.s = private unnamed_addr constant %struct.S1 { i32 3, %union.U1 { i32 6, [12 x i8] zeroinitializer } }, align 4
+// CHECK: @test10.s = internal global %struct.S1 { i32 3, %union.U1 { i32 6, [12 x i8] zeroinitializer } }, align 4
+// CHECK: @test12.a = internal global %union.U2 zeroinitializer, align 32
+// CHECK: @test14.s = internal global { i32, [4 x i8], i64, [8 x i8], [8 x i8] } zeroinitializer, align 32
+// CHECK: @__const.test15.s = private unnamed_addr constant { i32, [4 x i8], i64, [8 x i8], [8 x i8] } { i32 1, [4 x i8] zeroinitializer, i64 0, [8 x i8] zeroinitializer, [8 x i8] zeroinitializer }, align 32
+// CHECK: @test16.s = internal global { i32, [4 x i8], i64, [8 x i8], [8 x i8] } { i32 1, [4 x i8] zeroinitializer, i64 0, [8 x i8] zeroinitializer, [8 x i8] zeroinitializer }, align 32
+//.
+// CHECK-LABEL: define dso_local void @test1(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A:%.*]] = alloca [[UNION_U1:%.*]], align 4
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[A]], i8 0, i64 16, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test1() {
+  union U1 a = {};
+}
+
+// Test empty initializer for union. Use static variable.
+// CHECK-LABEL: define dso_local void @test2(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test2() {
+  static union U1 a = {};
+}
+
+// Test only initializing a small field for union.
+// CHECK-LABEL: define dso_local void @test3(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A:%.*]] = alloca [[UNION_U1:%.*]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[A]], ptr align 4 @__const.test3.a, i64 16, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test3() {
+  union U1 a = {3};
+}
+
+// Test only initializing a small field for union. Use static variable.
+// CHECK-LABEL: define dso_local void @test4(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test4() {
+  static union U1 a = {3};
+}
+
+// Test union in struct. Use empty initializer for the struct.
+// CHECK-LABEL: define dso_local void @test5(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S1:%.*]], align 4
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[S]], i8 0, i64 20, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test5() {
+  struct S1 s = {};
+}
+
+// Test union in struct. Use empty initializer for the struct. Use static variable.
+// CHECK-LABEL: define dso_local void @test6(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test6() {
+  static struct S1 s = {};
+}
+
+// Test union in struct. Initialize other fields of the struct.
+// CHECK-LABEL: define dso_local void @test7(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S1:%.*]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[S]], ptr align 4 @__const.test7.s, i64 20, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test7() {
+  struct S1 s = {
+      .x = 3,
+  };
+}
+
+// Test union in struct. Initialize other fields of the struct. Use static variable.
+// CHECK-LABEL: define dso_local void @test8(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test8() {
+  static struct S1 s = {
+      .x = 3,
+  };
+}
+
+// Test union in struct. Initialize a small field for union.
+// CHECK-LABEL: define dso_local void @test9(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S1:%.*]], align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[S]], ptr align 4 @__const.test9.s, i64 20, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test9() {
+  struct S1 s = {.x = 3,
+                .y = {
+                    .x = 6,
+                }};
+}
+
+// Test union in struct. Initialize a small field for union. Use static variable.
+// CHECK-LABEL: define dso_local void @test10(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test10() {
+  static struct S1 s = {.x = 3,
+                       .y = {
+                           .x = 6,
+                       }};
+}
+
+// Test empty initializer for union with padding.
+// CHECK-LABEL: define dso_local void @test11(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A:%.*]] = alloca [[UNION_U2:%.*]], align 32
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 32 [[A]], i8 0, i64 32, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test11() {
+  union U2 a = {};
+}
+
+// Test empty initializer for union with padding. Use static variable.
+// CHECK-LABEL: define dso_local void @test12(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test12() {
+  static union U2 a = {};
+}
+
+// Test empty initializer for struct with padding.
+// CHECK-LABEL: define dso_local void @test13(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S2:%.*]], align 32
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 32 [[S]], i8 0, i64 32, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test13() {
+  struct S2 s = {};
+}
+
+// Test empty initializer for struct with padding. Use static variable.
+// CHECK-LABEL: define dso_local void @test14(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test14() {
+  static struct S2 s = {};
+}
+
+// Test partial initialization for struct with padding.
+// CHECK-LABEL: define dso_local void @test15(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S2:%.*]], align 32
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 32 [[S]], ptr align 32 @__const.test15.s, i64 32, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test15() {
+  struct S2 s = {.x = 1};
+}
+
+// Test partial initialization for struct with padding. Use static variable.
+// CHECK-LABEL: define dso_local void @test16(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+//
+void test16() {
+  static struct S2 s = {.x = 1};
+}
+//.
+// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: write) }
+// CHECK: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
+//.
+// CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
diff --git a/clang/test/CodeGen/linux-kernel-struct-union-initializer2.c b/clang/test/CodeGen/linux-kernel-struct-union-initializer2.c
new file mode 100644
index 0000000000000000000000000000000000000000..df53d194c6aad77cbf3d5d4e68d2f4a7221d7c85
--- /dev/null
+++ b/clang/test/CodeGen/linux-kernel-struct-union-initializer2.c
@@ -0,0 +1,181 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --version 5
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=gnu11 -verify -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+union U1 {
+  int x;
+  char y[5];
+};
+
+struct S1 {
+  int x;
+  long long y;
+};
+
+struct S2 {
+  unsigned char b1 : 3; // 1st 3 bits (in 1st byte) are b1
+  unsigned char    : 2; // next 2 bits (in 1st byte) are blocked out as unused
+  unsigned char b2 : 6; // 6 bits for b2 - doesn't fit into the 1st byte => starts a 2nd
+  unsigned char b3 : 2; // 2 bits for b3 - next (and final) bits in the 2nd byte
+  int i;
+};
+
+struct S3 {
+  int x;
+} __attribute__((__aligned__(8)));
+
+struct S4 {
+  int a;
+  union U1 b;
+};
+
+struct S5 {
+  char x;
+  unsigned char y : 4;
+  unsigned char z : 7;
+} __attribute__((packed));
+
+// Test non-const initializer for union with padding.
+// CHECK-LABEL: define dso_local void @test1(
+// CHECK-SAME: i32 noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[A:%.*]] = alloca [[UNION_U1:%.*]], align 4
+// CHECK-NEXT:    store i32 [[X]], ptr [[X_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 4
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[TMP1]], i8 0, i64 4, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test1(int x) {
+  union U1 a = {x};
+}
+
+// Test non-const initializer for struct with padding.
+// CHECK-LABEL: define dso_local void @test2(
+// CHECK-SAME: i64 noundef [[Y:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[Y_ADDR:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S1:%.*]], align 8
+// CHECK-NEXT:    store i64 [[Y]], ptr [[Y_ADDR]], align 8
+// CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_S1]], ptr [[S]], i32 0, i32 0
+// CHECK-NEXT:    store i32 0, ptr [[X]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[S]], i64 4
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[TMP0]], i8 0, i64 4, i1 false)
+// CHECK-NEXT:    [[Y1:%.*]] = getelementptr inbounds nuw [[STRUCT_S1]], ptr [[S]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[Y_ADDR]], align 8
+// CHECK-NEXT:    store i64 [[TMP1]], ptr [[Y1]], align 8
+// CHECK-NEXT:    ret void
+//
+void test2(long long y) {
+  struct S1 s = {.y = y};
+}
+
+// Test non-const initializer for struct with padding and bit fields.
+// CHECK-LABEL: define dso_local void @test3(
+// CHECK-SAME: i8 noundef zeroext [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S2:%.*]], align 4
+// CHECK-NEXT:    store i8 [[B]], ptr [[B_ADDR]], align 1
+// CHECK-NEXT:    store i16 0, ptr [[S]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i16
+// CHECK-NEXT:    [[BF_LOAD:%.*]] = load i16, ptr [[S]], align 4
+// CHECK-NEXT:    [[BF_VALUE:%.*]] = and i16 [[TMP1]], 7
+// CHECK-NEXT:    [[BF_CLEAR:%.*]] = and i16 [[BF_LOAD]], -8
+// CHECK-NEXT:    [[BF_SET:%.*]] = or i16 [[BF_CLEAR]], [[BF_VALUE]]
+// CHECK-NEXT:    store i16 [[BF_SET]], ptr [[S]], align 4
+// CHECK-NEXT:    [[BF_LOAD1:%.*]] = load i16, ptr [[S]], align 4
+// CHECK-NEXT:    [[BF_CLEAR2:%.*]] = and i16 [[BF_LOAD1]], -16129
+// CHECK-NEXT:    [[BF_SET3:%.*]] = or i16 [[BF_CLEAR2]], 0
+// CHECK-NEXT:    store i16 [[BF_SET3]], ptr [[S]], align 4
+// CHECK-NEXT:    [[BF_LOAD4:%.*]] = load i16, ptr [[S]], align 4
+// CHECK-NEXT:    [[BF_CLEAR5:%.*]] = and i16 [[BF_LOAD4]], 16383
+// CHECK-NEXT:    [[BF_SET6:%.*]] = or i16 [[BF_CLEAR5]], 0
+// CHECK-NEXT:    store i16 [[BF_SET6]], ptr [[S]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[S]], i64 2
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 2 [[TMP2]], i8 0, i64 2, i1 false)
+// CHECK-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[S]], i32 0, i32 1
+// CHECK-NEXT:    store i32 0, ptr [[I]], align 4
+// CHECK-NEXT:    ret void
+//
+void test3(unsigned char b) {
+  struct S2 s = {.b1 = b};
+}
+
+// Test non-const initializer for struct with padding at the end of the struct.
+// CHECK-LABEL: define dso_local void @test4(
+// CHECK-SAME: i32 noundef [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S3:%.*]], align 8
+// CHECK-NEXT:    store i32 [[X]], ptr [[X_ADDR]], align 4
+// CHECK-NEXT:    [[X1:%.*]] = getelementptr inbounds nuw [[STRUCT_S3]], ptr [[S]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[X1]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[S]], i64 4
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[TMP1]], i8 0, i64 4, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test4(int x) {
+  struct S3 s = {x};
+}
+
+// Test non-const initializer for union in struct.
+// CHECK-LABEL: define dso_local void @test5(
+// CHECK-SAME: i32 noundef [[A:%.*]], i32 noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S4:%.*]], align 4
+// CHECK-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[B]], ptr [[B_ADDR]], align 4
+// CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_S4]], ptr [[S]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A1]], align 4
+// CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds nuw [[STRUCT_S4]], ptr [[S]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
+// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B2]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[B2]], i64 4
+// CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[TMP2]], i8 0, i64 4, i1 false)
+// CHECK-NEXT:    ret void
+//
+void test5(int a, int b) {
+  struct S4 s = {a, {b}};
+}
+
+// CHECK-LABEL: define dso_local void @test6(
+// CHECK-SAME: i8 noundef signext [[X:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S5:%.*]], align 1
+// CHECK-NEXT:    store i8 [[X]], ptr [[X_ADDR]], align 1
+// CHECK-NEXT:    [[X1:%.*]] = getelementptr inbounds nuw [[STRUCT_S5]], ptr [[S]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[X_ADDR]], align 1
+// CHECK-NEXT:    store i8 [[TMP0]], ptr [[X1]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[S]], i64 1
+// CHECK-NEXT:    store i16 0, ptr [[TMP1]], align 1
+// CHECK-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S5]], ptr [[S]], i32 0, i32 1
+// CHECK-NEXT:    [[BF_LOAD:%.*]] = load i16, ptr [[Y]], align 1
+// CHECK-NEXT:    [[BF_CLEAR:%.*]] = and i16 [[BF_LOAD]], -16
+// CHECK-NEXT:    [[BF_SET:%.*]] = or i16 [[BF_CLEAR]], 0
+// CHECK-NEXT:    store i16 [[BF_SET]], ptr [[Y]], align 1
+// CHECK-NEXT:    [[Z:%.*]] = getelementptr inbounds nuw [[STRUCT_S5]], ptr [[S]], i32 0, i32 1
+// CHECK-NEXT:    [[BF_LOAD2:%.*]] = load i16, ptr [[Z]], align 1
+// CHECK-NEXT:    [[BF_CLEAR3:%.*]] = and i16 [[BF_LOAD2]], -2033
+// CHECK-NEXT:    [[BF_SET4:%.*]] = or i16 [[BF_CLEAR3]], 0
+// CHECK-NEXT:    store i16 [[BF_SET4]], ptr [[Z]], align 1
+// CHECK-NEXT:    ret void
+//
+void test6(char x) {
+  struct S5 s = {.x = x};
+}
+//.
+// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+// CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: write) }
+//.
+// CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
diff --git a/clang/test/CodeGen/math-libcalls.c b/clang/test/CodeGen/math-libcalls.c
index 5b23a4a3faef36fb0efce077131db6c1638c540a..2226212eca94ee81902574c602c1e142be3e10f5 100644
--- a/clang/test/CodeGen/math-libcalls.c
+++ b/clang/test/CodeGen/math-libcalls.c
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm %s | FileCheck %s --check-prefix=NO__ERRNO
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -fmath-errno %s | FileCheck %s --check-prefix=HAS_ERRNO
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -disable-llvm-passes -O2 %s | FileCheck %s --check-prefix=NO__ERRNO
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -disable-llvm-passes -O2 -fmath-errno %s | FileCheck %s --check-prefix=HAS_ERRNO
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -ffp-exception-behavior=maytrap %s | FileCheck %s --check-prefix=HAS_MAYTRAP
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm %s | FileCheck %s --check-prefixes=COMMON,NO__ERRNO
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -fmath-errno %s | FileCheck %s --check-prefixes=COMMON,HAS_ERRNO
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -disable-llvm-passes -O2 %s | FileCheck %s --check-prefixes=COMMON,NO__ERRNO
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -disable-llvm-passes -O2 -fmath-errno %s | FileCheck %s --check-prefixes=COMMON,HAS_ERRNO
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -Wno-implicit-function-declaration -w -o - -emit-llvm -ffp-exception-behavior=maytrap %s | FileCheck %s --check-prefixes=COMMON,HAS_MAYTRAP
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown-gnu -Wno-implicit-function-declaration -w -o - -emit-llvm -fmath-errno %s | FileCheck %s --check-prefix=HAS_ERRNO_GNU
 // RUN: %clang_cc1 -triple x86_64-unknown-windows-msvc -Wno-implicit-function-declaration -w -o - -emit-llvm -fmath-errno %s | FileCheck %s --check-prefix=HAS_ERRNO_WIN
 
@@ -372,6 +372,18 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 // HAS_MAYTRAP: declare float @llvm.experimental.constrained.minnum.f32(
 // HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.minnum.f80(
 
+  fmaximum_num(*d,*d);       fmaximum_numf(f,f);      fmaximum_numl(*l,*l);
+
+// COMMON: declare double @llvm.maximumnum.f64(double, double) [[READNONE_INTRINSIC]]
+// COMMON: declare float @llvm.maximumnum.f32(float, float) [[READNONE_INTRINSIC]]
+// COMMON: declare x86_fp80 @llvm.maximumnum.f80(x86_fp80, x86_fp80) [[READNONE_INTRINSIC]]
+
+  fminimum_num(*d,*d);       fminimum_numf(f,f);      fminimum_numl(*l,*l);
+
+// COMMON: declare double @llvm.minimumnum.f64(double, double) [[READNONE_INTRINSIC]]
+// COMMON: declare float @llvm.minimumnum.f32(float, float) [[READNONE_INTRINSIC]]
+// COMMON: declare x86_fp80 @llvm.minimumnum.f80(x86_fp80, x86_fp80) [[READNONE_INTRINSIC]]
+
   hypot(f,f);      hypotf(f,f);     hypotl(f,f);
 
 // NO__ERRNO: declare double @hypot(double noundef, double noundef) [[READNONE]]
diff --git a/clang/test/CodeGen/memtag-globals.cpp b/clang/test/CodeGen/memtag-globals.cpp
index b4f5dc0d7dcf041ec4e6931b84985e2da26cf5f9..ae2d32ae8a56d9179d03bafcb09daef80455c10f 100644
--- a/clang/test/CodeGen/memtag-globals.cpp
+++ b/clang/test/CodeGen/memtag-globals.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -include %S/Inputs/sanitizer-extra-source.cpp \
+// RUN: %clang_cc1 -triple aarch64-linux-android34 \
+// RUN:   -include %S/Inputs/sanitizer-extra-source.cpp \
 // RUN:   -fsanitize-ignorelist=%S/Inputs/sanitizer-ignorelist-global.txt \
 // RUN:   -fsanitize=memtag-globals -emit-llvm -o - %s | FileCheck %s
 
-// RUN: %clang_cc1 -include %S/Inputs/sanitizer-extra-source.cpp \
+// RUN: %clang_cc1 -triple aarch64-linux-android34 \
+// RUN:    -include %S/Inputs/sanitizer-extra-source.cpp \
 // RUN:   -fsanitize-ignorelist=%S/Inputs/sanitizer-ignorelist-src.txt \
 // RUN:   -fsanitize=memtag-globals -emit-llvm -o - %s | \
 // RUN:   FileCheck %s --check-prefix=IGNORELIST
diff --git a/clang/test/CodeGen/mingw-long-double.c b/clang/test/CodeGen/mingw-long-double.c
index 4be97526f963193d50d11e9a965aa2c2636acb20..0fc8f015096827d2cfb39dce889d43bdd76340cc 100644
--- a/clang/test/CodeGen/mingw-long-double.c
+++ b/clang/test/CodeGen/mingw-long-double.c
@@ -11,12 +11,9 @@ struct {
   char c;
   long double ldb;
 } agggregate_LD = {};
-// GNU32: %struct.anon = type { i8, x86_fp80 }
-// GNU32: @agggregate_LD = dso_local global %struct.anon zeroinitializer, align 4
-// GNU64: %struct.anon = type { i8, x86_fp80 }
-// GNU64: @agggregate_LD = dso_local global %struct.anon zeroinitializer, align 16
-// MSC64: %struct.anon = type { i8, double }
-// MSC64: @agggregate_LD = dso_local global %struct.anon zeroinitializer, align 8
+// GNU32: @agggregate_LD = dso_local global { i8, [3 x i8], x86_fp80 } zeroinitializer, align 4
+// GNU64: @agggregate_LD = dso_local global { i8, [15 x i8], x86_fp80 } zeroinitializer, align 16
+// MSC64: @agggregate_LD = dso_local global { i8, [7 x i8], double } zeroinitializer, align 8
 
 long double dataLD = 1.0L;
 // GNU32: @dataLD = dso_local global x86_fp80 0xK3FFF8000000000000000, align 4
diff --git a/clang/test/CodeGen/mms-bitfields.c b/clang/test/CodeGen/mms-bitfields.c
index 49c5c1c3e7d40d84b3f982eed155f05ac0e45d37..2ccce326c7131d8666a70fceea94d00ed4b4b06b 100644
--- a/clang/test/CodeGen/mms-bitfields.c
+++ b/clang/test/CodeGen/mms-bitfields.c
@@ -61,5 +61,5 @@ union HEADER {
 struct Inner variable = { 1,0,1, 21 };
 union HEADER hdr = {{1,2,3,4}};
 
-// CHECK: @variable ={{.*}} global { i8, [3 x i8], i8, i8, i8, i8 } { i8 5, [3 x i8] undef, i8 21, i8 0, i8 0, i8 0 }, align 1
-// CHECK: @hdr ={{.*}} global { { i8, i8, [2 x i8], i8, i8, i8, i8, i8, [3 x i8] } } { { i8, i8, [2 x i8], i8, i8, i8, i8, i8, [3 x i8] } { i8 8, i8 0, [2 x i8] undef, i8 2, i8 0, i8 0, i8 3, i8 4, [3 x i8] undef } }, align 1
+// CHECK: @variable ={{.*}} global { i8, [3 x i8], i8, i8, i8, i8 } { i8 5, [3 x i8] zeroinitializer, i8 21, i8 0, i8 0, i8 0 }, align 1
+// CHECK: @hdr ={{.*}} global { { i8, i8, [2 x i8], i8, i8, i8, i8, i8, [3 x i8] } } { { i8, i8, [2 x i8], i8, i8, i8, i8, i8, [3 x i8] } { i8 8, i8 0, [2 x i8] zeroinitializer, i8 2, i8 0, i8 0, i8 3, i8 4, [3 x i8] zeroinitializer } }, align 1
diff --git a/clang/test/CodeGen/union-init2.c b/clang/test/CodeGen/union-init2.c
index 048ff00517b4e865576bee50749dfd136221ab53..ee35e78a4f30100da2977c96eae7a7d2046a1c97 100644
--- a/clang/test/CodeGen/union-init2.c
+++ b/clang/test/CodeGen/union-init2.c
@@ -2,11 +2,11 @@
 // RUN: %clang_cc1 -x c++ %s -emit-llvm -triple x86_64-linux-gnu -o - | FileCheck %s --check-prefixes=CHECK-CXX
 
 // Make sure we generate something sane instead of a ptrtoint
-// CHECK: @r, [4 x i8] undef
+// CHECK: @r, [4 x i8] zeroinitializer
 union x {long long b;union x* a;} r = {.a = &r};
 
 
-// CHECK: global { [3 x i8], [5 x i8] } { [3 x i8] zeroinitializer, [5 x i8] undef }
+// CHECK: global { [3 x i8], [5 x i8] } zeroinitializer
 union z {
   char a[3];
   long long b;
diff --git a/clang/test/CodeGen/windows-swiftcall.c b/clang/test/CodeGen/windows-swiftcall.c
index bc7832d9d3ac28319259f5784d63e7535f4e1553..41569c2606622fbe30271677d3eeea29499f2c64 100644
--- a/clang/test/CodeGen/windows-swiftcall.c
+++ b/clang/test/CodeGen/windows-swiftcall.c
@@ -5,8 +5,6 @@
 #define ERROR __attribute__((swift_error_result))
 #define CONTEXT __attribute__((swift_context))
 
-// CHECK: [[STRUCT2_RESULT:@.*]] = private {{.*}} constant [[STRUCT2_TYPE:%.*]] { i32 0, i8 0, i8 undef, i8 0, i32 0, i32 0 }
-
 /*****************************************************************************/
 /****************************** PARAMETER ABIS *******************************/
 /*****************************************************************************/
@@ -142,8 +140,8 @@ typedef struct {
 } struct_2;
 TEST(struct_2);
 // CHECK-LABEL: define dso_local swiftcc { i64, i64 } @return_struct_2() {{.*}}{
-// CHECK:   [[RET:%.*]] = alloca [[STRUCT2_TYPE]], align 4
-// CHECK:   call void @llvm.memcpy{{.*}}({{.*}}[[RET]], {{.*}}[[STRUCT2_RESULT]]
+// CHECK:   [[RET:%.*]] = alloca [[STRUCT2:%.*]], align 4
+// CHECK:   call void @llvm.memset
 // CHECK:   [[GEP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RET]], i32 0, i32 0
 // CHECK:   [[T0:%.*]] = load i64, ptr [[GEP0]], align 4
 // CHECK:   [[GEP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RET]], i32 0, i32 1
@@ -153,7 +151,7 @@ TEST(struct_2);
 // CHECK:   ret { i64, i64 } [[R1]]
 // CHECK: }
 // CHECK-LABEL: define dso_local swiftcc void @take_struct_2(i64 %0, i64 %1) {{.*}}{
-// CHECK:   [[V:%.*]] = alloca [[STRUCT:%.*]], align 4
+// CHECK:   [[V:%.*]] = alloca [[STRUCT2]], align 4
 // CHECK:   [[GEP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[V]], i32 0, i32 0
 // CHECK:   store i64 %0, ptr [[GEP0]], align 4
 // CHECK:   [[GEP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[V]], i32 0, i32 1
@@ -161,7 +159,7 @@ TEST(struct_2);
 // CHECK:   ret void
 // CHECK: }
 // CHECK-LABEL: define dso_local void @test_struct_2() {{.*}} {
-// CHECK:   [[TMP:%.*]] = alloca [[STRUCT2_TYPE]], align 4
+// CHECK:   [[TMP:%.*]] = alloca [[STRUCT2]], align 4
 // CHECK:   [[CALL:%.*]] = call swiftcc { i64, i64 } @return_struct_2()
 // CHECK:   [[GEP:%.*]] = getelementptr inbounds nuw {{.*}} [[TMP]], i32 0, i32 0
 // CHECK:   [[T0:%.*]] = extractvalue { i64, i64 } [[CALL]], 0
@@ -234,7 +232,7 @@ typedef union {
 TEST(union_het_fp)
 // CHECK-LABEL: define dso_local swiftcc i64 @return_union_het_fp()
 // CHECK:  [[RET:%.*]] = alloca [[UNION:%.*]], align 8
-// CHECK:  call void @llvm.memcpy{{.*}}(ptr align {{[0-9]+}} [[RET]]
+// CHECK:  call void @llvm.memset{{.*}}(ptr align {{[0-9]+}} [[RET]]
 // CHECK:  [[GEP:%.*]] = getelementptr inbounds nuw { i64 }, ptr [[RET]], i32 0, i32 0
 // CHECK:  [[R0:%.*]] = load i64, ptr [[GEP]], align 8
 // CHECK:  ret i64 [[R0]]
diff --git a/clang/test/CodeGenCXX/ext-int.cpp b/clang/test/CodeGenCXX/ext-int.cpp
index 7758e92d74a05fe6b9c83d76947cdd8deb43bd20..a098f5a9057c0e0f2fd67ebae6f7d665abe56048 100644
--- a/clang/test/CodeGenCXX/ext-int.cpp
+++ b/clang/test/CodeGenCXX/ext-int.cpp
@@ -614,3 +614,18 @@ void TBAATest(_BitInt(sizeof(int) * 8) ExtInt,
 // NewStructPathTBAA-DAG: ![[EXTINT_TBAA_ROOT]] = !{![[CHAR_TBAA_ROOT]], i64 4, !"_BitInt(32)"}
 // NewStructPathTBAA-DAG: ![[EXTINT6_TBAA]] = !{![[EXTINT6_TBAA_ROOT:.+]], ![[EXTINT6_TBAA_ROOT]], i64 0, i64 1}
 // NewStructPathTBAA-DAG: ![[EXTINT6_TBAA_ROOT]] = !{![[CHAR_TBAA_ROOT]], i64 1, !"_BitInt(6)"}
+
+namespace A {
+template <int N> struct S {
+  using T = _BitInt(N);
+  T Data;
+};
+template <int N> void foo(S<N> B) {
+  const auto Var = B.Data;
+}
+
+void bar() {
+  S<2080> a;
+  foo(a);
+}
+}
diff --git a/clang/test/CodeGenObjC/designated-initializers.m b/clang/test/CodeGenObjC/designated-initializers.m
index a67f82e1afbea84d8887baa69ecc50f5610e8a66..ce58f6c367338e61d45a948a2a3a40863029bf8f 100644
--- a/clang/test/CodeGenObjC/designated-initializers.m
+++ b/clang/test/CodeGenObjC/designated-initializers.m
@@ -4,4 +4,4 @@ struct overwrite_string_struct {
   char L[3];
   int M;
 } overwrite_string[] = { { { @encode(void**) }, 1 }, [0].L[1] = 'x'};
-// CHECK: [3 x i8] c"^xv", i32 1
+// CHECK: [3 x i8] c"^xv", i8 0, i32 1
diff --git a/clang/test/Driver/Xlinker-args.c b/clang/test/Driver/Xlinker-args.c
index b2a913e7cef6205b90d3571a30505fd25230ad6b..87f98a0cdbd837f9167086503f0a1eb30c7d2ef7 100644
--- a/clang/test/Driver/Xlinker-args.c
+++ b/clang/test/Driver/Xlinker-args.c
@@ -8,7 +8,7 @@
 
 /// -T is reordered to the last to make sure -L takes precedence.
 // RUN: %clang -target x86_64-pc-linux-gnu -### \
-// RUN:   -e _start -T a.lds -Xlinker one -Xlinker --no-demangle \
+// RUN:   -e _start -T a.lds -t -Xlinker one -Xlinker --no-demangle \
 // RUN:   -Wl,two,--no-demangle,three -Xlinker four -z five -r %s 2> %t
 // RUN: FileCheck -check-prefix=LINUX < %t %s
 
@@ -23,7 +23,7 @@
 
 // DARWIN-NOT: --no-demangle
 // DARWIN: "one" "two" "three" "four" "-z" "five" "-r"
-// LINUX: "--no-demangle" "-e" "_start" "one" "two" "three" "four" "-z" "five" "-r" {{.*}} "-T" "a.lds"
+// LINUX: "--no-demangle" "-e" "_start" "one" "two" "three" "four" "-z" "five" "-r" {{.*}} "-T" "a.lds" "-t"
 // MINGW: "--no-demangle"
 // AIX: "-b" "one" "-b" "two"
 
diff --git a/clang/test/Driver/arm-cortex-cpus-2.c b/clang/test/Driver/arm-cortex-cpus-2.c
index 5ce3758655b500ff1cea13c97db1df0ae3c6820f..2f17a0814043612a8f348139f89bc6b14502609c 100644
--- a/clang/test/Driver/arm-cortex-cpus-2.c
+++ b/clang/test/Driver/arm-cortex-cpus-2.c
@@ -559,8 +559,10 @@
 // CHECK-CPUV8MBASE:  "-cc1"{{.*}} "-triple" "thumbv8m.base-
 
 // RUN: %clang -target arm -mcpu=cortex-m33 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-M33 %s
+// RUN: %clang -target arm -mcpu=star-mc1 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-STAR-MC1 %s
 // RUN: %clang -target arm -mcpu=cortex-m35p -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-M35P %s
 // CHECK-CORTEX-M33:  "-cc1"{{.*}} "-triple" "thumbv8m.main-{{.*}} "-target-cpu" "cortex-m33"
+// CHECK-STAR-MC1:  "-cc1"{{.*}} "-triple" "thumbv8m.main-{{.*}} "-target-cpu" "star-mc1"
 // CHECK-CORTEX-M35P:  "-cc1"{{.*}} "-triple" "thumbv8m.main-{{.*}} "-target-cpu" "cortex-m35p"
 
 // RUN: %clang -target arm -mcpu=cortex-m55 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-M55 %s
diff --git a/clang/test/Driver/avr-mmcu.c b/clang/test/Driver/avr-mmcu.c
index 6c13ab5b68d3b9a05b799b8b90346199a98d37d3..0edb05dad136ab0cec01f924a73c7624947f4653 100644
--- a/clang/test/Driver/avr-mmcu.c
+++ b/clang/test/Driver/avr-mmcu.c
@@ -1,8 +1,7 @@
 // A test for the propagation of the -mmcu option to -cc1 and -cc1as
 
-// RUN: %clang -### --target=avr -mmcu=attiny11 -save-temps %s 2>&1 | FileCheck -check-prefix=CHECK0 %s
-// CHECK0: "-cc1" {{.*}} "-target-cpu" "attiny11"
-// CHECK0: "-cc1as" {{.*}} "-target-cpu" "attiny11"
+// RUN: not %clang -### --target=avr -mmcu=attiny11 %s 2>&1 | FileCheck -check-prefix=CHECK0 %s
+// CHECK0: error: '-mmcu=attiny11' invalid for input of type c/c++
 
 // RUN: %clang -### --target=avr -mmcu=at90s2313 -save-temps %s 2>&1 | FileCheck -check-prefix=CHECK1 %s
 // CHECK1: "-cc1" {{.*}} "-target-cpu" "at90s2313"
diff --git a/clang/test/Driver/hip-link-save-temps.hip b/clang/test/Driver/hip-link-save-temps.hip
index 5656614626b9cd1cd2590506e359f0bec08aebac..e321970274bb4b32f2cbb33c7811fb4b29670183 100644
--- a/clang/test/Driver/hip-link-save-temps.hip
+++ b/clang/test/Driver/hip-link-save-temps.hip
@@ -39,10 +39,10 @@
 // CHECK-NOT: {{".*/opt"}}
 // CHECK-NOT: {{".*/llc"}}
 // CHECK: "{{.*lld.*}}" {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
-// CHECK-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx900" "obj1-hip-amdgcn-amd-amdhsa-gfx900.o" "obj2-hip-amdgcn-amd-amdhsa-gfx900.o"
+// CHECK-SAME: "-o" "[[HIPFB1:.+]]" "obj1-hip-amdgcn-amd-amdhsa-gfx900.o" "obj2-hip-amdgcn-amd-amdhsa-gfx900.o"
 // CHECK: "{{.*lld.*}}" {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
-// CHECK-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx906" "obj1-hip-amdgcn-amd-amdhsa-gfx906.o" "obj2-hip-amdgcn-amd-amdhsa-gfx906.o"
-// CHECK: {{".*llvm-mc.*"}} "-o" "[[OBJBUNDLE:.*.o]]" "{{.*}}.mcin" "--filetype=obj"
+// CHECK-SAME: "-o" "[[HIPFB2:.+]]" "obj1-hip-amdgcn-amd-amdhsa-gfx906.o" "obj2-hip-amdgcn-amd-amdhsa-gfx906.o"
+// CHECK: "{{.*clang.*}}" "-target" "x86_64-unknown-linux-gnu" "-o" "[[OBJBUNDLE:.+.o]]" "-x" "assembler" "{{.*}}.mcin" "-c"
 // OUT: "{{.*ld.*}}" {{.*}} "-o" "executable" {{.*}} "[[OBJBUNDLE]]"
 // NOUT: "{{.*ld.*}}" {{.*}} "-o" "a.out" {{.*}} "[[OBJBUNDLE]]"
 // SLO: "{{.*llvm-ar.*}}" "rcsD" "libTest.a" {{.*}} "[[OBJBUNDLE]]"
diff --git a/clang/test/Driver/hip-partial-link.hip b/clang/test/Driver/hip-partial-link.hip
index c8451ec81ed37ef86415f24157f1723c690f5ac8..8b27f78f3bdd124b178a38bcdcf99eab08808372 100644
--- a/clang/test/Driver/hip-partial-link.hip
+++ b/clang/test/Driver/hip-partial-link.hip
@@ -35,7 +35,7 @@
 // LD-R: "{{.*}}/clang-offload-bundler" {{.*}}-unbundle
 // LD-R: "{{.*}}/lld" -flavor gnu -m elf64_amdgpu
 // LD-R: "{{.*}}/clang-offload-bundler"
-// LD-R: "{{.*}}/llvm-mc" -triple x86_64-unknown-linux-gnu
+// LD-R: "{{.*}}/clang{{.*}}" -target x86_64-unknown-linux-gnu
 // LD-R: "{{.*}}/ld.lld" {{.*}} -r
 
 // RUN: llvm-nm  %t.lib.o | FileCheck -check-prefix=OBJ %s
@@ -65,7 +65,7 @@
 // STATIC: "{{.*}}/clang-offload-bundler" {{.*}}-unbundle
 // STATIC: "{{.*}}/lld" -flavor gnu -m elf64_amdgpu
 // STATIC: "{{.*}}/clang-offload-bundler"
-// STATIC: "{{.*}}/llvm-mc" -triple x86_64-unknown-linux-gnu
+// STATIC: "{{.*}}/clang{{.*}}" -target x86_64-unknown-linux-gnu
 // STATIC: "{{.*}}/llvm-ar"
 
 // RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
diff --git a/clang/test/Driver/hip-save-temps.hip b/clang/test/Driver/hip-save-temps.hip
index 6dedfdafb11a13bdb750a8754c55b5bbff89e60e..142c3f1611a360259b011310fbe229524147299f 100644
--- a/clang/test/Driver/hip-save-temps.hip
+++ b/clang/test/Driver/hip-save-temps.hip
@@ -58,7 +58,7 @@
 // RDCC: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.o"
 // RDCC: "{{.*clang-offload-bundler.*}}" {{.*}} "-output=hip-save-temps.o"
 // RDCL: "{{.*clang-offload-bundler.*}}" {{.*}} "-output=hip-save-temps-hip-amdgcn-amd-amdhsa.hipfb"
-// RDCL: {{.*}}llvm-mc{{.*}}"-o" "hip-save-temps-hip-amdgcn-amd-amdhsa.o" "hip-save-temps-hip-amdgcn-amd-amdhsa.mcin" "--filetype=obj"
+// RDCL: "{{.*clang.*}}" "-target" "x86_64-unknown-linux-gnu" "-o" "{{.*}}.o" "-x" "assembler" "{{.*}}.mcin" "-c"
 
 // -fno-gpu-rdc host object path
 // NORDC: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.hipi"
diff --git a/clang/test/Driver/hip-toolchain-rdc-separate.hip b/clang/test/Driver/hip-toolchain-rdc-separate.hip
index 92f493912adb4df1182db3d80d1a5448e9e48d74..0ce5ea5174e1b1f642e0cb0634f560d57278b3ef 100644
--- a/clang/test/Driver/hip-toolchain-rdc-separate.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-separate.hip
@@ -145,8 +145,8 @@
 // LINK-BUNDLE-SAME: "-input={{.*}}" "-input=[[IMG_DEV1]]" "-input=[[IMG_DEV2]]" "-output=[[BUNDLE:.*]]"
 // LINK-NOBUNDLE-NOT: {{".*clang-offload-bundler"}} "-type=o"
 
-// LINK-EMBED: {{".*llvm-mc.*"}} "-o" "[[OBJBUNDLE:.*o]]" "{{.*}}.mcin" "--filetype=obj"
-// LINK-NOEMBED-NOT: {{".*llvm-mc.*"}} "-o"
+// LINK-EMBED: {{".*clang.*"}} "-o" "[[OBJBUNDLE:.*o]]" "{{.*}}.mcin"
+// LINK-NOEMBED-NOT: {{".*clang.*"}} "-o"
 
 // LINK-EMBED: [[LD:".*ld.*"]] {{.*}} "-o" "a.out" {{.*}} "[[A_OBJ_HOST]]"
 // LINK-EMBED-SAME: "[[B_OBJ_HOST]]" "[[OBJBUNDLE]]"
diff --git a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
index 780426907e90e712c3bb3bae7a5a0d463d2ffe28..5276faf31bdc2b065254a94307ed09ebaa0c67dc 100644
--- a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
@@ -86,6 +86,6 @@
 // CHECK-SAME: "-targets={{.*}},hipv4-amdgcn-amd-amdhsa--gfx803,hipv4-amdgcn-amd-amdhsa--gfx900"
 // CHECK-SAME: "-input=[[IMG_DEV1]]" "-input=[[IMG_DEV2]]" "-output=[[BUNDLE:.*hipfb]]"
 
-// CHECK: [[MC:".*llvm-mc.*"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
+// CHECK: [[MC:".*clang.*"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin"
 
 // CHECK: [[AR:".*llvm-ar.*"]] "rcsD" "{{.*}}.out" [[A_OBJ_HOST]] [[B_OBJ_HOST]] [[OBJBUNDLE]]
diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip
index ec79bf06afb92c2a5fd954ed43a16f57fdfe30eb..6d3f46f8a94674e44483682252a15e919d0bec01 100644
--- a/clang/test/Driver/hip-toolchain-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-rdc.hip
@@ -155,7 +155,7 @@
 // CHECK-SAME: "-targets={{.*}},hipv4-amdgcn-amd-amdhsa--gfx803,hipv4-amdgcn-amd-amdhsa--gfx900"
 // CHECK-SAME: "-input={{.*}}" "-input=[[IMG_DEV1]]" "-input=[[IMG_DEV2]]" "-output=[[BUNDLE]]"
 
-// CHECK: [[MC:".*llvm-mc.*"]] "-triple" [[HOST]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
+// CHECK: [[MC:".*clang.*"]] "-target" [[HOST]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin"
 
 // output the executable
 // LNX: [[LD:".*ld.*"]] {{.*}}"-o" "a.out" {{.*}} [[A_OBJ_HOST]] [[B_OBJ_HOST]] [[OBJBUNDLE]]
diff --git a/clang/test/Driver/hip-unbundle-preproc.hipi b/clang/test/Driver/hip-unbundle-preproc.hipi
index da5c68ef1c158d5c2ed6fb62b831d88baff64cc0..a56cafea0e8d4e2accabc634c0f996660a9b3df1 100644
--- a/clang/test/Driver/hip-unbundle-preproc.hipi
+++ b/clang/test/Driver/hip-unbundle-preproc.hipi
@@ -23,5 +23,5 @@
 // RDC: {{".*clang.*"}} "-cc1" {{.*}}"-target-cpu" "gfx803" {{.*}}"-o" "[[DEV_BC:.*bc]]" {{.*}}"[[DEV_PP]]"
 // RDC: {{".*lld.*"}} {{.*}}"-o" "[[DEV_ISA:.*]]" "[[DEV_BC]]"
 // RDC: {{".*clang-offload-bundler.*"}} {{.*}}"-input={{.*}}" "-input=[[DEV_ISA]]" "-output=[[FATBIN:.*]]"
-// RDC: {{".*llvm-mc.*"}} "-o" "[[FATBIN_O:.*o]]"
+// RDC: {{".*clang.*"}} "-o" "[[FATBIN_O:.*o]]"
 // RDC: {{".*ld.*"}} {{.*}}"[[HOST_O]]" "[[FATBIN_O]]"
diff --git a/clang/test/Driver/hipspv-toolchain-rdc.hip b/clang/test/Driver/hipspv-toolchain-rdc.hip
index 2e7528a9996d87be4d776f5b69a9b7f53f6d7f00..acdadacc490649c04fe077a00601baefa5f8b600 100644
--- a/clang/test/Driver/hipspv-toolchain-rdc.hip
+++ b/clang/test/Driver/hipspv-toolchain-rdc.hip
@@ -53,8 +53,7 @@
 // CHECK-SAME: "-targets={{.*}},hip-spirv64----generic"
 // CHECK-SAME: "-input=/dev/null" "-input=[[AB_SPIRV]]"
 // CHECK-SAME: "-output=[[AB_FATBIN:.*hipfb]]"
-// CHECK: {{".*llvm-mc.*"}} "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin"
-// CHECK-SAME: "--filetype=obj"
+// CHECK: {{".*clang.*"}} "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin"
 
 // Output the executable
 // CHECK: {{".*ld.*"}} {{.*}}"-o" "a.out" {{.*}} [[A_OBJ_HOST]] [[B_OBJ_HOST]]
diff --git a/clang/test/Driver/hipstdpar.c b/clang/test/Driver/hipstdpar.c
index 32e040ef70d754bbbf379410864c166f47f3e115..b759c5fb2084a3cd0df372ab7f6d79406ed11afc 100644
--- a/clang/test/Driver/hipstdpar.c
+++ b/clang/test/Driver/hipstdpar.c
@@ -1,21 +1,17 @@
-// REQUIRES: x86-registered-target
-// REQUIRES: amdgpu-registered-target
-// REQUIRES: system-linux
-// UNSUPPORTED: target={{.*}}-zos{{.*}}
-// XFAIL: target={{.*}}hexagon{{.*}}
-// XFAIL: target={{.*}}-scei{{.*}}
-// XFAIL: target={{.*}}-sie{{.*}}
+// REQUIRES: x86-registered-target, amdgpu-registered-target
 
-// RUN: not %clang -### --hipstdpar --hipstdpar-path=/does/not/exist -nogpulib \
+// RUN: not %clang -### --target=x86_64-unknown-linux-gnu \
+// RUN:   --hipstdpar --hipstdpar-path=/does/not/exist -nogpulib    \
 // RUN:   -nogpuinc --compile %s 2>&1 | \
 // RUN:   FileCheck --check-prefix=HIPSTDPAR-MISSING-LIB %s
-// RUN: %clang -### --hipstdpar --hipstdpar-path=%S/Inputs/hipstdpar \
+// RUN: %clang -### --target=x86_64-unknown-linux-gnu \
+// RUN:   --hipstdpar --hipstdpar-path=%S/Inputs/hipstdpar             \
 // RUN:   --hipstdpar-thrust-path=%S/Inputs/hipstdpar/thrust \
 // RUN:   --hipstdpar-prim-path=%S/Inputs/hipstdpar/rocprim \
 // RUN:   -nogpulib -nogpuinc --compile %s 2>&1 | \
 // RUN:   FileCheck --check-prefix=HIPSTDPAR-COMPILE %s
 // RUN: touch %t.o
-// RUN: %clang -### --hipstdpar %t.o 2>&1 | FileCheck --check-prefix=HIPSTDPAR-LINK %s
+// RUN: %clang -### --target=x86_64-unknown-linux-gnu --hipstdpar %t.o 2>&1 | FileCheck --check-prefix=HIPSTDPAR-LINK %s
 
 // HIPSTDPAR-MISSING-LIB: error: cannot find HIP Standard Parallelism Acceleration library; provide it via '--hipstdpar-path'
 // HIPSTDPAR-COMPILE: "-x" "hip"
diff --git a/clang/test/FixIt/fixit-c++11.cpp b/clang/test/FixIt/fixit-c++11.cpp
index 10f4a9d0554ccc2fd3e1969a141be0852362025e..e635c14e0c215578f97feddad515ff13fbc2bf5f 100644
--- a/clang/test/FixIt/fixit-c++11.cpp
+++ b/clang/test/FixIt/fixit-c++11.cpp
@@ -74,8 +74,8 @@ const char *p = "foo"bar; // expected-error {{requires a space between}}
 #define ord - '0'
 int k = '4'ord; // expected-error {{requires a space between}}
 
-void operator"x" _y(char); // expected-error {{must be '""'}}
-void operator L"" _z(char); // expected-error {{encoding prefix}}
+void operator"x"_y(char); // expected-error {{must be '""'}}
+void operator L""_z(char); // expected-error {{encoding prefix}}
 void operator "x" "y" U"z" ""_whoops "z" "y"(char); // expected-error {{must be '""'}}
 
 void f() {
diff --git a/clang/test/Frontend/plugin-attribute.cpp b/clang/test/Frontend/plugin-attribute.cpp
index 1c5a2440b2688829d978d90eb3fb2866628bec59..094ce9f5cbb85f23e07f9705189dca33667879fc 100644
--- a/clang/test/Frontend/plugin-attribute.cpp
+++ b/clang/test/Frontend/plugin-attribute.cpp
@@ -4,19 +4,57 @@
 // REQUIRES: plugins, examples
 //--- good_attr.cpp
 // expected-no-diagnostics
-void fn1a() __attribute__((example)) {}
-[[example]] void fn1b() {}
-[[plugin::example]] void fn1c() {}
-void fn2() __attribute__((example("somestring", 1, 2.0))) {}
-// CHECK-COUNT-4: -AnnotateAttr 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} "example"
+void fn1a() __attribute__((example)) {
+  __attribute__((example)) for (int i = 0; i < 9; ++i) {}
+}
+[[example]] void fn1b() {
+  [[example]] for (int i = 0; i < 9; ++i) {}
+}
+[[plugin::example]] void fn1c() {
+  [[plugin::example]] for (int i = 0; i < 9; ++i) {}
+}
+void fn2() __attribute__((example("somestring", 1, 2.0))) {
+  __attribute__((example("abc", 3, 4.0))) for (int i = 0; i < 9; ++i) {}
+}
+template <int N> void template_fn() __attribute__((example("template", N))) {
+  __attribute__((example("def", N + 1))) for (int i = 0; i < 9; ++i) {}
+}
+void fn3() { template_fn<5>(); }
+// CHECK: -AttributedStmt 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}}
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -AttributedStmt 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}}
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -AttributedStmt 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}}
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -AttributedStmt 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}}
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -StringLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'const char[{{[0-9]+}}]' lvalue "abc"
+// CHECK: -IntegerLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'int' 3
+// CHECK: -FloatingLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'double' 4.000000e+00
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}} "example"
 // CHECK: -StringLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'const char[{{[0-9]+}}]' lvalue "somestring"
 // CHECK: -IntegerLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'int' 1
 // CHECK: -FloatingLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'double' 2.000000e+00
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} Implicit "example"
+// CHECK: -StringLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'const char[{{[0-9]+}}]' lvalue "def"
+// CHECK: -BinaryOperator 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'int' '+'
+// CHECK: -IntegerLiteral 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}} 'int' 5
+// CHECK: -IntegerLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'int' 1
+// CHECK: -AnnotateAttr 0x{{[0-9a-z]+}} {{<line:[0-9]+:[0-9]+(, col:[0-9]+)?>}} "example"
+// CHECK: -StringLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'const char[{{[0-9]+}}]' lvalue "template"
+// CHECK: -IntegerLiteral 0x{{[0-9a-z]+}} {{<col:[0-9]+(, col:[0-9]+)?>}} 'int' 5
 
 //--- bad_attr.cpp
 int var1 __attribute__((example("otherstring"))) = 1; // expected-warning {{'example' attribute only applies to functions}}
 class Example {
   void __attribute__((example)) fn3(); // expected-error {{'example' attribute only allowed at file scope}}
 };
-void fn4() __attribute__((example(123))) { } // expected-error {{first argument to the 'example' attribute must be a string literal}}
-void fn5() __attribute__((example("a","b", 3, 4.0))) { } // expected-error {{'example' attribute only accepts at most three arguments}}
+void fn4() __attribute__((example(123))) { // expected-error {{first argument to the 'example' attribute must be a string literal}}
+  __attribute__((example("somestring"))) while (true); // expected-warning {{'example' attribute only applies to for loop statements}}
+}
+void fn5() __attribute__((example("a","b", 3, 4.0))) { // expected-error {{'example' attribute only accepts at most three arguments}}
+  __attribute__((example("a","b", 3, 4.0))) for (int i = 0; i < 10; ++i) {} // expected-error {{'example' attribute only accepts at most three arguments}}
+}
diff --git a/clang/test/Headers/import_header_unit_after_pragma.cpp b/clang/test/Headers/import_header_unit_after_pragma.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b1ad3b07fea29cc6d88314dacd7b4e3c0292e48c
--- /dev/null
+++ b/clang/test/Headers/import_header_unit_after_pragma.cpp
@@ -0,0 +1,18 @@
+// RUN: rm -fR %t
+// RUN: split-file %s %t
+// RUN: cd %t
+// RUN: %clang_cc1 -verify -std=c++20 -emit-header-unit -xc++-user-header bz0.h
+// RUN: %clang_cc1 -verify -std=c++20 -emit-header-unit -xc++-user-header -fmodule-file=bz0.pcm bz.cpp
+
+//--- compare
+#pragma GCC visibility push(default)
+#pragma GCC visibility pop
+
+//--- bz0.h
+#include "compare"
+// expected-no-diagnostics
+
+//--- bz.cpp
+#include "compare"
+
+import "bz0.h"; // expected-warning {{the implementation of header units is in an experimental phase}}
diff --git a/clang/test/Headers/ms-intrin.cpp b/clang/test/Headers/ms-intrin.cpp
index cb7cd47956205c209718d13cbb74db19e6e5dc68..d630883e79d6a56b11408fbc9b19ddd9f68063e0 100644
--- a/clang/test/Headers/ms-intrin.cpp
+++ b/clang/test/Headers/ms-intrin.cpp
@@ -1,31 +1,31 @@
 // RUN: %clang_cc1 -triple i386-pc-win32 -target-cpu pentium4 \
 // RUN:     -fms-extensions -fms-compatibility -fms-compatibility-version=17.00 \
-// RUN:     -ffreestanding -fsyntax-only -Werror \
+// RUN:     -ffreestanding -fsyntax-only -Werror -Wsystem-headers \
 // RUN:     -isystem %S/Inputs/include %s
 
 // RUN: %clang_cc1 -triple i386-pc-win32 -target-cpu broadwell \
 // RUN:     -fms-extensions -fms-compatibility -fms-compatibility-version=17.00 \
-// RUN:     -ffreestanding -emit-obj -o /dev/null -Werror \
+// RUN:     -ffreestanding -emit-obj -o /dev/null -Werror -Wsystem-headers \
 // RUN:     -isystem %S/Inputs/include %s
 
 // RUN: %clang_cc1 -triple x86_64-pc-win32  \
 // RUN:     -fms-extensions -fms-compatibility -fms-compatibility-version=17.00 \
-// RUN:     -ffreestanding -emit-obj -o /dev/null -Werror \
+// RUN:     -ffreestanding -emit-obj -o /dev/null -Werror -Wsystem-headers \
 // RUN:     -isystem %S/Inputs/include %s
 
 // RUN: %clang_cc1 -triple thumbv7--windows \
 // RUN:     -fms-compatibility -fms-compatibility-version=17.00 \
-// RUN:     -ffreestanding -fsyntax-only -Werror \
+// RUN:     -ffreestanding -fsyntax-only -Werror -Wsystem-headers \
 // RUN:     -isystem %S/Inputs/include %s
 
 // RUN: %clang_cc1 -triple aarch64--windows \
 // RUN:     -fms-compatibility -fms-compatibility-version=17.00 \
-// RUN:     -ffreestanding -fsyntax-only -Werror \
+// RUN:     -ffreestanding -fsyntax-only -Werror -Wsystem-headers \
 // RUN:     -isystem %S/Inputs/include %s
 
 // RUN: %clang_cc1 -triple arm64ec--windows \
 // RUN:     -fms-compatibility -fms-compatibility-version=17.00 \
-// RUN:     -ffreestanding -fsyntax-only -Werror \
+// RUN:     -ffreestanding -fsyntax-only -Werror -Wsystem-headers \
 // RUN:     -isystem %S/Inputs/include %s
 
 // REQUIRES: x86-registered-target
diff --git a/clang/test/Lexer/gnu-flags.c b/clang/test/Lexer/gnu-flags.c
index 4d6d216b101f4a690cabcf15458e370bfd83004b..6c7bf9405ddf0a6968c54d637f697e32cf4d98ca 100644
--- a/clang/test/Lexer/gnu-flags.c
+++ b/clang/test/Lexer/gnu-flags.c
@@ -2,13 +2,12 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s -DALL -Wgnu
 // RUN: %clang_cc1 -fsyntax-only -verify %s -DALL \
 // RUN:   -Wgnu-zero-variadic-macro-arguments \
-// RUN:   -Wgnu-imaginary-constant -Wgnu-zero-line-directive
+// RUN:   -Wgnu-zero-line-directive
 // RUN: %clang_cc1 -fsyntax-only -verify %s -DNONE -Wgnu \
 // RUN:   -Wno-gnu-zero-variadic-macro-arguments \
-// RUN:   -Wno-gnu-imaginary-constant -Wno-gnu-zero-line-directive
+// RUN:   -Wno-gnu-zero-line-directive
 // Additional disabled tests:
 // %clang_cc1 -fsyntax-only -verify %s -DZEROARGS -Wgnu-zero-variadic-macro-arguments
-// %clang_cc1 -fsyntax-only -verify %s -DIMAGINARYCONST -Wgnu-imaginary-constant
 // %clang_cc1 -fsyntax-only -verify %s -DLINE0 -Wgnu-zero-line-directive
 
 #if NONE
@@ -28,13 +27,6 @@ void foo( const char* c )
 }
 
 
-#if ALL || IMAGINARYCONST
-// expected-warning@+3 {{imaginary constants are a GNU extension}}
-#endif
-
-float _Complex c = 1.if;
-
-
 // This case is handled differently because lit has a bug whereby #line 0 is reported to be on line 4294967295
 // http://llvm.org/bugs/show_bug.cgi?id=16952
 #if ALL || LINE0
diff --git a/clang/test/Misc/target-invalid-cpu-note/arm.c b/clang/test/Misc/target-invalid-cpu-note/arm.c
index 27608cc6eb29fcb93922a4a75bec55d218f356b5..17280a9edd221deddaf5bbd235f1c87d3137f06d 100644
--- a/clang/test/Misc/target-invalid-cpu-note/arm.c
+++ b/clang/test/Misc/target-invalid-cpu-note/arm.c
@@ -65,6 +65,7 @@
 // CHECK-SAME: {{^}}, cortex-m7
 // CHECK-SAME: {{^}}, cortex-m23
 // CHECK-SAME: {{^}}, cortex-m33
+// CHECK-SAME: {{^}}, star-mc1
 // CHECK-SAME: {{^}}, cortex-m35p
 // CHECK-SAME: {{^}}, cortex-m55
 // CHECK-SAME: {{^}}, cortex-m85
diff --git a/clang/test/Misc/target-invalid-cpu-note/nvptx.c b/clang/test/Misc/target-invalid-cpu-note/nvptx.c
index af4ccff6b07b6c6063a84e23f3cea18c85041896..43524ab2906bf965fe91c457f5596cd31bde72c8 100644
--- a/clang/test/Misc/target-invalid-cpu-note/nvptx.c
+++ b/clang/test/Misc/target-invalid-cpu-note/nvptx.c
@@ -26,6 +26,7 @@
 // CHECK-SAME: {{^}}, sm_89
 // CHECK-SAME: {{^}}, sm_90
 // CHECK-SAME: {{^}}, sm_90a
+// CHECK-SAME: {{^}}, sm_100
 // CHECK-SAME: {{^}}, gfx600
 // CHECK-SAME: {{^}}, gfx601
 // CHECK-SAME: {{^}}, gfx602
diff --git a/clang/test/Modules/cxx-templates.cpp b/clang/test/Modules/cxx-templates.cpp
index 953fc0d9e867c415fd653b2422a5ba5713f23dd4..44b02ad3ec748c5a1e551f24ceb38c62a5be06a0 100644
--- a/clang/test/Modules/cxx-templates.cpp
+++ b/clang/test/Modules/cxx-templates.cpp
@@ -40,14 +40,23 @@ void g() {
 
   template_param_kinds_1<0>(); // ok, from cxx-templates-a.h
   template_param_kinds_1<int>(); // ok, from cxx-templates-b.h
-  template_param_kinds_2<Tmpl_T_C>(); // ok, from cxx-templates-b.h
+
+  template_param_kinds_2<Tmpl_T_C>(); // expected-error {{no matching function}}
+  // expected-note@Inputs/cxx-templates-a.h:11 {{invalid explicitly-specified argument}}
+  // expected-note@Inputs/cxx-templates-b.h:11 {{invalid explicitly-specified argument}}
 
   template_param_kinds_2<Tmpl_T_I_I>(); // expected-error {{ambiguous}}
   // expected-note@Inputs/cxx-templates-a.h:11 {{candidate}}
   // expected-note@Inputs/cxx-templates-b.h:11 {{candidate}}
 
-  template_param_kinds_3<Tmpl_T_T_A>();
-  template_param_kinds_3<Tmpl_T_T_B>();
+  // FIXME: This should be valid, but we incorrectly match the template template
+  // argument against both template template parameters.
+  template_param_kinds_3<Tmpl_T_T_A>(); // expected-error {{ambiguous}}
+  // expected-note@Inputs/cxx-templates-a.h:12 {{candidate}}
+  // expected-note@Inputs/cxx-templates-b.h:12 {{candidate}}
+  template_param_kinds_3<Tmpl_T_T_B>(); // expected-error {{ambiguous}}
+  // expected-note@Inputs/cxx-templates-a.h:12 {{candidate}}
+  // expected-note@Inputs/cxx-templates-b.h:12 {{candidate}}
 
   // Trigger the instantiation of a template in 'a' that uses a type defined in
   // 'common'. That type is not visible here.
@@ -190,9 +199,7 @@ namespace hidden_specializations {
     cls<char*> uk4; // expected-error 1+{{partial specialization of 'cls<T *>' must be imported}} expected-error 1+{{definition of}}
     cls<void>::nested_cls unk1; // expected-error 1+{{explicit specialization of 'nested_cls' must be imported}} expected-error 1+{{definition of}}
     cls<void>::nested_cls_t<int> unk2; // expected-error 1+{{explicit specialization of 'nested_cls_t' must be imported}} expected-error 1+{{definition of}}
-    // expected-error@cxx-templates-unimported.h:29 {{explicit specialization of 'nested_cls_t' must be imported}}
-    // expected-note@-2 {{in evaluation of exception specification}}
-    cls<void>::nested_cls_t<char> unk3; // expected-error 1+{{explicit specialization of 'nested_cls_t' must be imported}}
+    cls<void>::nested_cls_t<char> unk3; // expected-error 1+{{explicit specialization of 'nested_cls_t' must be imported}} expected-error 1+{{definition of}}
 
     // For enums, uses that would trigger instantiations of definitions are not
     // allowed.
diff --git a/clang/test/Parser/cxx11-user-defined-literals.cpp b/clang/test/Parser/cxx11-user-defined-literals.cpp
index cdd06729efc39a78f38416eeea9e045be4618513..27a7181bc9f9170eb8e851bda8ce9515809212ae 100644
--- a/clang/test/Parser/cxx11-user-defined-literals.cpp
+++ b/clang/test/Parser/cxx11-user-defined-literals.cpp
@@ -39,19 +39,19 @@ int cake() __attribute__((availability(macosx, unavailable, message = "is a lie"
 #endif
 
 // But they can appear in expressions.
-constexpr char operator"" _id(char c) { return c; }
-constexpr wchar_t operator"" _id(wchar_t c) { return c; }
-constexpr char16_t operator"" _id(char16_t c) { return c; }
-constexpr char32_t operator"" _id(char32_t c) { return c; }
+constexpr char operator""_id(char c) { return c; }
+constexpr wchar_t operator""_id(wchar_t c) { return c; }
+constexpr char16_t operator""_id(char16_t c) { return c; }
+constexpr char32_t operator""_id(char32_t c) { return c; }
 
 using size_t = decltype(sizeof(int));
-constexpr const char operator"" _id(const char *p, size_t n) { return *p; }
-constexpr const wchar_t operator"" _id(const wchar_t *p, size_t n) { return *p; }
-constexpr const char16_t operator"" _id(const char16_t *p, size_t n) { return *p; }
-constexpr const char32_t operator"" _id(const char32_t *p, size_t n) { return *p; }
+constexpr const char operator""_id(const char *p, size_t n) { return *p; }
+constexpr const wchar_t operator""_id(const wchar_t *p, size_t n) { return *p; }
+constexpr const char16_t operator""_id(const char16_t *p, size_t n) { return *p; }
+constexpr const char32_t operator""_id(const char32_t *p, size_t n) { return *p; }
 
-constexpr unsigned long long operator"" _id(unsigned long long n) { return n; }
-constexpr long double operator"" _id(long double d) { return d; }
+constexpr unsigned long long operator""_id(unsigned long long n) { return n; }
+constexpr long double operator""_id(long double d) { return d; }
 
 template<int n> struct S {};
 S<"a"_id> sa;
@@ -98,19 +98,22 @@ _no_such_suffix; // expected-error {{'operator""_no_such_suffix'}}
 // is "" in translation phase 7.
 void operator "\
 " _foo(unsigned long long); // ok
+// expected-warning@-1{{identifier '_foo' preceded by whitespace in a literal operator declaration is deprecated}}
 
 void operator R"xyzzy()xyzzy" _foo(long double); // ok
+// expected-warning@-1{{identifier '_foo' preceded by whitespace in a literal operator declaration is deprecated}}
 
 void operator"" "" R"()" "" _foo(const char *); // ok
+// expected-warning@-1{{identifier '_foo' preceded by whitespace}}
 
 void operator ""_no_space(const char *); // ok
 
 // Ensure we diagnose the bad cases.
-void operator "\0" _non_empty(const char *); // expected-error {{must be '""'}}
-void operator L"" _not_char(const char *); // expected-error {{cannot have an encoding prefix}}
+void operator "\0"_non_empty(const char *); // expected-error {{must be '""'}}
+void operator L""_not_char(const char *); // expected-error {{cannot have an encoding prefix}}
 void operator "" ""
 U"" // expected-error {{cannot have an encoding prefix}}
-"" _also_not_char(const char *);
+""_also_not_char(const char *);
 void operator "" u8"" "\u0123" "hello"_all_of_the_things ""(const char*); // expected-error {{must be '""'}}
 
 // Make sure we treat UCNs and UTF-8 as equivalent.
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 6c382379a8a7ea2dad529428b8f8f0727360f868..81c48335cf0c425d06e0a277eca2c6cabebf2bb7 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -949,7 +949,6 @@ void IntExprParsing() {
 #pragma acc loop vector(length:returns_int())
   for(;;);
 
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
 #pragma acc loop worker
   for(;;);
   // expected-error@+1{{expected expression}}
@@ -959,8 +958,8 @@ void IntExprParsing() {
   // expected-error@+1{{expected expression}}
 #pragma acc loop worker(invalid:)
   for(;;);
-  // expected-error@+2{{invalid tag 'invalid' on 'worker' clause}}
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
+#pragma acc kernels
+  // expected-error@+1{{invalid tag 'invalid' on 'worker' clause}}
 #pragma acc loop worker(invalid:5)
   for(;;);
   // expected-error@+1{{expected expression}}
@@ -983,21 +982,21 @@ void IntExprParsing() {
   // expected-note@+1{{to match this '('}}
 #pragma acc loop worker(length:6,4)
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
+#pragma acc kernels
 #pragma acc loop worker(5)
   for(;;);
-  // expected-error@+2{{invalid tag 'length' on 'worker' clause}}
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
+#pragma acc kernels
+  // expected-error@+1{{invalid tag 'length' on 'worker' clause}}
 #pragma acc loop worker(length:5)
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
+#pragma acc kernels
 #pragma acc loop worker(num:5)
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
+#pragma acc kernels
 #pragma acc loop worker(returns_int())
   for(;;);
-  // expected-error@+2{{invalid tag 'length' on 'worker' clause}}
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
+#pragma acc kernels
+  // expected-error@+1{{invalid tag 'length' on 'worker' clause}}
 #pragma acc loop worker(length:returns_int())
   for(;;);
 }
@@ -1202,7 +1201,6 @@ void Tile() {
 }
 
 void Gang() {
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
 #pragma acc loop gang
   for(;;){}
   // expected-error@+3{{expected expression}}
@@ -1210,68 +1208,58 @@ void Gang() {
   // expected-note@+1{{to match this '('}}
 #pragma acc loop gang(
   for(;;){}
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang()
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(5, *)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(*)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(5, num:*)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(num:5, *)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(num:5, num:*)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(num:*)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
-#pragma acc loop gang(dim:5)
+#pragma acc loop gang(dim:2)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(dim:5, dim:*)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop gang(dim:*)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
 #pragma acc loop gang(static:*)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+2{{OpenACC 'gang' clause may have at most one 'static' argument}}
+  // expected-note@+1{{previous expression is here}}
 #pragma acc loop gang(static:*, static:5)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+#pragma acc kernels
 #pragma acc loop gang(static:*, 5)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+#pragma acc kernels
 #pragma acc loop gang(static:45, 5)
   for(;;){}
 
@@ -1330,11 +1318,16 @@ void Gang() {
 #pragma acc loop gang(dim:45
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
-#pragma acc loop gang(static:*, dim:returns_int(), 5)
+#pragma acc kernels
+#pragma acc loop gang(static:*, 5)
+  for(;;){}
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be a constant expression}}
+#pragma acc loop gang(static:*, dim:returns_int())
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
+  // expected-error@+2 2{{'num' argument on 'gang' clause is not permitted on an orphaned 'loop' construct}}
+  // expected-error@+1{{argument to 'gang' clause dimension must be a constant expression}}
 #pragma acc loop gang(num: 32, static:*, dim:returns_int(), 5)
   for(;;){}
 
diff --git a/clang/test/Preprocessor/feature_tests.cpp b/clang/test/Preprocessor/feature_tests.cpp
index 13e2a9a261b6672103c421603b64b64e9210a2e0..029f446113af4502a339502b6ca4ca13c55fba56 100644
--- a/clang/test/Preprocessor/feature_tests.cpp
+++ b/clang/test/Preprocessor/feature_tests.cpp
@@ -58,7 +58,17 @@
 
 // Check __has_constexpr_builtin
 #if  !__has_constexpr_builtin(__builtin_fmax) || \
-     !__has_constexpr_builtin(__builtin_fmin)
+     !__has_constexpr_builtin(__builtin_fmin) || \
+     !__has_constexpr_builtin(__builtin_fmaximum_num) || \
+     !__has_constexpr_builtin(__builtin_fminimum_num)
+#error Clang should have these constexpr builtins
+#endif
+
+#if !__has_constexpr_builtin(__builtin_convertvector)
+#error Clang should have these constexpr builtins
+#endif
+
+#if !__has_constexpr_builtin(__builtin_shufflevector)
 #error Clang should have these constexpr builtins
 #endif
 
diff --git a/clang/test/Preprocessor/time64.c b/clang/test/Preprocessor/time64.c
new file mode 100644
index 0000000000000000000000000000000000000000..e9e26e08ecd870c1ad368e1ed6da3c91494a1f8c
--- /dev/null
+++ b/clang/test/Preprocessor/time64.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -E -dM -triple=i686-pc-linux-gnu /dev/null | FileCheck -match-full-lines -check-prefix TIME32 %s
+// RUN: %clang_cc1 -E -dM -triple=i686-pc-linux-gnut64 /dev/null | FileCheck -match-full-lines -check-prefix TIME64 %s
+// RUN: %clang_cc1 -E -dM -triple=armv5tel-softfloat-linux-gnueabi /dev/null | FileCheck -match-full-lines -check-prefix TIME32 %s
+// RUN: %clang_cc1 -E -dM -triple=armv5tel-softfloat-linux-gnueabit64 /dev/null | FileCheck -match-full-lines -check-prefix TIME64 %s
+// RUN: %clang_cc1 -E -dM -triple=armv7a-unknown-linux-gnueabihf /dev/null | FileCheck -match-full-lines -check-prefix TIME32 %s
+// RUN: %clang_cc1 -E -dM -triple=armv7a-unknown-linux-gnueabihft64 /dev/null | FileCheck -match-full-lines -check-prefix TIME64 %s
+//
+// TIME32-NOT:#define _FILE_OFFSET_BITS 64
+// TIME32-NOT:#define _TIME_BITS 64
+//
+// TIME64:#define _FILE_OFFSET_BITS 64
+// TIME64:#define _TIME_BITS 64
diff --git a/clang/test/Sema/builtin-assume-aligned.c b/clang/test/Sema/builtin-assume-aligned.c
index c2e4f9d659dd4d21a6434bf5f2c845dabd9c78b3..33e85578451529f833de2af5cca5df38de7cc24a 100644
--- a/clang/test/Sema/builtin-assume-aligned.c
+++ b/clang/test/Sema/builtin-assume-aligned.c
@@ -1,5 +1,7 @@
 // RUN: %clang_cc1 -DSIZE_T_64 -fsyntax-only -Wno-strict-prototypes -triple x86_64-linux -verify %s
 // RUN: %clang_cc1 -fsyntax-only -Wno-strict-prototypes -triple i386-freebsd -verify %s
+// RUN: %clang_cc1 -DSIZE_T_64 -fsyntax-only -Wno-strict-prototypes -triple x86_64-linux -verify %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -fsyntax-only -Wno-strict-prototypes -triple i386-freebsd -verify %s -fexperimental-new-constant-interpreter
 
 // __builtin_assume_aligned's second parameter is size_t, which may be 32 bits,
 // so test differently when size_t is 32 bits and when it is 64 bits.
diff --git a/clang/test/Sema/builtins.c b/clang/test/Sema/builtins.c
index d176dec9e9b3a3c73272f2eb69de3fbf64dacce4..b669ee68cdd9540a66bb1429120c838aa2ba53ec 100644
--- a/clang/test/Sema/builtins.c
+++ b/clang/test/Sema/builtins.c
@@ -40,7 +40,7 @@ void test9(short v) {
 
   old = __sync_fetch_and_add();  // expected-error {{too few arguments to function call}}
   old = __sync_fetch_and_add(&old);  // expected-error {{too few arguments to function call}}
-  old = __sync_fetch_and_add((unsigned*)0, 42i); // expected-warning {{imaginary constants are a GNU extension}}
+  old = __sync_fetch_and_add((unsigned*)0, 42i); // expected-warning {{imaginary constants are a C2y extension}}
 
   // PR7600: Pointers are implicitly casted to integers and back.
   void *old_ptr = __sync_val_compare_and_swap((void**)0, 0, 0);
diff --git a/clang/test/Sema/constant-builtins-fmaximum-num.cpp b/clang/test/Sema/constant-builtins-fmaximum-num.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..32d415ca9ee9845523475fa0763ee8f28ab861b5
--- /dev/null
+++ b/clang/test/Sema/constant-builtins-fmaximum-num.cpp
@@ -0,0 +1,69 @@
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify -fexperimental-new-constant-interpreter %s
+// expected-no-diagnostics
+
+constexpr double NaN = __builtin_nan("");
+constexpr double SNaN = __builtin_nans("");
+constexpr double Inf = __builtin_inf();
+constexpr double NegInf = -__builtin_inf();
+
+#define FMAXIMUMNUM_TEST_SIMPLE(T, FUNC)                           \
+    static_assert(T(6.7890) == FUNC(T(1.2345), T(6.7890))); \
+    static_assert(T(6.7890) == FUNC(T(6.7890), T(1.2345)));
+
+#define FMAXIMUMNUM_TEST_SNAN(T, FUNC)                          \
+    static_assert(Inf == FUNC(SNaN, Inf));               \
+    static_assert(NegInf == FUNC(NegInf, SNaN));         \
+    static_assert(0.0 == FUNC(SNaN, 0.0));               \
+    static_assert(-0.0 == FUNC(-0.0, SNaN));             \
+    static_assert(T(-1.2345) == FUNC(SNaN, T(-1.2345))); \
+    static_assert(T(1.2345) == FUNC(T(1.2345), SNaN));   \
+    static_assert(__builtin_isnan(FUNC(SNaN, SNaN)));    \
+    static_assert(__builtin_isnan(FUNC(NaN, SNaN)));    \
+    static_assert(!__builtin_issignaling(FUNC(SNaN, SNaN)));  \
+    static_assert(!__builtin_issignaling(FUNC(NaN, SNaN)));
+
+#define FMAXIMUMNUM_TEST_NAN(T, FUNC)                          \
+    static_assert(Inf == FUNC(NaN, Inf));               \
+    static_assert(NegInf == FUNC(NegInf, NaN));         \
+    static_assert(0.0 == FUNC(NaN, 0.0));               \
+    static_assert(-0.0 == FUNC(-0.0, NaN));             \
+    static_assert(T(-1.2345) == FUNC(NaN, T(-1.2345))); \
+    static_assert(T(1.2345) == FUNC(T(1.2345), NaN));   \
+    static_assert(__builtin_isnan(FUNC(NaN, NaN)));
+
+#define FMAXIMUMNUM_TEST_INF(T, FUNC)                  \
+    static_assert(Inf == FUNC(NegInf, Inf));    \
+    static_assert(Inf == FUNC(Inf, 0.0));       \
+    static_assert(Inf == FUNC(-0.0, Inf));      \
+    static_assert(Inf == FUNC(Inf, T(1.2345))); \
+    static_assert(Inf == FUNC(T(-1.2345), Inf));
+
+#define FMAXIMUMNUM_TEST_NEG_INF(T, FUNC)                         \
+    static_assert(Inf == FUNC(Inf, NegInf));               \
+    static_assert(0.0 == FUNC(NegInf, 0.0));               \
+    static_assert(-0.0 == FUNC(-0.0, NegInf));             \
+    static_assert(T(-1.2345) == FUNC(NegInf, T(-1.2345))); \
+    static_assert(T(1.2345) == FUNC(T(1.2345), NegInf));
+
+#define FMAXIMUMNUM_TEST_BOTH_ZERO(T, FUNC)       \
+    static_assert(__builtin_copysign(1.0, FUNC(0.0, 0.0)) == 1.0);  \
+    static_assert(__builtin_copysign(1.0, FUNC(-0.0, 0.0)) == 1.0);  \
+    static_assert(__builtin_copysign(1.0, FUNC(0.0, -0.0)) == 1.0);  \
+    static_assert(__builtin_copysign(1.0, FUNC(-0.0, -0.0)) == -1.0);
+
+#define LIST_FMAXIMUMNUM_TESTS(T, FUNC) \
+    FMAXIMUMNUM_TEST_SIMPLE(T, FUNC)    \
+    FMAXIMUMNUM_TEST_NAN(T, FUNC)       \
+    FMAXIMUMNUM_TEST_SNAN(T, FUNC)       \
+    FMAXIMUMNUM_TEST_INF(T, FUNC)       \
+    FMAXIMUMNUM_TEST_NEG_INF(T, FUNC)   \
+    FMAXIMUMNUM_TEST_BOTH_ZERO(T, FUNC)
+
+LIST_FMAXIMUMNUM_TESTS(double, __builtin_fmaximum_num)
+LIST_FMAXIMUMNUM_TESTS(float, __builtin_fmaximum_numf)
+LIST_FMAXIMUMNUM_TESTS((long double), __builtin_fmaximum_numl)
+LIST_FMAXIMUMNUM_TESTS(__fp16, __builtin_fmaximum_numf16)
+#ifdef __FLOAT128__
+LIST_FMAXIMUMNUM_TESTS(__float128, __builtin_fmaximum_numf128)
+#endif
diff --git a/clang/test/Sema/constant-builtins-fminimum-num.cpp b/clang/test/Sema/constant-builtins-fminimum-num.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d54f1ec84ad6fc80f29be74eed1015f9dc04ea64
--- /dev/null
+++ b/clang/test/Sema/constant-builtins-fminimum-num.cpp
@@ -0,0 +1,69 @@
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify -fexperimental-new-constant-interpreter %s
+// expected-no-diagnostics
+
+constexpr double NaN = __builtin_nan("");
+constexpr double SNaN = __builtin_nans("");
+constexpr double Inf = __builtin_inf();
+constexpr double NegInf = -__builtin_inf();
+
+#define FMINIMUMNUM_TEST_SIMPLE(T, FUNC)                           \
+    static_assert(T(1.2345) == FUNC(T(1.2345), T(6.7890))); \
+    static_assert(T(1.2345) == FUNC(T(6.7890), T(1.2345)));
+
+#define FMINIMUMNUM_TEST_NAN(T, FUNC)                          \
+    static_assert(Inf == FUNC(NaN, Inf));               \
+    static_assert(NegInf == FUNC(NegInf, NaN));         \
+    static_assert(0.0 == FUNC(NaN, 0.0));               \
+    static_assert(-0.0 == FUNC(-0.0, NaN));             \
+    static_assert(T(-1.2345) == FUNC(NaN, T(-1.2345))); \
+    static_assert(T(1.2345) == FUNC(T(1.2345), NaN));   \
+    static_assert(__builtin_isnan(FUNC(NaN, NaN)));
+
+#define FMINIMUMNUM_TEST_SNAN(T, FUNC)                          \
+    static_assert(Inf == FUNC(SNaN, Inf));               \
+    static_assert(NegInf == FUNC(NegInf, SNaN));         \
+    static_assert(0.0 == FUNC(SNaN, 0.0));               \
+    static_assert(-0.0 == FUNC(-0.0, SNaN));             \
+    static_assert(T(-1.2345) == FUNC(SNaN, T(-1.2345))); \
+    static_assert(T(1.2345) == FUNC(T(1.2345), SNaN));   \
+    static_assert(__builtin_isnan(FUNC(SNaN, SNaN)));    \
+    static_assert(__builtin_isnan(FUNC(NaN, SNaN)));    \
+    static_assert(!__builtin_issignaling(FUNC(SNaN, SNaN)));  \
+    static_assert(!__builtin_issignaling(FUNC(NaN, SNaN)));
+
+#define FMINIMUMNUM_TEST_INF(T, FUNC)                        \
+    static_assert(NegInf == FUNC(NegInf, Inf));       \
+    static_assert(0.0 == FUNC(Inf, 0.0));             \
+    static_assert(-0.0 == FUNC(-0.0, Inf));           \
+    static_assert(T(1.2345) == FUNC(Inf, T(1.2345))); \
+    static_assert(T(-1.2345) == FUNC(T(-1.2345), Inf));
+
+#define FMINIMUMNUM_TEST_NEG_INF(T, FUNC)                     \
+    static_assert(NegInf == FUNC(Inf, NegInf));        \
+    static_assert(NegInf == FUNC(NegInf, 0.0));        \
+    static_assert(NegInf == FUNC(-0.0, NegInf));       \
+    static_assert(NegInf == FUNC(NegInf, T(-1.2345))); \
+    static_assert(NegInf == FUNC(T(1.2345), NegInf));
+
+#define FMINIMUMNUM_TEST_BOTH_ZERO(T, FUNC)                                 \
+    static_assert(__builtin_copysign(1.0, FUNC(0.0, 0.0)) == 1.0);   \
+    static_assert(__builtin_copysign(1.0, FUNC(-0.0, 0.0)) == -1.0); \
+    static_assert(__builtin_copysign(1.0, FUNC(0.0, -0.0)) == -1.0); \
+    static_assert(__builtin_copysign(1.0, FUNC(-0.0, -0.0)) == -1.0);
+
+#define LIST_FMINIMUMNUM_TESTS(T, FUNC) \
+    FMINIMUMNUM_TEST_SIMPLE(T, FUNC)    \
+    FMINIMUMNUM_TEST_NAN(T, FUNC)       \
+    FMINIMUMNUM_TEST_SNAN(T, FUNC)      \
+    FMINIMUMNUM_TEST_INF(T, FUNC)       \
+    FMINIMUMNUM_TEST_NEG_INF(T, FUNC)   \
+    FMINIMUMNUM_TEST_BOTH_ZERO(T, FUNC)
+
+LIST_FMINIMUMNUM_TESTS(double, __builtin_fminimum_num)
+LIST_FMINIMUMNUM_TESTS(float, __builtin_fminimum_numf)
+LIST_FMINIMUMNUM_TESTS((long double), __builtin_fminimum_numl)
+LIST_FMINIMUMNUM_TESTS(__fp16, __builtin_fminimum_numf16)
+#ifdef __FLOAT128__
+LIST_FMINIMUMNUM_TESTS(__float128, __builtin_fminimum_numf128)
+#endif
diff --git a/clang/test/Sema/exprs.c b/clang/test/Sema/exprs.c
index 3203d961dd0a49a28a755f6220d20223b3fd0497..caf173a9aa501155af505020ab30a32f74102610 100644
--- a/clang/test/Sema/exprs.c
+++ b/clang/test/Sema/exprs.c
@@ -40,7 +40,7 @@ _Complex double test1(void) {
 }
 
 _Complex double test2(void) {
-  return 1.0if;    // expected-warning {{imaginary constants are a GNU extension}}
+  return 1.0if;    // expected-warning {{imaginary constants are a C2y extension}}
 }
 
 void test3(void) {
@@ -57,7 +57,7 @@ void test4(void) {
 
       var =+5;  // no warning when the subexpr of the unary op has no space before it.
       var =-5;
-  
+
 #define FIVE 5
       var=-FIVE;  // no warning with macros.
       var=-FIVE;
@@ -152,7 +152,7 @@ void test17(int x) {
   x = x % 0;  // expected-warning {{remainder by zero is undefined}}
   x /= 0;  // expected-warning {{division by zero is undefined}}
   x %= 0;  // expected-warning {{remainder by zero is undefined}}
-  
+
   x = sizeof(x/0);  // no warning.
 }
 
@@ -205,7 +205,7 @@ int test20(int x) {
                  // expected-note {{remove constant to silence this warning}}
 
   return x && sizeof(int) == 4;  // no warning, RHS is logical op.
-  
+
   // no warning, this is an idiom for "true" in old C style.
   return x && (signed char)1;
 
diff --git a/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp b/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp
index 731639ab16a735c332a62df5766e3492340dfa6a..688f55edfe84df4c06f5f9f0be0f04b942c6382b 100644
--- a/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp
+++ b/clang/test/Sema/warn-lifetime-analysis-nocfg.cpp
@@ -164,14 +164,16 @@ template<typename T>
 struct initializer_list {
   const T* ptr; size_t sz;
 };
-template <typename T>
+template<typename T> class allocator {};
+template <typename T, typename Alloc = allocator<T>>
 struct vector {
   typedef __gnu_cxx::basic_iterator<T> iterator;
   iterator begin();
   iterator end();
   const T *data() const;
   vector();
-  vector(initializer_list<T> __l);
+  vector(initializer_list<T> __l,
+         const Alloc& alloc = Alloc());
 
   template<typename InputIterator>
 	vector(InputIterator first, InputIterator __last);
diff --git a/clang/test/SemaCXX/GH41441.cpp b/clang/test/SemaCXX/GH41441.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7a6260fef91b568f130f3bef08612a46ed9d6f29
--- /dev/null
+++ b/clang/test/SemaCXX/GH41441.cpp
@@ -0,0 +1,46 @@
+// RUN: %clang --target=x86_64-pc-linux -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 %s -fsyntax-only -verify
+
+namespace std {
+  using size_t = decltype(sizeof(int));
+};
+void* operator new[](std::size_t, void*) noexcept;
+
+// CHECK: call void @llvm.memset.p0.i64(ptr align 1 %x, i8 0, i64 8, i1 false)
+// CHECK: call void @llvm.memset.p0.i64(ptr align 16 %x, i8 0, i64 32, i1 false)
+template <typename TYPE>
+void f()
+{
+    typedef TYPE TArray[8];
+
+    TArray x;
+    new(&x) TArray();
+}
+
+template <typename T>
+void f1() {
+  int (*x)[1] = new int[1][1];
+}
+template void f1<char>();
+void f2() {
+  int (*x)[1] = new int[1][1];
+}
+
+int main()
+{
+    f<char>();
+    f<int>();
+}
+
+// expected-no-diagnostics
+template <typename T> struct unique_ptr {unique_ptr(T* p){}};
+
+template <typename T>
+unique_ptr<T> make_unique(unsigned long long n) {
+  return unique_ptr<T>(new T[n]());
+}
+
+auto boro(int n){
+	typedef double HistoryBuffer[4];
+	return make_unique<HistoryBuffer>(n);
+}
diff --git a/clang/test/SemaCXX/block-packs.cpp b/clang/test/SemaCXX/block-packs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8f63c98fa2745186b68f605900c8bd28769386ab
--- /dev/null
+++ b/clang/test/SemaCXX/block-packs.cpp
@@ -0,0 +1,67 @@
+// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin -fsyntax-only -verify -Wno-unused %s
+// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin -fsyntax-only -verify -Wno-unused %s -frecovery-ast -frecovery-ast-type
+
+// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin -fsyntax-only -verify -Wno-unused -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -fblocks -triple x86_64-apple-darwin -fsyntax-only -verify -Wno-unused -frecovery-ast -frecovery-ast-type -fexperimental-new-constant-interpreter %s
+
+template <typename ...Ts>
+void f() {
+  ((^ { Ts t; }), ...);
+  ((^ (Ts t) {}), ...);
+  ((^ Ts () {}), ...);
+
+  ^ { Ts t; }; // expected-error {{unexpanded parameter pack 'Ts'}}
+  ^ (Ts t) {}; // expected-error {{unexpanded parameter pack 'Ts'}}
+  ^ Ts () {};  // expected-error {{unexpanded parameter pack 'Ts'}}
+}
+
+template <typename ...Ts>
+void gh109148() {
+  (^Ts); // expected-error {{expected expression}}
+
+  [] {
+    (^Ts); // expected-error {{expected expression}}
+    ^Ts;   // expected-error {{expected expression}}
+    ^(Ts); // expected-error {{expected expression}}
+    ^ Ts); // expected-error {{expected expression}}
+  };
+
+  ([] {
+    (^Ts); // expected-error {{expected expression}}
+    ^Ts;   // expected-error {{expected expression}}
+    ^(Ts); // expected-error {{expected expression}}
+    ^ Ts); // expected-error {{expected expression}}
+  }, ...); // expected-error {{pack expansion does not contain any unexpanded parameter packs}}
+
+  [] { // expected-error {{unexpanded parameter pack 'Ts'}}
+    ^ (Ts) {};
+  };
+
+  [] { // expected-error {{unexpanded parameter pack 'Ts'}}
+    ^ { Ts x; };
+  };
+
+  [] { // expected-error {{unexpanded parameter pack 'Ts'}}
+    Ts s;
+    (^Ts); // expected-error {{expected expression}}
+  };
+
+  ([] {
+    Ts s;
+    (^Ts); // expected-error {{expected expression}}
+  }, ...);
+
+  [] { // expected-error {{unexpanded parameter pack 'Ts'}}
+    ^ { Ts s; return not_defined; }; // expected-error {{use of undeclared identifier 'not_defined'}}
+  };
+}
+
+void g() {
+  f<>();
+  f<int>();
+  f<long, float>();
+
+  gh109148<>();
+  gh109148<int>();
+  gh109148<long, float>();
+}
diff --git a/clang/test/SemaCXX/builtin-assume-aligned.cpp b/clang/test/SemaCXX/builtin-assume-aligned.cpp
index 48bd8414fc50a10d56c4db878216e0809680c75c..85a7faee916181a14d9a245d779c78792248b554 100644
--- a/clang/test/SemaCXX/builtin-assume-aligned.cpp
+++ b/clang/test/SemaCXX/builtin-assume-aligned.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -triple x86_64-linux-gnu %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -triple x86_64-linux-gnu %s -fexperimental-new-constant-interpreter
 
 int n;
 constexpr int *p = 0;
diff --git a/clang/test/SemaCXX/cxx11-user-defined-literals-unused.cpp b/clang/test/SemaCXX/cxx11-user-defined-literals-unused.cpp
index cd93ffbf21edf4a3a32e6e5bfdc5399edafa0201..cf95ae7de98e94829868577fe4ab029d10fe17f4 100644
--- a/clang/test/SemaCXX/cxx11-user-defined-literals-unused.cpp
+++ b/clang/test/SemaCXX/cxx11-user-defined-literals-unused.cpp
@@ -1,8 +1,8 @@
 // RUN: %clang_cc1 -std=c++11 -verify %s -Wunused
 
 namespace {
-double operator"" _x(long double value) { return double(value); }
-int operator"" _ii(long double value) { return int(value); } // expected-warning {{not needed and will not be emitted}}
+double operator""_x(long double value) { return double(value); }
+int operator""_ii(long double value) { return int(value); } // expected-warning {{not needed and will not be emitted}}
 }
 
 namespace rdar13589856 {
diff --git a/clang/test/SemaCXX/cxx11-user-defined-literals.cpp b/clang/test/SemaCXX/cxx11-user-defined-literals.cpp
index b5d4d9976c1372fd7c21e4e4c1ee8aca91525e86..bdf88ee01c3a1215fc49555eda22709a0c849e70 100644
--- a/clang/test/SemaCXX/cxx11-user-defined-literals.cpp
+++ b/clang/test/SemaCXX/cxx11-user-defined-literals.cpp
@@ -6,18 +6,18 @@ enum class LitKind {
   CharStr, WideStr, Char16Str, Char32Str,
   Integer, Floating, Raw, Template
 };
-constexpr LitKind operator"" _kind(char p) { return LitKind::Char; }
-constexpr LitKind operator"" _kind(wchar_t p) { return LitKind::WideChar; }
-constexpr LitKind operator"" _kind(char16_t p) { return LitKind::Char16; }
-constexpr LitKind operator"" _kind(char32_t p) { return LitKind::Char32; }
-constexpr LitKind operator"" _kind(const char *p, size_t n) { return LitKind::CharStr; }
-constexpr LitKind operator"" _kind(const wchar_t *p, size_t n) { return LitKind::WideStr; }
-constexpr LitKind operator"" _kind(const char16_t *p, size_t n) { return LitKind::Char16Str; }
-constexpr LitKind operator"" _kind(const char32_t *p, size_t n) { return LitKind::Char32Str; }
-constexpr LitKind operator"" _kind(unsigned long long n) { return LitKind::Integer; }
-constexpr LitKind operator"" _kind(long double n) { return LitKind::Floating; }
-constexpr LitKind operator"" _kind2(const char *p) { return LitKind::Raw; }
-template<char ...Cs> constexpr LitKind operator"" _kind3() { return LitKind::Template; }
+constexpr LitKind operator""_kind(char p) { return LitKind::Char; }
+constexpr LitKind operator""_kind(wchar_t p) { return LitKind::WideChar; }
+constexpr LitKind operator""_kind(char16_t p) { return LitKind::Char16; }
+constexpr LitKind operator""_kind(char32_t p) { return LitKind::Char32; }
+constexpr LitKind operator""_kind(const char *p, size_t n) { return LitKind::CharStr; }
+constexpr LitKind operator""_kind(const wchar_t *p, size_t n) { return LitKind::WideStr; }
+constexpr LitKind operator""_kind(const char16_t *p, size_t n) { return LitKind::Char16Str; }
+constexpr LitKind operator""_kind(const char32_t *p, size_t n) { return LitKind::Char32Str; }
+constexpr LitKind operator""_kind(unsigned long long n) { return LitKind::Integer; }
+constexpr LitKind operator""_kind(long double n) { return LitKind::Floating; }
+constexpr LitKind operator""_kind2(const char *p) { return LitKind::Raw; }
+template<char ...Cs> constexpr LitKind operator""_kind3() { return LitKind::Template; }
 
 static_assert('x'_kind == LitKind::Char, "");
 static_assert(L'x'_kind == LitKind::WideChar, "");
@@ -41,7 +41,7 @@ static_assert(4e6_kind3 == LitKind::Template, "");
 constexpr const char *fractional_digits_impl(const char *p) {
   return *p == '.' ? p + 1 : *p ? fractional_digits_impl(p + 1) : 0;
 }
-constexpr const char *operator"" _fractional_digits(const char *p) {
+constexpr const char *operator""_fractional_digits(const char *p) {
   return fractional_digits_impl(p) ?: p;
 }
 constexpr bool streq(const char *p, const char *q) {
@@ -56,57 +56,57 @@ static_assert(streq(1e+97_fractional_digits, "1e+97"), "");
 static_assert(streq(0377_fractional_digits, "0377"), "");
 static_assert(streq(0377.5_fractional_digits, "5"), "");
 
-int operator"" _ambiguous(char); // expected-note {{candidate}}
+int operator""_ambiguous(char); // expected-note {{candidate}}
 namespace N {
-  void *operator"" _ambiguous(char); // expected-note {{candidate}}
+  void *operator""_ambiguous(char); // expected-note {{candidate}}
 }
 using namespace N;
 int k = 'x'_ambiguous; // expected-error {{ambiguous}}
 
-int operator"" _deleted(unsigned long long) = delete; // expected-note {{here}}
+int operator""_deleted(unsigned long long) = delete; // expected-note {{here}}
 int m = 42_deleted; // expected-error {{attempt to use a deleted}}
 
 namespace Using {
   namespace M {
-    int operator"" _using(char);
+    int operator""_using(char);
   }
   int k1 = 'x'_using; // expected-error {{no matching literal operator for call to 'operator""_using'}}
 
-  using M::operator "" _using;
+  using M::operator ""_using;
   int k2 = 'x'_using;
 }
 
 namespace AmbiguousRawTemplate {
-  int operator"" _ambig1(const char *); // expected-note {{candidate}}
-  template<char...> int operator"" _ambig1(); // expected-note {{candidate}}
+  int operator""_ambig1(const char *); // expected-note {{candidate}}
+  template<char...> int operator""_ambig1(); // expected-note {{candidate}}
 
   int k1 = 123_ambig1; // expected-error {{call to 'operator""_ambig1' is ambiguous}}
 
   namespace Inner {
-    template<char...> int operator"" _ambig2(); // expected-note 3{{candidate}}
+    template<char...> int operator""_ambig2(); // expected-note 3{{candidate}}
   }
-  int operator"" _ambig2(const char *); // expected-note 3{{candidate}}
-  using Inner::operator"" _ambig2;
+  int operator""_ambig2(const char *); // expected-note 3{{candidate}}
+  using Inner::operator""_ambig2;
 
   int k2 = 123_ambig2; // expected-error {{call to 'operator""_ambig2' is ambiguous}}
 
   namespace N {
-    using Inner::operator"" _ambig2;
+    using Inner::operator""_ambig2;
 
     int k3 = 123_ambig2; // ok
 
-    using AmbiguousRawTemplate::operator"" _ambig2;
+    using AmbiguousRawTemplate::operator""_ambig2;
 
     int k4 = 123_ambig2; // expected-error {{ambiguous}}
 
     namespace M {
 
-      template<char...> int operator"" _ambig2();
+      template<char...> int operator""_ambig2();
 
       int k5 = 123_ambig2; // ok
     }
 
-    int operator"" _ambig2(unsigned long long);
+    int operator""_ambig2(unsigned long long);
 
     int k6 = 123_ambig2; // ok
     int k7 = 123._ambig2; // expected-error {{ambiguous}}
@@ -121,7 +121,7 @@ template<char C, char...Cs> constexpr unsigned hash(unsigned a) {
  return hash<Cs...>(mash(a ^ mash(C)));
 }
 template<typename T, T v> struct constant { constexpr static T value = v; };
-template<char...Cs> constexpr unsigned operator"" _hash() {
+template<char...Cs> constexpr unsigned operator""_hash() {
   return constant<unsigned, hash<Cs...>(0)>::value;
 }
 static_assert(0x1234_hash == 0x103eff5e, "");
@@ -129,7 +129,7 @@ static_assert(hash<'0', 'x', '1', '2', '3', '4'>(0) == 0x103eff5e, "");
 
 // Functions and literal suffixes go in separate namespaces.
 namespace Namespace {
-  template<char...> int operator"" _x();
+  template<char...> int operator""_x();
   int k = _x(); // expected-error {{undeclared identifier '_x'}}
 
   int _y(unsigned long long);
@@ -138,7 +138,7 @@ namespace Namespace {
 
 namespace PR14950 {
   template<...> // expected-error {{expected template parameter}}
-  int operator"" _b(); // expected-error {{no function template matches function template specialization}}
+  int operator""_b(); // expected-error {{no function template matches function template specialization}}
   int main() { return 0_b; } // expected-error {{no matching literal operator for call to 'operator""_b'}}
 }
 
diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp
index ae331055c52b2e87f2a912053ae0fe6953481e8d..fef4674d178412e72be0aaaa5a632d3b626923fd 100644
--- a/clang/test/SemaCXX/cxx2a-consteval.cpp
+++ b/clang/test/SemaCXX/cxx2a-consteval.cpp
@@ -158,17 +158,17 @@ int i3 = f1(f1(f1(&f1, &f1), f1(&f1, &f1), f1(f1(&f1, &f1), &f1)));
 
 namespace user_defined_literal {
 
-consteval int operator"" _test(unsigned long long i) {
+consteval int operator""_test(unsigned long long i) {
 // expected-note@-1+ {{declared here}}
   return 0;
 }
 
 int i = 0_test;
 
-auto ptr = &operator"" _test;
+auto ptr = &operator""_test;
 // expected-error@-1 {{take address}}
 
-consteval auto operator"" _test1(unsigned long long i) {
+consteval auto operator""_test1(unsigned long long i) {
   return &f_eval;
 }
 
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index 63bf92e8d5edd3ae9836fa0d6422b5f2202b0ba4..2a984a75f37d2108585f1616a82529a5373764fd 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -142,8 +142,8 @@ struct Corresponding {
     void d(this Corresponding&&);
     void d(this const Corresponding&);
     void d(this const int&);
-    void d(this const int);
-    void d(this int);
+    void d(this const int);  // expected-note {{previous declaration is here}}
+    void d(this int);        // expected-error {{class member cannot be redeclared}}
 
     void e(this const Corresponding&&); // expected-note {{here}}
     void e() const &&; // expected-error{{cannot be redeclared}}
@@ -171,9 +171,8 @@ struct CorrespondingTpl {
     void d(this Corresponding&&);
     void d(this const Corresponding&);
     void d(this const int&);
-    void d(this const int);
-    void d(this int);
-
+    void d(this const int); // expected-note {{previous declaration is here}}
+    void d(this int);       // expected-error {{class member cannot be redeclared}}
     void e(this const CorrespondingTpl&&); // expected-note {{here}}
     void e() const &&; // expected-error{{cannot be redeclared}}
 };
@@ -1073,3 +1072,28 @@ int main() {
   return foo[]; // expected-error {{no viable overloaded operator[] for type 'Foo'}}
 }
 }
+
+namespace GH100394 {
+struct C1 {
+  void f(this const C1);
+  void f() const;        // ok
+};
+
+struct C2 {
+  void f(this const C2);    // expected-note {{previous declaration is here}}
+  void f(this volatile C2); // expected-error {{class member cannot be redeclared}} \
+                            // expected-warning {{volatile-qualified parameter type 'volatile C2' is deprecated}}
+};
+
+struct C3 {
+  void f(this volatile C3); // expected-note {{previous declaration is here}} \
+                            // expected-warning {{volatile-qualified parameter type 'volatile C3' is deprecated}}
+  void f(this const C3);    // expected-error {{class member cannot be redeclared}}
+};
+
+struct C4 {
+  void f(this const C4);          // expected-note {{previous declaration is here}}
+  void f(this const volatile C4); // expected-error {{class member cannot be redeclared}} \
+                                  // expected-warning {{volatile-qualified parameter type 'const volatile C4' is deprecated}}
+};
+}
diff --git a/clang/test/SemaCXX/cxx98-compat.cpp b/clang/test/SemaCXX/cxx98-compat.cpp
index 3ce69d6908c5b22db3b406653b59987a8f3a928a..28547d42c64902d734c9dd660f9309de42c83619 100644
--- a/clang/test/SemaCXX/cxx98-compat.cpp
+++ b/clang/test/SemaCXX/cxx98-compat.cpp
@@ -84,7 +84,7 @@ struct DelayedDefaultArgumentParseInitList {
   }
 };
 
-int operator"" _hello(const char *); // expected-warning {{literal operators are incompatible with C++98}}
+int operator""_hello(const char *); // expected-warning {{literal operators are incompatible with C++98}}
 
 enum EnumFixed : int { // expected-warning {{enumeration types with a fixed underlying type are incompatible with C++98}}
 };
diff --git a/clang/test/SemaCXX/lambda-capture-type-deduction.cpp b/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
index 7bf36a6a9cab732c4d53a420f5385467a9ff1dfd..a86f30189899271f8e84b03713b27442e9104871 100644
--- a/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
+++ b/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
@@ -297,3 +297,25 @@ void __trans_tmp_1() {
 }
 
 }
+
+namespace GH47400 {
+
+struct Foo {};
+
+template <int, Foo> struct Arr {};
+
+template <int> struct S {};
+
+constexpr void foo() {
+  constexpr Foo f;
+  [&]<int is>() {
+    [&](Arr<is, f>) {}({}); // f constitutes an ODR-use
+  }.template operator()<42>();
+
+  constexpr int C = 1;
+  [] {
+    [](S<C>) { }({}); // ... while C doesn't
+  }();
+}
+
+} // namespace GH47400
diff --git a/clang/test/SemaCXX/literal-operators.cpp b/clang/test/SemaCXX/literal-operators.cpp
index 067e151606202ee0760223ff0f49b19f34c1ad7a..2a41e048914b786de54b175793dcbc72806513d2 100644
--- a/clang/test/SemaCXX/literal-operators.cpp
+++ b/clang/test/SemaCXX/literal-operators.cpp
@@ -3,46 +3,46 @@
 #include <stddef.h>
 
 struct tag {
-  void operator "" _tag_bad (const char *); // expected-error {{literal operator 'operator""_tag_bad' must be in a namespace or global scope}}
-  friend void operator "" _tag_good (const char *);
+  void operator ""_tag_bad (const char *); // expected-error {{literal operator 'operator""_tag_bad' must be in a namespace or global scope}}
+  friend void operator ""_tag_good (const char *);
 };
 
-namespace ns { void operator "" _ns_good (const char *); }
+namespace ns { void operator ""_ns_good (const char *); }
 
 // Check extern "C++" declarations
-extern "C++" void operator "" _extern_good (const char *);
-extern "C++" { void operator "" _extern_good (const char *); }
+extern "C++" void operator ""_extern_good (const char *);
+extern "C++" { void operator ""_extern_good (const char *); }
 
-void fn () { void operator "" _fn_good (const char *); }
+void fn () { void operator ""_fn_good (const char *); }
 
 // One-param declarations (const char * was already checked)
-void operator "" _good (char);
-void operator "" _good (wchar_t);
-void operator "" _good (char16_t);
-void operator "" _good (char32_t);
-void operator "" _good (unsigned long long);
-void operator "" _good (long double);
+void operator ""_good (char);
+void operator ""_good (wchar_t);
+void operator ""_good (char16_t);
+void operator ""_good (char32_t);
+void operator ""_good (unsigned long long);
+void operator ""_good (long double);
 
 // Two-param declarations
-void operator "" _good (const char *, size_t);
-void operator "" _good (const wchar_t *, size_t);
-void operator "" _good (const char16_t *, size_t);
-void operator "" _good (const char32_t *, size_t);
+void operator ""_good (const char *, size_t);
+void operator ""_good (const wchar_t *, size_t);
+void operator ""_good (const char16_t *, size_t);
+void operator ""_good (const char32_t *, size_t);
 
 // Check typedef and array equivalences
-void operator "" _good (const char[]);
+void operator ""_good (const char[]);
 typedef const char c;
-void operator "" _good (c*);
+void operator ""_good (c*);
 
 // Check extra cv-qualifiers
-void operator "" _cv_good (volatile const char *, const size_t); // expected-error {{invalid literal operator parameter type 'const volatile char *', did you mean 'const char *'?}}
+void operator ""_cv_good (volatile const char *, const size_t); // expected-error {{invalid literal operator parameter type 'const volatile char *', did you mean 'const char *'?}}
 
 // Template declaration
-template <char...> void operator "" _good ();
+template <char...> void operator ""_good ();
 
-template <typename...> void operator "" _invalid(); // expected-error {{template parameter list for literal operator must be either 'char...' or 'typename T, T...'}}
-template <wchar_t...> void operator "" _invalid();  // expected-error {{template parameter list for literal operator must be either 'char...' or 'typename T, T...'}}
-template <unsigned long long...> void operator "" _invalid();  // expected-error {{template parameter list for literal operator must be either 'char...' or 'typename T, T...'}}
+template <typename...> void operator ""_invalid(); // expected-error {{template parameter list for literal operator must be either 'char...' or 'typename T, T...'}}
+template <wchar_t...> void operator ""_invalid();  // expected-error {{template parameter list for literal operator must be either 'char...' or 'typename T, T...'}}
+template <unsigned long long...> void operator ""_invalid();  // expected-error {{template parameter list for literal operator must be either 'char...' or 'typename T, T...'}}
 
 _Complex float operator""if(long double); // expected-warning {{reserved}}
 _Complex float test_if_1() { return 2.0f + 1.5if; };
diff --git a/clang/test/SemaCXX/make_integer_seq.cpp b/clang/test/SemaCXX/make_integer_seq.cpp
index 71b7b8260d4abc83e1b181e437d8e4ee924d1e1e..8f72ce15eef476ab7cf7b8baf81d2cc215fcfb55 100644
--- a/clang/test/SemaCXX/make_integer_seq.cpp
+++ b/clang/test/SemaCXX/make_integer_seq.cpp
@@ -48,5 +48,6 @@ using illformed2 = ErrorSeq<int, -5>; // expected-note{{in instantiation}}
 template <typename T, T N> void f() {}
 __make_integer_seq<f, int, 0> x; // expected-error{{template template parameter must be a class template or type alias template}}
 
-__make_integer_seq<__make_integer_seq, int, 10> PR28494; // expected-note{{different template parameters}}
-// expected-error@make_integer_seq.cpp:* {{template argument for template template parameter must be a class template or type alias template}}
+__make_integer_seq<__make_integer_seq, int, 10> PR28494; // expected-error{{different template parameters}}
+// expected-note@make_integer_seq.cpp:* {{template parameter has a different kind}}
+// expected-note@make_integer_seq.cpp:* {{previous template template parameter is here}}
diff --git a/clang/test/SemaCXX/no-warn-user-defined-literals-in-system-headers.cpp b/clang/test/SemaCXX/no-warn-user-defined-literals-in-system-headers.cpp
index ee5b0c47b9ffcd2e0d125027a06266983dbdf44e..61b08954c40b5863f2c1e6690e89437d7c742128 100644
--- a/clang/test/SemaCXX/no-warn-user-defined-literals-in-system-headers.cpp
+++ b/clang/test/SemaCXX/no-warn-user-defined-literals-in-system-headers.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -Wsystem-headers -isystem %S %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -isystem %S %s
 
 #include <no-warn-user-defined-literals-in-system-headers.h>
 
diff --git a/clang/test/SemaCXX/reserved-identifier.cpp b/clang/test/SemaCXX/reserved-identifier.cpp
index eaa5fe8330337e77ca51e90777e5ff7fc2f18b19..a3e6d8ead61fd8a9266c4ecffbf4ad8f01840c1d 100644
--- a/clang/test/SemaCXX/reserved-identifier.cpp
+++ b/clang/test/SemaCXX/reserved-identifier.cpp
@@ -76,7 +76,8 @@ namespace {
 int _barbatruc; // no-warning
 }
 
-long double operator"" _BarbeBleue(long double) // expected-warning {{identifier '_BarbeBleue' is reserved because it starts with '_' followed by a capital letter}}
+long double operator"" _BarbeBleue(long double) // expected-warning {{identifier '_BarbeBleue' is reserved because it starts with '_' followed by a capital letter}}\
+                                                // expected-warning {{identifier '_BarbeBleue' preceded by whitespace in a literal operator declaration is deprecated}}
 {
   return 0.;
 }
@@ -86,10 +87,11 @@ long double operator""_SacreBleu(long double) // no-warning
   return 0.;
 }
 
-long double sacrebleu = operator"" _SacreBleu(1.2); // expected-warning {{identifier '_SacreBleu' is reserved because it starts with '_' followed by a capital letter}}
+long double sacrebleu = operator"" _SacreBleu(1.2); // expected-warning {{identifier '_SacreBleu' is reserved because it starts with '_' followed by a capital letter}} \
+                                                    // expected-warning {{identifier '_SacreBleu' preceded by whitespace in a literal operator declaration is deprecated}}
 long double sangbleu = operator""_SacreBleu(1.2);   // no-warning
 
-void operator"" _lowercase(unsigned long long); // no-warning
+void operator"" _lowercase(unsigned long long); // expected-warning {{identifier '_lowercase' preceded by whitespace in a literal operator declaration is deprecated}}
 void operator""_lowercase(unsigned long long); // no-warning
 
 struct _BarbeRouge { // expected-warning {{identifier '_BarbeRouge' is reserved because it starts with '_' followed by a capital letter}}
diff --git a/clang/test/SemaCXX/warn-xor-as-pow.cpp b/clang/test/SemaCXX/warn-xor-as-pow.cpp
index 88719bd1a9d7757126d76a468f9da26230705154..ade977f97e436655109a21dd016306508a620895 100644
--- a/clang/test/SemaCXX/warn-xor-as-pow.cpp
+++ b/clang/test/SemaCXX/warn-xor-as-pow.cpp
@@ -19,10 +19,10 @@
 #define flexor 7
 
 #ifdef __cplusplus
-constexpr long long operator"" _xor(unsigned long long v) { return v; }
+constexpr long long operator""_xor(unsigned long long v) { return v; }
 
-constexpr long long operator"" _0b(unsigned long long v) { return v; }
-constexpr long long operator"" _0X(unsigned long long v) { return v; }
+constexpr long long operator""_0b(unsigned long long v) { return v; }
+constexpr long long operator""_0X(unsigned long long v) { return v; }
 #else
 #define xor ^ // iso646.h
 #endif
diff --git a/clang/test/SemaObjCXX/attr-nonblocking-constraints.mm b/clang/test/SemaObjCXX/attr-nonblocking-constraints.mm
index abd0938ac321af1fe069097c58d0cf84fe20ba73..d1c9b3cc98a1c0fb1f9e2912afbc49a3fe82f478 100644
--- a/clang/test/SemaObjCXX/attr-nonblocking-constraints.mm
+++ b/clang/test/SemaObjCXX/attr-nonblocking-constraints.mm
@@ -23,4 +23,17 @@ void nb4() [[clang::nonblocking]] {
 	}
 	@catch (...) { // expected-warning {{function with 'nonblocking' attribute must not throw or catch exceptions}}
 	}
+	@finally { // expected-warning {{function with 'nonblocking' attribute must not throw or catch exceptions}}
+	}
+}
+
+@class Lock;
+extern Lock *someLock;
+
+void nb5() [[clang::nonblocking]] {
+	@autoreleasepool { // expected-warning {{function with 'nonblocking' attribute must not access ObjC methods or properties}}
+	}
+
+	@synchronized(someLock) { // expected-warning {{function with 'nonblocking' attribute must not access ObjC methods or properties}}
+	}
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
index d08497a7782edbc7f1fa3f552ea9ffb58f6c6cb2..aaf8b76e1f3dfbee22c4aac252d801cd41a6f505 100644
--- a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
@@ -61,8 +61,7 @@ void uses() {
   // expected-error@+1{{OpenACC 'auto' clause is not valid on 'kernels' directive}}
 #pragma acc kernels device_type(*) auto
   while(1);
-  // expected-error@+2{{OpenACC clause 'worker' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
+  // expected-error@+1{{OpenACC 'worker' clause is not valid on 'kernels' directive}}
 #pragma acc kernels device_type(*) worker
   while(1);
   // expected-error@+2{{OpenACC clause 'nohost' may not follow a 'device_type' clause in a compute construct}}
@@ -214,8 +213,7 @@ void uses() {
   // expected-error@+1{{OpenACC 'tile' clause is not valid on 'kernels' directive}}
 #pragma acc kernels device_type(*) tile(Var, 1)
   while(1);
-  // expected-error@+2{{OpenACC clause 'gang' may not follow a 'dtype' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
+  // expected-error@+1{{OpenACC 'gang' clause is not valid on 'kernels' directive}}
 #pragma acc kernels dtype(*) gang
   while(1);
 #pragma acc kernels device_type(*) wait
diff --git a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
index 3da7f0e9836be8723ddbe03054fb48ce11609c9c..6a975956f3ff5c5e84d859816571b65721d601bd 100644
--- a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
+++ b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
@@ -43,7 +43,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc loop auto if_present
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc loop auto worker
   for(;;);
   // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
@@ -168,7 +167,6 @@ void uses() {
 #pragma acc loop auto tile(1+2, 1)
   for(;;)
     for(;;);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop auto gang
   for(;;);
   // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
@@ -181,7 +179,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc loop if_present auto
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc loop worker auto
   for(;;);
   // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
@@ -306,7 +303,6 @@ void uses() {
 #pragma acc loop tile(1+2, 1) auto
   for(;;)
     for(;;);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop gang auto
   for(;;);
   // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
@@ -320,7 +316,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc loop independent if_present
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc loop independent worker
   for(;;);
   // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
@@ -445,7 +440,6 @@ void uses() {
 #pragma acc loop independent tile(1+2, 1)
   for(;;)
     for(;;);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop independent gang
   for(;;);
   // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
@@ -458,7 +452,6 @@ void uses() {
   // expected-warning@+1{{OpenACC clause 'if_present' not yet implemented}}
 #pragma acc loop if_present independent
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
 #pragma acc loop worker independent
   for(;;);
   // expected-warning@+1{{OpenACC clause 'vector' not yet implemented}}
@@ -583,7 +576,6 @@ void uses() {
 #pragma acc loop tile(1+2, 1) independent
   for(;;)
     for(;;);
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop gang independent
   for(;;);
   // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
@@ -591,14 +583,12 @@ void uses() {
   for(;;);
 
   // 'seq' cannot be combined with 'gang', 'worker' or 'vector'
-  // expected-error@+3{{OpenACC clause 'gang' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
-  // expected-note@+2{{previous clause is here}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
+  // expected-error@+2{{OpenACC clause 'gang' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
+  // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq gang
   for(;;);
-  // expected-error@+3{{OpenACC clause 'worker' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
-  // expected-note@+2{{previous clause is here}}
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
+  // expected-error@+2{{OpenACC clause 'worker' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
+  // expected-note@+1{{previous clause is here}}
 #pragma acc loop seq worker
   for(;;);
   // expected-error@+3{{OpenACC clause 'vector' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
@@ -735,16 +725,12 @@ void uses() {
 #pragma acc loop seq wait
   for(;;);
 
-  // TODO OpenACC: when 'gang' is implemented and makes it to the AST, this should diagnose because of a conflict with 'seq'.
-  // TODOexpected-error@+3{{OpenACC clause 'gang' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
-  // TODOexpected-note@+2{{previous clause is here}}
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'gang' clause on a 'loop' construct}}
+  // expected-note@+1{{previous clause is here}}
 #pragma acc loop gang seq
   for(;;);
-  // TODO OpenACC: when 'worker' is implemented and makes it to the AST, this should diagnose because of a conflict with 'seq'.
-  // TODOexpected-error@+3{{OpenACC clause 'worker' may not appear on the same construct as a 'seq' clause on a 'loop' construct}}
-  // TODOexpected-note@+2{{previous clause is here}}
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented}}
+  // expected-error@+2{{OpenACC clause 'seq' may not appear on the same construct as a 'worker' clause on a 'loop' construct}}
+  // expected-note@+1{{previous clause is here}}
 #pragma acc loop worker seq
   for(;;);
   // TODO OpenACC: when 'vector' is implemented and makes it to the AST, this should diagnose because of a conflict with 'seq'.
diff --git a/clang/test/SemaOpenACC/loop-construct-device_type-clause.c b/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
index 3d77c031f42630c02507d5d326a6e1ed105dc9f0..51da8565f4e39944e1f27ca8b1af9db527424daf 100644
--- a/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
@@ -56,7 +56,6 @@ void uses() {
   for(;;);
 #pragma acc loop device_type(*) auto
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'worker' not yet implemented, clause ignored}}
 #pragma acc loop device_type(*) worker
   for(;;);
   // expected-error@+2{{OpenACC clause 'nohost' may not follow a 'device_type' clause in a 'loop' construct}}
@@ -193,7 +192,6 @@ void uses() {
   for(;;)
     for(;;);
 
-  // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
 #pragma acc loop dtype(*) gang
   for(;;);
   // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
diff --git a/clang/test/SemaOpenACC/loop-construct-gang-ast.cpp b/clang/test/SemaOpenACC/loop-construct-gang-ast.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e797d842e240dc5ffa4daf35e069f4541bdd87ee
--- /dev/null
+++ b/clang/test/SemaOpenACC/loop-construct-gang-ast.cpp
@@ -0,0 +1,330 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: IntegerLiteral{{.*}}'int' 1
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc loop gang(dim:1)
+  for(;;);
+
+  int Val;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} used Val 'int'
+
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc loop gang(static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause num
+  // CHECK-NEXT: IntegerLiteral{{.*}}'int' 1
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc kernels
+#pragma acc loop gang(num:1) gang(static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause dim static
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: IntegerLiteral{{.*}}'int' 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc parallel
+#pragma acc loop gang(dim:1, static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc serial
+#pragma acc loop gang(static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc serial
+#pragma acc loop gang(static:*)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc serial
+#pragma acc loop gang
+  for(;;);
+}
+
+template<typename T, unsigned One>
+void TemplateUses(T Val) {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplateUses
+  // CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'unsigned int' depth 0 index 1 One
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplateUses 'void (T)'
+  // CHECK-NEXT: ParmVarDecl{{.*}} referenced Val 'T'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: DeclRefExpr{{.*}}'unsigned int' NonTypeTemplateParm{{.*}} 'One' 'unsigned int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc loop gang(dim:One)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc loop gang(static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc loop gang(static:*)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: DeclRefExpr{{.*}}'unsigned int' NonTypeTemplateParm{{.*}} 'One' 'unsigned int'
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc parallel
+#pragma acc loop gang(dim:One) gang(static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause dim static
+  // CHECK-NEXT: DeclRefExpr{{.*}}'unsigned int' NonTypeTemplateParm{{.*}} 'One' 'unsigned int'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc parallel
+#pragma acc loop gang(dim:One, static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 'Val' 'T'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc serial
+#pragma acc loop gang(static:Val)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+#pragma acc serial
+#pragma acc loop gang
+  for(;;);
+
+  // Instantiation:
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplateUses 'void (int)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: TemplateArgument integral '1U'
+  // CHECK-NEXT: ParmVarDecl{{.*}} used Val 'int'
+  // CHECK-NEXT: CompoundStmt
+  //
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: ConstantExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 1 One
+  // CHECK-NEXT: IntegerLiteral{{.*}}'unsigned int' 1
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause dim
+  // CHECK-NEXT: ConstantExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 1 One
+  // CHECK-NEXT: IntegerLiteral{{.*}}'unsigned int' 1
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause dim static
+  // CHECK-NEXT: ConstantExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 1 One
+  // CHECK-NEXT: IntegerLiteral{{.*}}'unsigned int' 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause static
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 'Val' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: gang clause
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: <<<NULL>>
+  // CHECK-NEXT: NullStmt
+}
+
+void inst() {
+  TemplateUses<int, 1>(5);
+}
+
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/loop-construct-gang-clause.cpp b/clang/test/SemaOpenACC/loop-construct-gang-clause.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ab6439ae57619340e942d85745b66cec67ae659e
--- /dev/null
+++ b/clang/test/SemaOpenACC/loop-construct-gang-clause.cpp
@@ -0,0 +1,335 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct S{};
+struct Converts{
+  operator int();
+};
+
+template<typename T, unsigned Zero, unsigned Two, unsigned Four>
+void ParallelOrOrphanTempl() {
+  T i;
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop gang(i)
+  for(;;);
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop gang(num:i)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'parallel' compute construct}}
+#pragma acc parallel
+#pragma acc loop gang(i)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'parallel' compute construct}}
+#pragma acc parallel
+#pragma acc loop gang(num:i)
+  for(;;);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be a constant expression}}
+#pragma acc loop gang(dim:i)
+  for(;;);
+
+  // expected-error@+2{{argument to 'gang' clause dimension must be a constant expression}}
+#pragma acc parallel
+#pragma acc loop gang(dim:i)
+  for(;;);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 0}}
+#pragma acc loop gang(dim:Zero)
+  for(;;);
+
+  // expected-error@+2{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 0}}
+#pragma acc parallel
+#pragma acc loop gang(dim:Zero)
+  for(;;);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 4}}
+#pragma acc loop gang(dim:Four)
+  for(;;);
+
+  // expected-error@+2{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 4}}
+#pragma acc parallel
+#pragma acc loop gang(dim:Four)
+  for(;;);
+
+#pragma acc loop gang(static:i) gang(dim:Two)
+  for(;;);
+
+#pragma acc parallel
+#pragma acc loop gang(dim:Two) gang(static:*)
+  for(;;);
+
+#pragma acc parallel
+#pragma acc loop gang(dim:Two, static:i)
+  for(;;);
+
+  // expected-error@+4{{OpenACC 'gang' clause may have at most one 'static' argument}}
+  // expected-note@+3{{previous expression is here}}
+  // expected-error@+2{{OpenACC 'gang' clause may have at most one 'dim' argument}}
+  // expected-note@+1{{previous expression is here}}
+#pragma acc loop gang(static:i, static:i, dim:Two, dim:1)
+  for(;;);
+}
+
+void ParallelOrOrphan() {
+  ParallelOrOrphanTempl<int, 0, 2, 4>(); // expected-note{{in instantiation of function template}}
+
+  int i;
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop gang(i)
+  for(;;);
+  // expected-error@+1{{'num' argument on 'gang' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop gang(num:i)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'parallel' compute construct}}
+#pragma acc parallel
+#pragma acc loop gang(i)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'parallel' compute construct}}
+#pragma acc parallel
+#pragma acc loop gang(num:i)
+  for(;;);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be a constant expression}}
+#pragma acc loop gang(dim:i)
+  for(;;);
+
+  // expected-error@+2{{argument to 'gang' clause dimension must be a constant expression}}
+#pragma acc parallel
+#pragma acc loop gang(dim:i)
+  for(;;);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 0}}
+#pragma acc loop gang(dim:0)
+  for(;;);
+
+  // expected-error@+2{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 0}}
+#pragma acc parallel
+#pragma acc loop gang(dim:0)
+  for(;;);
+
+  // expected-error@+1{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 4}}
+#pragma acc loop gang(dim:4)
+  for(;;);
+
+  // expected-error@+2{{argument to 'gang' clause dimension must be 1, 2, or 3: evaluated to 4}}
+#pragma acc parallel
+#pragma acc loop gang(dim:4)
+  for(;;);
+
+#pragma acc loop gang(static:i) gang(dim:2)
+  for(;;);
+
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:i)
+  for(;;);
+
+  S s;
+  // expected-error@+2{{OpenACC clause 'gang' requires expression of integer type ('S' invalid)}}
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:s)
+  for(;;);
+
+  Converts C;
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:C)
+  for(;;);
+}
+
+template<typename SomeS, typename SomeC, typename Int>
+void StaticIsIntegralTempl() {
+  SomeS s;
+  // expected-error@+2{{OpenACC clause 'gang' requires expression of integer type ('S' invalid)}}
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:s)
+  for(;;);
+
+  SomeC C;
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:C)
+  for(;;);
+  Int i;
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:i)
+  for(;;);
+
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:*)
+  for(;;);
+}
+
+void StaticIsIntegral() {
+  StaticIsIntegralTempl<S, Converts, int>();// expected-note{{in instantiation of function template}}
+
+  S s;
+  // expected-error@+2{{OpenACC clause 'gang' requires expression of integer type ('S' invalid)}}
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:s)
+  for(;;);
+
+  Converts C;
+#pragma acc parallel
+#pragma acc loop gang(dim:2) gang(static:C)
+  for(;;);
+}
+
+template<unsigned I>
+void SerialTempl() {
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop gang(I)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop gang(num:I)
+  for(;;);
+
+  // expected-error@+2{{'dim' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop gang(dim:I)
+  for(;;);
+
+#pragma acc serial
+#pragma acc loop gang(static:I)
+  for(;;);
+}
+
+void Serial() {
+  SerialTempl<2>();
+
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop gang(1)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop gang(num:1)
+  for(;;);
+
+  // expected-error@+2{{'dim' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop gang(dim:1)
+  for(;;);
+
+#pragma acc serial
+#pragma acc loop gang(static:1)
+  for(;;);
+
+  int i;
+
+#pragma acc serial
+#pragma acc loop gang(static:i)
+  for(;;);
+}
+
+template<typename T>
+void KernelsTempl() {
+  T t;
+  // expected-error@+2{{'dim' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'kernels' compute construct}}
+#pragma acc kernels
+#pragma acc loop gang(dim:t)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop gang(static:t)
+  for(;;);
+
+  // expected-error@+3{{'num' argument to 'gang' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_gangs(t)
+#pragma acc loop gang(t)
+  for(;;);
+
+  // expected-error@+3{{'num' argument to 'gang' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_gangs(t)
+#pragma acc loop gang(num:t)
+  for(;;);
+}
+
+void Kernels() {
+  KernelsTempl<unsigned>();
+
+  // expected-error@+2{{'dim' argument on 'gang' clause is not permitted on a 'loop' construct associated with a 'kernels' compute construct}}
+#pragma acc kernels
+#pragma acc loop gang(dim:1)
+  for(;;);
+  unsigned t;
+#pragma acc kernels
+#pragma acc loop gang(static:t)
+  for(;;);
+
+  // expected-error@+3{{'num' argument to 'gang' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_gangs(1)
+#pragma acc loop gang(1)
+  for(;;);
+
+  // expected-error@+3{{'num' argument to 'gang' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'num_gangs' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_gangs(1)
+#pragma acc loop gang(num:1)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop gang(num:1)
+  for(;;) {
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'gang' clause on a 'kernels' compute construct}}
+    // expected-note@-3{{previous clause is here}}
+#pragma acc loop gang(static:1)
+    for(;;);
+  }
+
+#pragma acc kernels
+#pragma acc loop gang(num:1)
+  for(;;) {
+    // allowed, intervening compute construct
+#pragma acc serial
+#pragma acc loop gang(static:1)
+    for(;;);
+  }
+
+#pragma acc kernels
+#pragma acc loop gang(num:1)
+  for(;;);
+
+  // OK, on a different 'loop', not in the assoc statement.
+#pragma acc loop gang(static:1)
+  for(;;);
+
+  // expected-error@+3{{OpenACC 'gang' clause may have at most one unnamed or 'num' argument}}
+  // expected-note@+2{{previous expression is here}}
+#pragma acc kernels
+#pragma acc loop gang(5, num:1)
+  for(;;);
+
+  // expected-error@+3{{OpenACC 'gang' clause may have at most one unnamed or 'num' argument}}
+  // expected-note@+2{{previous expression is here}}
+#pragma acc kernels
+#pragma acc loop gang(num:5, 1)
+  for(;;);
+
+  // expected-error@+3{{OpenACC 'gang' clause may have at most one unnamed or 'num' argument}}
+  // expected-note@+2{{previous expression is here}}
+#pragma acc kernels
+#pragma acc loop gang(num:5, num:1)
+  for(;;);
+}
+
+void MaxOneEntry() {
+  // expected-error@+3{{OpenACC 'gang' clause may have at most one 'static' argument}}
+  // expected-note@+2{{previous expression is here}}
+#pragma acc kernels
+#pragma acc loop gang(static: 1, static:1)
+    for(;;);
+
+#pragma acc kernels
+#pragma acc loop gang gang(static:1)
+    for(;;);
+}
+
+
diff --git a/clang/test/SemaOpenACC/loop-construct-worker-ast.cpp b/clang/test/SemaOpenACC/loop-construct-worker-ast.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6347e1419fd5c63913892cbe75a1f446f0f69970
--- /dev/null
+++ b/clang/test/SemaOpenACC/loop-construct-worker-ast.cpp
@@ -0,0 +1,270 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+template<unsigned I, typename ConvertsToInt, typename Int>
+void TemplUses(ConvertsToInt CTI, Int IsI) {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplUses
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 0 I
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 1 ConvertsToInt
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}typename depth 0 index 2 Int
+  // CHECK-NEXT: FunctionDecl{{.*}}TemplUses 'void (ConvertsToInt, Int)'
+  // CHECK-NEXT: ParmVarDecl{{.*}}CTI 'ConvertsToInt'
+  // CHECK-NEXT: ParmVarDecl{{.*}}IsI 'Int'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc loop worker
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc parallel
+#pragma acc loop worker
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc serial
+#pragma acc loop worker
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'ConvertsToInt' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc kernels
+#pragma acc loop worker(CTI)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Int' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc kernels
+#pragma acc loop worker(num:IsI)
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'unsigned int' NonTypeTemplateParm{{.*}}'I' 'unsigned int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc kernels
+#pragma acc loop worker(num:I)
+  for(;;);
+
+  // Instantiations:
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void (Converts, int)' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument integral '3U'
+  // CHECK-NEXT: TemplateArgument type 'Converts'
+  // CHECK-NEXT: RecordType{{.*}}'Converts'
+  // CHECK-NEXT: CXXRecord{{.*}}'Converts
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}}'int'
+  // CHECK-NEXT: ParmVarDecl{{.*}} CTI 'Converts'
+  // CHECK-NEXT: ParmVarDecl{{.*}} IsI 'int'
+  // CHECK-NEXT: CompoundStmt
+  //
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}} 'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator int
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'Converts' lvalue ParmVar
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'int' lvalue ParmVar{{.*}} 'IsI' 'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+  //
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}}'unsigned int' depth 0 index 0 I
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'unsigned int' 3
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+}
+
+struct Converts{
+  operator int();
+};
+
+void uses() {
+  // CHECK: FunctionDecl{{.*}} uses
+  // CHECK-NEXT: CompoundStmt
+  //
+  // CHECK-NEXT: CallExpr
+  TemplUses<3>(Converts{}, 5);
+
+  // CHECK: OpenACCLoopConstruct{{.*}}<orphan>
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc loop worker
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} parallel
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc parallel
+#pragma acc loop worker
+  for(;;);
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} serial
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc serial
+#pragma acc loop worker
+  for(;;);
+
+  Converts CTI;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl
+  // CHECK-NEXT: CXXConstructExpr
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}} 'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator int
+  // CHECK-NEXT: DeclRefExpr{{.*}}'Converts' lvalue Var
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc kernels
+#pragma acc loop worker(CTI)
+  for(;;);
+
+  int IsI;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl
+
+  // CHECK-NEXT: OpenACCComputeConstruct 0x[[COMPUTE_ADDR:[0-9a-f]+]]{{.*}} kernels
+  // CHECK-NEXT: OpenACCLoopConstruct{{.*}} parent: 0x[[COMPUTE_ADDR]]
+  // CHECK-NEXT: worker clause{{.*}}
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+#pragma acc kernels
+#pragma acc loop worker(num:IsI)
+  for(;;);
+}
+
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/loop-construct-worker-clause.cpp b/clang/test/SemaOpenACC/loop-construct-worker-clause.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f7d2f365a5aa7900f876cc4086352e9af49014d1
--- /dev/null
+++ b/clang/test/SemaOpenACC/loop-construct-worker-clause.cpp
@@ -0,0 +1,202 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+template<unsigned I, typename NotInt, typename ConvertsToInt, typename Int>
+void TemplUses(NotInt NI, ConvertsToInt CTI, Int IsI) {
+  int i;
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop worker(i)
+  for(;;);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop worker(num:IsI)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker(i)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker(CTI)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker(IsI)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker(I)
+  for(;;);
+
+#pragma acc kernels
+  // expected-error@+1{{OpenACC clause 'worker' requires expression of integer type ('NoConvert' invalid)}}
+#pragma acc loop worker(NI)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker(num:i)
+  for(;;);
+
+  // expected-error@+3{{'num' argument to 'worker' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'num_workers' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_workers(IsI)
+#pragma acc loop worker(num:CTI)
+  for(;;);
+ for(;;);
+}
+
+struct NoConvert{};
+struct Converts{
+  operator int();
+};
+
+void uses() {
+  TemplUses<3>(NoConvert{}, Converts{}, 5); // expected-note{{in instantiation of function template specialization}}
+
+#pragma acc loop worker
+  for(;;);
+
+#pragma acc parallel
+#pragma acc loop worker
+  for(;;);
+
+  int i;
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop worker(i)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'worker' clause is not permitted on a 'loop' construct associated with a 'parallel' compute construct}}
+#pragma acc parallel
+#pragma acc loop worker(i)
+  for(;;);
+
+  // expected-error@+1{{'num' argument on 'worker' clause is not permitted on an orphaned 'loop' construct}}
+#pragma acc loop worker(num:i)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'worker' clause is not permitted on a 'loop' construct associated with a 'parallel' compute construct}}
+#pragma acc parallel
+#pragma acc loop worker(num:i)
+  for(;;);
+
+#pragma acc serial
+#pragma acc loop worker
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'worker' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop worker(i)
+  for(;;);
+
+  // expected-error@+2{{'num' argument on 'worker' clause is not permitted on a 'loop' construct associated with a 'serial' compute construct}}
+#pragma acc serial
+#pragma acc loop worker(num:i)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker(i)
+  for(;;);
+
+  Converts Cvts;
+
+#pragma acc kernels
+#pragma acc loop worker(Cvts)
+  for(;;);
+
+  NoConvert NoCvts;
+
+#pragma acc kernels
+  // expected-error@+1{{OpenACC clause 'worker' requires expression of integer type ('NoConvert' invalid)}}
+#pragma acc loop worker(NoCvts)
+  for(;;);
+
+#pragma acc kernels
+#pragma acc loop worker(num:i)
+  for(;;);
+
+  // expected-error@+3{{'num' argument to 'worker' clause not allowed on a 'loop' construct associated with a 'kernels' construct that has a 'num_workers' clause}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels num_workers(i)
+#pragma acc loop worker(num:i)
+  for(;;);
+
+#pragma acc loop worker
+  for(;;) {
+    // expected-error@+3{{loop with a 'worker' clause may not exist in the region of a 'worker' clause}}
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-4 2{{previous clause is here}}
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+
+#pragma acc loop worker
+  for(;;) {
+#pragma acc parallel
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+
+
+#pragma acc parallel
+#pragma acc loop worker
+  for(;;) {
+    // expected-error@+3{{loop with a 'worker' clause may not exist in the region of a 'worker' clause}}
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-4 2{{previous clause is here}}
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+
+#pragma acc parallel
+#pragma acc loop worker
+  for(;;) {
+#pragma acc parallel
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+
+#pragma acc serial
+#pragma acc loop worker
+  for(;;) {
+    // expected-error@+3{{loop with a 'worker' clause may not exist in the region of a 'worker' clause}}
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-4 2{{previous clause is here}}
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+
+#pragma acc serial
+#pragma acc loop worker
+  for(;;) {
+#pragma acc parallel
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+
+#pragma acc kernels
+#pragma acc loop worker
+  for(;;) {
+    // expected-error@+3{{loop with a 'worker' clause may not exist in the region of a 'worker' clause}}
+    // expected-error@+2{{loop with a 'gang' clause may not exist in the region of a 'worker' clause}}
+    // expected-note@-4 2{{previous clause is here}}
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+
+#pragma acc kernels
+#pragma acc loop worker
+  for(;;) {
+#pragma acc parallel
+#pragma acc loop worker, gang
+  for(;;) {}
+  }
+}
diff --git a/clang/test/SemaTemplate/cwg2398.cpp b/clang/test/SemaTemplate/cwg2398.cpp
index 3825239de4a2855e7cec464c46c2c4763f474d07..6dc7af6ea17899151006e2a7752c7ebceb76234d 100644
--- a/clang/test/SemaTemplate/cwg2398.cpp
+++ b/clang/test/SemaTemplate/cwg2398.cpp
@@ -106,10 +106,12 @@ namespace type_pack3 {
   template<class T3> struct B;
 
   template<template<class T4              > class TT1, class T5              > struct B<TT1<T5        >>;
-
-  template<template<class T6, class ...T7s> class TT2, class T8, class ...T9s> struct B<TT2<T8, T9s...>> {};
+  // new-note@-1 {{template is declared here}}
+  template<template<class T6, class ...T7s> class TT2, class T8, class ...T9s> struct B<TT2<T8, T9s...>>;
+  // old-note@-1 {{template is declared here}}
 
   template struct B<A<int>>;
+  // expected-error@-1 {{explicit instantiation of undefined template}}
 } // namespace type_pack3
 
 namespace gcc_issue {
@@ -156,14 +158,16 @@ namespace ttp_defaults {
 namespace ttp_only {
   template <template <class...    > class TT1> struct A      { static constexpr int V = 0; };
   template <template <class       > class TT2> struct A<TT2> { static constexpr int V = 1; };
+  // new-note@-1 {{partial specialization matches}}
   template <template <class, class> class TT3> struct A<TT3> { static constexpr int V = 2; };
+  // new-note@-1 {{partial specialization matches}}
 
   template <class ...          > struct B;
   template <class              > struct C;
   template <class, class       > struct D;
   template <class, class, class> struct E;
 
-  static_assert(A<B>::V == 0);
+  static_assert(A<B>::V == 0); // new-error {{ambiguous partial specializations}}
   static_assert(A<C>::V == 1);
   static_assert(A<D>::V == 2);
   static_assert(A<E>::V == 0);
@@ -360,152 +364,6 @@ namespace classes {
   } // namespace defaulted
 } // namespace classes
 
-namespace packs {
-  namespace t1 {
-    // FIXME: This should be rejected
-    template<template<int, int...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'int' is here}}
-
-    template<char> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'char' in template argument}}
-
-    template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
-  } // namespace t1
-  namespace t2 {
-    template<template<char, int...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'char' is here}}
-
-    template<int> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'int' in template argument}}
-
-    template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
-  } // namespace t2
-  namespace t3 {
-    // FIXME: This should be rejected
-    template<template<int...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'int' is here}}
-
-    template<char> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'char' in template argument}}
-
-    template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
-  } // namespace t3
-  namespace t4 {
-    template<template<char...> class> struct A {};
-    // old-note@-1 {{previous non-type template parameter with type 'char' is here}}
-
-    template<int> struct B;
-    // old-note@-1 {{template non-type parameter has a different type 'int' in template argument}}
-
-    template struct A<B>;
-    // old-error@-1 {{has different template parameters}}
-  } // namespace t4
-} // namespace packs
-
-namespace fun_tmpl_call {
-  namespace match_func {
-    template <template <class> class TT> void f(TT<int>) {};
-    // old-note@-1 {{has different template parameters}}
-    template <class...> struct A {};
-    void test() { f(A<int>()); }
-    // old-error@-1 {{no matching function for call to 'f'}}
-  } // namespace match_func
-  namespace order_func_nonpack {
-    template <template <class> class TT> void f(TT<int>) {}
-    template <template <class...> class TT> void f(TT<int>) = delete;
-
-    template <class> struct A {};
-    void test() { f(A<int>()); }
-  } // namespace order_func_nonpack
-  namespace order_func_pack {
-    template <template <class> class TT> void f(TT<int>) = delete;
-    template <template <class...> class TT> void f(TT<int>) {}
-
-    template <class...> struct A {};
-    void test() { f(A<int>()); }
-  } // namespace order_func_pack
-  namespace match_method {
-    struct A {
-      template <template <class> class TT> void f(TT<int>) {};
-      // old-note@-1 {{has different template parameters}}
-    };
-    template <class...> struct B {};
-    void test() { A().f(B<int>()); }
-    // old-error@-1 {{no matching member function for call to 'f'}}
-  } // namespace t2
-  namespace order_method_nonpack {
-    struct A {
-      template <template <class> class TT> void f(TT<int>) {}
-      template <template <class...> class TT> void f(TT<int>) = delete;
-    };
-    template <class> struct B {};
-    void test() { A().f(B<int>()); }
-  } // namespace order_method_nonpack
-  namespace order_method_pack {
-    struct A {
-      template <template <class> class TT> void f(TT<int>) = delete;
-      template <template <class...> class TT> void f(TT<int>) {}
-    };
-    template <class...> struct B {};
-    void test() { A().f(B<int>()); }
-  } // namespace order_method_pack
-  namespace match_conv {
-    struct A {
-      template <template <class> class TT> operator TT<int>() { return {}; }
-      // old-note@-1 {{different template parameters}}
-    };
-    template <class...> struct B {};
-    // old-note@-1 2{{not viable}}
-    void test() { B<int> b = A(); }
-    // old-error@-1 {{no viable conversion from 'A' to 'B<int>'}}
-  } // namespace match_conv
-  namespace order_conv_nonpack {
-    struct A {
-      template <template <class> class TT> operator TT<int>() { return {}; };
-      template <template <class...> class TT> operator TT<int>() = delete;
-    };
-    template <class> struct B {};
-    void test() { B<int> b = A(); }
-  } // namespace order_conv_nonpack
-  namespace order_conv_pack {
-    struct A {
-      template <template <class> class TT> operator TT<int>() = delete;
-      template <template <class...> class TT> operator TT<int>() { return {}; }
-    };
-    template <class...> struct B {};
-    void test() { B<int> b = A(); }
-  } // namespace order_conv_pack
-  namespace regression1 {
-    template <template <class, class...> class TT, class T1, class... T2s>
-    void f(TT<T1, T2s...>) {}
-    template <class> struct A {};
-    void test() { f(A<int>()); }
-  } // namespace regression1
-} // namespace fun_tmpl_packs
-
-namespace partial {
-  namespace t1 {
-    template<template<class... T1s> class TT1> struct A {};
-
-    template<template<class T2> class TT2> struct A<TT2>;
-
-    template<class... T3s> struct B;
-    template struct A<B>;
-  } // namespace t1
-  namespace t2 {
-    template<template<class... T1s> class TT1> struct A;
-
-    template<template<class T2> class TT2> struct A<TT2> {};
-
-    template<class T3> struct B;
-    template struct A<B>;
-  } // namespace t1
-
-} // namespace partial
-
 namespace regression1 {
   template <typename T, typename Y> struct map {};
   template <typename T> class foo {};
@@ -522,93 +380,6 @@ namespace regression1 {
   }
 } // namespace regression1
 
-namespace constraints {
-  template <class T> concept C1 = true;
-  // new-note@-1 {{similar constraint expression here}}
-  // new-note@-2 2{{similar constraint expressions not considered equivalent}}
-
-  template <class T> concept C2 = C1<T> && true;
-  // new-note@-1 2{{similar constraint expression here}}
-
-  template <class T> concept D1 = true;
-  // new-note@-1 {{similar constraint expressions not considered equivalent}}
-
-  namespace t1 {
-    template<template<C1, class... T1s> class TT1> // new-note {{TT1' declared here}}
-    struct A {};
-    template<D1, class T2> struct B {}; // new-note {{'B' declared here}}
-    template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
-  } // namespace t1
-  namespace t2 {
-    template<template<C2, class... T1s> class TT1> struct A {};
-    template<C1, class T2> struct B {};
-    template struct A<B>;
-  } // namespace t2
-  namespace t3 {
-    template<template<C1, class... T1s> class TT1> // new-note {{'TT1' declared here}}
-    struct A {};
-    template<C2, class T2> struct B {}; // new-note {{'B' declared here}}
-    template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
-  } // namespace t2
-  namespace t4 {
-    // FIXME: This should be accepted.
-    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
-    struct A {};
-    template<C1 T2> struct B {}; // new-note {{'B' declared here}}
-    template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
-  } // namespace t4
-  namespace t5 {
-    // FIXME: This should be accepted
-    template<template<C2... T1s> class TT1> // new-note {{'TT1' declared here}}
-    struct A {};
-    template<C1 T2> struct B {}; // new-note {{'B' declared here}}
-    template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
-  } // namespace t5
-  namespace t6 {
-    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
-    struct A {};
-    template<C2 T2> struct B {}; // new-note {{'B' declared here}}
-    template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
-  } // namespace t6
-  namespace t7 {
-    template<template<class... T1s> class TT1>
-    struct A {};
-    template<C1 T2> struct B {};
-    template struct A<B>;
-  } // namespace t7
-  namespace t8 {
-    template<template<C1... T1s> class TT1>
-    struct A {};
-    template<class T2> struct B {};
-    template struct A<B>;
-  } // namespace t8
-  namespace t9 {
-    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
-    struct A {};
-    template<D1 T2> struct B {}; // new-note {{'B' declared here}}
-    template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
-  } // namespace t9
-  namespace t10 {
-    template<template<class...> requires C1<int> class TT1> // new-note {{'TT1' declared here}}
-    struct A {};
-
-    template<class> requires C2<int> struct B {}; // new-note {{'B' declared here}}
-    template struct A<B>;
-    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
-  } // namespace t10
-  namespace t11 {
-    template<template<class...> requires C2<int> class TT1> struct A {};
-    template<class> requires C1<int> struct B {};
-    template struct A<B>;
-  } // namespace t11
-} // namespace constraints
-
 namespace regression2 {
   template <class> struct D {};
 
diff --git a/clang/test/SemaTemplate/temp_arg_nontype.cpp b/clang/test/SemaTemplate/temp_arg_nontype.cpp
index 8b270b22a12b4612b235defd18b6bdda03ce3d6d..f360aa14950edd0e06499d22f6c2f704db0bc4a6 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype.cpp
@@ -387,11 +387,12 @@ namespace PR17696 {
 
 namespace partial_order_different_types {
   template<int, int, typename T, typename, T> struct A;
-  // expected-note@-1 {{template is declared here}}
-  template<int N, typename T, typename U, T V> struct A<0, N, T, U, V> {};
-  template<int N, typename T, typename U, U V> struct A<0, N, T, U, V>;
-  // expected-error@-1 {{class template partial specialization is not more specialized than the primary template}}
-  A<0, 0, int, int, 0> a;
+  template<int N, typename T, typename U, T V> struct A<0, N, T, U, V>; // expected-note {{matches}}
+  // FIXME: It appears that this partial specialization should be ill-formed as
+  // it is not more specialized than the primary template. V is not deducible
+  // because it does not have the same type as the corresponding parameter.
+  template<int N, typename T, typename U, U V> struct A<0, N, T, U, V> {}; // expected-note {{matches}}
+  A<0, 0, int, int, 0> a; // expected-error {{ambiguous}}
 }
 
 namespace partial_order_references {
@@ -457,24 +458,13 @@ namespace dependent_nested_partial_specialization {
 namespace nondependent_default_arg_ordering {
   int n, m;
   template<typename A, A B = &n> struct X {};
-
   template<typename A> void f(X<A>);
-  // expected-note@-1 {{candidate function}}
   template<typename A> void f(X<A, &m>);
-  // expected-note@-1 {{candidate function}}
   template<typename A, A B> void f(X<A, B>);
-  // expected-note@-1 2{{candidate function}}
   template<template<typename U, U> class T, typename A, int *B> void f(T<A, B>);
-  // expected-note@-1 2{{candidate function}}
-
-  // FIXME: When partial ordering, we get an inconsistent deduction between
-  // `A` (type-parameter-0-0) and `int *`, when deducing the first parameter.
-  // The deduction mechanism needs to be extended to be able to correctly
-  // handle these cases where the argument's template parameters appear in
-  // the result.
   void g() {
-    X<int *, &n> x; f(x); // expected-error {{call to 'f' is ambiguous}}
-    X<int *, &m> y; f(y); // expected-error {{call to 'f' is ambiguous}}
+    X<int *, &n> x; f(x);
+    X<int *, &m> y; f(y);
   }
 }
 
diff --git a/clang/test/SemaTemplate/temp_arg_template.cpp b/clang/test/SemaTemplate/temp_arg_template.cpp
index 9908af5e78669d95c64ec4581375950db51b37c3..a7236669276aa3d54fa4dbf73679436585cc5b37 100644
--- a/clang/test/SemaTemplate/temp_arg_template.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template.cpp
@@ -1,40 +1,33 @@
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,precxx17 %std_cxx98-14 %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx17 -std=c++17 %s
 
-template<template<typename T> class X> struct A; // #A
-// expected-note@-1 2{{previous template template parameter is here}}
+template<template<typename T> class X> struct A; // expected-note 2{{previous template template parameter is here}}
 
 template<template<typename T, int I> class X> struct B; // expected-note{{previous template template parameter is here}}
 
-template<template<int I> class X> struct C;
-// precxx17-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('int' vs 'const int &')}}
-// cxx17-error@-2 {{conversion from 'int' to 'const int &' in converted constant expression would bind reference to a temporary}}
-// expected-note@-3 {{previous template template parameter is here}}
+template<template<int I> class X> struct C;  // expected-note {{previous non-type template parameter with type 'int' is here}}
 
-template<class> struct X; // expected-note {{template is declared here}}
-template<int N> struct Y; // expected-note {{template parameter is declared here}}
+template<class> struct X; // expected-note{{too few template parameters in template template argument}}
+template<int N> struct Y; // expected-note{{template parameter has a different kind in template argument}}
 template<long N> struct Ylong;
-template<const int &N> struct Yref; // precxx17-note {{template parameter is declared here}}
+template<const int &N> struct Yref; // expected-note{{template non-type parameter has a different type 'const int &' in template argument}}
 
 namespace N {
   template<class> struct Z;
 }
-template<class, class> struct TooMany; // expected-note{{template is declared here}}
+template<class, class> struct TooMany; // expected-note{{too many template parameters in template template argument}}
 
 
 A<X> *a1;
 A<N::Z> *a2;
 A< ::N::Z> *a3;
 
-A<Y> *a4; // expected-error@#A {{template argument for non-type template parameter must be an expression}}
-          // expected-note@-1 {{different template parameters}}
-A<TooMany> *a5; // expected-error {{too few template arguments for class template 'TooMany'}}
-                // expected-note@-1 {{different template parameters}}
-B<X> *a6; // expected-error {{too many template arguments for class template 'X'}}
-          // expected-note@-1 {{different template parameters}}
+A<Y> *a4; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
+A<TooMany> *a5; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
+B<X> *a6; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
 C<Y> *a7;
 C<Ylong> *a8;
-C<Yref> *a9; // expected-note {{different template parameters}}
+C<Yref> *a9; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
 
 template<typename T> void f(int);
 
@@ -110,9 +103,9 @@ void foo() {
 
 namespace CheckDependentNonTypeParamTypes {
   template<template<typename T, typename U, T v> class X> struct A {
-    // expected-note@-1 {{previous template template parameter is here}}
     void f() {
-      X<int, void*, 3> x;
+      X<int, void*, 3> x; // precxx17-error {{does not refer to any declaration}} \
+                             cxx17-error {{value of type 'int' is not implicitly convertible to 'void *'}}
     }
     void g() {
       X<int, long, 3> x;
@@ -131,16 +124,15 @@ namespace CheckDependentNonTypeParamTypes {
     }
   };
 
-  template<typename T, typename U, U v> struct B {
-    // expected-error@-1 {{conflicting deduction 'U' against 'T' for parameter}}
+  template<typename T, typename U, U v> struct B { // precxx17-note {{parameter}}
     static const U value = v;
   };
 
   // FIXME: This should probably be rejected, but the rules are at best unclear.
-  A<B> ab; // expected-note {{different template parameters}}
+  A<B> ab;
 
   void use() {
-    ab.f();
+    ab.f(); // expected-note {{instantiation of}}
     ab.g();
     ab.h();
   }
diff --git a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
index d40577d52704684223d72304d4c0d9149dc25fc2..6f6568b9ab77641b0ac8ef4fcfd546b285aa9d23 100644
--- a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
@@ -1,16 +1,14 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
 
-// expected-note@temp_arg_template_p0522.cpp:* 1+{{template is declared here}}
-// expected-note@temp_arg_template_p0522.cpp:* 1+{{template parameter is declared here}}
-// expected-note@temp_arg_template_p0522.cpp:* 1+{{previous template template parameter is here}}
+// expected-note@temp_arg_template_p0522.cpp:* 1+{{}}
 
-template<template<int> typename> struct Ti; // #Ti
-template<template<int...> typename> struct TPi; // #TPi
+template<template<int> typename> struct Ti;
+template<template<int...> typename> struct TPi;
 template<template<int, int...> typename> struct TiPi;
 template<template<int..., int...> typename> struct TPiPi; // FIXME: Why is this not ill-formed?
 
-template<typename T, template<T> typename> struct tT0; // #tT0
-template<template<typename T, T> typename> struct Tt0; // #Tt0
+template<typename T, template<T> typename> struct tT0;
+template<template<typename T, T> typename> struct Tt0;
 
 template<template<typename> typename> struct Tt;
 template<template<typename, typename...> typename> struct TtPt;
@@ -21,8 +19,8 @@ template<int, int> struct ii;
 template<int...> struct Pi;
 template<int, int, int...> struct iiPi;
 
-template<int, typename = int> struct iDt; // #iDt
-template<int, typename> struct it; // #it
+template<int, typename = int> struct iDt;
+template<int, typename> struct it;
 
 template<typename T, T v> struct t0;
 
@@ -33,14 +31,10 @@ namespace IntParam {
         Ti<iDi>,
         Ti<Pi>,
         Ti<iDt>>;
-  using err1 = Ti<ii>; // expected-error {{too few template arguments for class template 'ii'}}
-                       // expected-note@-1 {{different template parameters}}
-  using err2 = Ti<iiPi>; // expected-error {{too few template arguments for class template 'iiPi'}}
-                         // expected-note@-1 {{different template parameters}}
-  using err3 = Ti<t0>; // expected-error@#Ti {{template argument for template type parameter must be a type}}
-                       // expected-note@-1 {{different template parameters}}
-  using err4 = Ti<it>; // expected-error {{too few template arguments for class template 'it'}}
-                       // expected-note@-1 {{different template parameters}}
+  using err1 = Ti<ii>; // expected-error {{different template parameters}}
+  using err2 = Ti<iiPi>; // expected-error {{different template parameters}}
+  using err3 = Ti<t0>; // expected-error {{different template parameters}}
+  using err4 = Ti<it>; // expected-error {{different template parameters}}
 }
 
 // These are accepted by the backwards-compatibility "parameter pack in
@@ -48,12 +42,9 @@ namespace IntParam {
 namespace IntPackParam {
   using ok = TPi<Pi>;
   using ok_compat = Pt<TPi<i>, TPi<iDi>, TPi<ii>, TPi<iiPi>>;
-  using err1 = TPi<t0>; // expected-error@#TPi {{template argument for template type parameter must be a type}}
-                        // expected-note@-1 {{different template parameters}}
-  using err2 = TPi<iDt>; // expected-error@#iDt {{could not match 'type-parameter-0-1' against}}
-                         // expected-note@-1 {{different template parameters}}
-  using err3 = TPi<it>; // expected-error@#it {{could not match 'type-parameter-0-1' against}}
-                        // expected-note@-1 {{different template parameters}}
+  using err1 = TPi<t0>; // expected-error {{different template parameters}}
+  using err2 = TPi<iDt>; // expected-error {{different template parameters}}
+  using err3 = TPi<it>; // expected-error {{different template parameters}}
 }
 
 namespace IntAndPackParam {
@@ -64,50 +55,42 @@ namespace IntAndPackParam {
 
 namespace DependentType {
   using ok = Pt<tT0<int, i>, tT0<int, iDi>>;
-  using err1 = tT0<int, ii>; // expected-error {{too few template arguments for class template 'ii'}}
-                             // expected-note@-1 {{different template parameters}}
+  using err1 = tT0<int, ii>; // expected-error {{different template parameters}}
   using err2 = tT0<short, i>; // FIXME: should this be OK?
   using err2a = tT0<long long, i>; // FIXME: should this be OK (if long long is larger than int)?
-  using err2b = tT0<void*, i>; // expected-error@#tT0 {{value of type 'void *' is not implicitly convertible to 'int'}}
-                               // expected-note@-1 {{different template parameters}}
-  using err3 = tT0<short, t0>; // expected-error@#tT0 {{template argument for template type parameter must be a type}}
-                               // expected-note@-1 {{different template parameters}}
+  using err2b = tT0<void*, i>; // expected-error {{different template parameters}}
+  using err3 = tT0<short, t0>; // expected-error {{different template parameters}}
 
   using ok2 = Tt0<t0>;
-  using err4 = Tt0<it>; // expected-error@#Tt0 {{template argument for non-type template parameter must be an expression}}
-                        // expected-note@-1 {{different template parameters}}
+  using err4 = Tt0<it>; // expected-error {{different template parameters}}
 }
 
 namespace Auto {
-  template<template<int> typename T> struct TInt {}; // #TInt
-  template<template<int*> typename T> struct TIntPtr {}; // #TIntPtr
+  template<template<int> typename T> struct TInt {};
+  template<template<int*> typename T> struct TIntPtr {};
   template<template<auto> typename T> struct TAuto {};
   template<template<auto*> typename T> struct TAutoPtr {};
   template<template<decltype(auto)> typename T> struct TDecltypeAuto {};
   template<auto> struct Auto;
-  template<auto*> struct AutoPtr; // #AutoPtr
+  template<auto*> struct AutoPtr;
   template<decltype(auto)> struct DecltypeAuto;
   template<int> struct Int;
   template<int*> struct IntPtr;
 
   TInt<Auto> ia;
-  TInt<AutoPtr> iap; // expected-error@#TInt {{non-type template parameter '' with type 'auto *' has incompatible initializer of type 'int'}}
-                     // expected-note@-1 {{different template parameters}}
+  TInt<AutoPtr> iap; // FIXME: ill-formed (?)
   TInt<DecltypeAuto> ida;
   TInt<Int> ii;
-  TInt<IntPtr> iip; // expected-error@#TInt {{conversion from 'int' to 'int *' is not allowed in a converted constant expression}}
-                    // expected-note@-1 {{different template parameters}}
+  TInt<IntPtr> iip; // expected-error {{different template parameters}}
 
   TIntPtr<Auto> ipa;
   TIntPtr<AutoPtr> ipap;
   TIntPtr<DecltypeAuto> ipda;
-  TIntPtr<Int> ipi; // expected-error@#TIntPtr {{value of type 'int *' is not implicitly convertible to 'int'}}
-                    // expected-note@-1 {{different template parameters}}
+  TIntPtr<Int> ipi; // expected-error {{different template parameters}}
   TIntPtr<IntPtr> ipip;
 
   TAuto<Auto> aa;
-  TAuto<AutoPtr> aap; // expected-error@#AutoPtr {{could not match 'auto *' against 'auto'}}
-                      // expected-note@-1 {{different template parameters}}
+  TAuto<AutoPtr> aap; // FIXME: ill-formed (?)
   TAuto<Int> ai; // FIXME: ill-formed (?)
   TAuto<IntPtr> aip; // FIXME: ill-formed (?)
 
@@ -128,8 +111,7 @@ namespace Auto {
   // parameters (such as 'user-defined-type &') that are not valid 'auto'
   // parameters.
   TDecltypeAuto<Auto> daa;
-  TDecltypeAuto<AutoPtr> daap; // expected-error@#AutoPtr {{could not match 'auto *' against 'decltype(auto)'}}
-                               // expected-note@-1 {{different template parameters}}
+  TDecltypeAuto<AutoPtr> daap; // FIXME: should probably be ill-formed
 
   int n;
   template<auto A, decltype(A) B = &n> struct SubstFailure;
@@ -146,7 +128,7 @@ namespace GH62529 {
 } // namespace GH62529
 
 namespace GH101394 {
-  struct X {}; // #X
+  struct X {};
   struct Y {
     constexpr Y(const X &) {}
   };
@@ -157,12 +139,8 @@ namespace GH101394 {
     template struct A<B>;
   } // namespace t1
   namespace t2 {
-    template<template<Y> class> struct A {}; // #A
-    template<X> struct B; // #B
-    template struct A<B>;
-    // expected-error@#A {{no viable conversion from 'const Y' to 'X'}}
-    // expected-note@-2  {{different template parameters}}
-    // expected-note@#X 2{{not viable}}
-    // expected-note@#B  {{passing argument to parameter here}}
+    template<template<Y> class> struct A {};
+    template<X> struct B;
+    template struct A<B>; // expected-error {{different template parameters}}
   } // namespace t2
 } // namespace GH101394
diff --git a/clang/test/Templight/templight-empty-entries-fix.cpp b/clang/test/Templight/templight-empty-entries-fix.cpp
index d13b748068efecc9ff7ed6a15002693003f2178f..e17be9012e59cc803313b1abfb7761aeab075b77 100644
--- a/clang/test/Templight/templight-empty-entries-fix.cpp
+++ b/clang/test/Templight/templight-empty-entries-fix.cpp
@@ -314,18 +314,6 @@ void foo() {
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
-// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
-// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
-// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
-// CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
diff --git a/clang/test/Templight/templight-prior-template-arg.cpp b/clang/test/Templight/templight-prior-template-arg.cpp
index 14bcb6a4d48f6bb5eff328e68e223b5f874a0f48..e9b1dd47bb603a56cfc41e8f5ae14e2ca26d0a07 100644
--- a/clang/test/Templight/templight-prior-template-arg.cpp
+++ b/clang/test/Templight/templight-prior-template-arg.cpp
@@ -10,76 +10,63 @@ class B {};
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:1'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:1'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B::Outer'$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:1'$}}
-//
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'B::Outer'$}}
-// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
-// CHECK: {{^event:[ ]+Begin$}}
-// CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:3'$}}
-// CHECK-LABEL: {{^---$}}
-// CHECK: {{^name:[ ]+'B::Outer'$}}
-// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
-// CHECK: {{^event:[ ]+End$}}
-// CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:3'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:1'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
 B<A> b;
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index e984d9ba8b4f0e9476c344f870e3ccee65e504d4..f1142aeddcc5ab11d0cc030da264e198c37483ad 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2921,6 +2921,11 @@ void OpenACCClauseEnqueue::VisitAsyncClause(const OpenACCAsyncClause &C) {
   if (C.hasIntExpr())
     Visitor.AddStmt(C.getIntExpr());
 }
+
+void OpenACCClauseEnqueue::VisitWorkerClause(const OpenACCWorkerClause &C) {
+  if (C.hasIntExpr())
+    Visitor.AddStmt(C.getIntExpr());
+}
 void OpenACCClauseEnqueue::VisitWaitClause(const OpenACCWaitClause &C) {
   if (const Expr *DevNumExpr = C.getDevNumExpr())
     Visitor.AddStmt(DevNumExpr);
@@ -2940,6 +2945,11 @@ void OpenACCClauseEnqueue::VisitSeqClause(const OpenACCSeqClause &C) {}
 void OpenACCClauseEnqueue::VisitCollapseClause(const OpenACCCollapseClause &C) {
   Visitor.AddStmt(C.getLoopCount());
 }
+void OpenACCClauseEnqueue::VisitGangClause(const OpenACCGangClause &C) {
+  for (unsigned I = 0; I < C.getNumExprs(); ++I) {
+    Visitor.AddStmt(C.getExpr(I).second);
+  }
+}
 } // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt
index 968b46acb784cbeaa8a30eaeb2b9440bc9bec952..00a1223c0831a7c9049f502dba730a08d0fb4475 100644
--- a/clang/tools/libclang/CMakeLists.txt
+++ b/clang/tools/libclang/CMakeLists.txt
@@ -167,6 +167,10 @@ if(ENABLE_SHARED)
       PROPERTIES
       VERSION ${LIBCLANG_LIBRARY_VERSION}
       DEFINE_SYMBOL _CINDEX_LIB_)
+      # Avoid declaring clang c++ symbols that are statically linked into libclang as dllimport'ed.
+      # If llvm/libclang-cpp dll is also being built for windows clang c++ symbols will still be
+      # implicitly be exported from libclang.
+      target_compile_definitions(libclang PRIVATE CLANG_BUILD_STATIC)
   elseif(APPLE)
     set(LIBCLANG_LINK_FLAGS " -Wl,-compatibility_version -Wl,1")
     set(LIBCLANG_LINK_FLAGS "${LIBCLANG_LINK_FLAGS} -Wl,-current_version -Wl,${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}")
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index 611e1f9ba5327c60150a56f4c2fde4ef3c7538a4..d696375547acce84a88b570fdb744a2de35a2202 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -2599,6 +2599,35 @@ TEST_P(ASTMatchersTest, HasName_MatchesInlinedNamespaces) {
   EXPECT_TRUE(matches(code, recordDecl(hasName("::a::C"))));
 }
 
+TEST_P(ASTMatchersTest, HasName_MatchesSpecializedInlinedNamespace) {
+  if (!GetParam().isCXX11OrLater()) {
+    return;
+  }
+
+  StringRef code = R"(
+namespace a {
+    inline namespace v1 {
+        template<typename T> T foo(T);
+    }
+}
+
+namespace a {
+    enum Tag{T1, T2};
+
+    template <Tag, typename T> T foo(T);
+}
+
+auto v1 = a::foo(1);
+auto v2 = a::foo<a::T1>(1);
+)";
+  EXPECT_TRUE(matches(
+      code, varDecl(hasName("v1"), hasDescendant(callExpr(callee(
+                                       functionDecl(hasName("::a::foo"))))))));
+  EXPECT_TRUE(matches(
+      code, varDecl(hasName("v2"), hasDescendant(callExpr(callee(
+                                       functionDecl(hasName("::a::foo"))))))));
+}
+
 TEST_P(ASTMatchersTest, HasName_MatchesAnonymousNamespaces) {
   if (!GetParam().isCXX()) {
     return;
diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp
index 9abb8a110bf8ee93ef585da950bee6956222c772..318f08c04759b9d9ab4bf4eee5a08c2899e5023a 100644
--- a/clang/unittests/Format/ConfigParseTest.cpp
+++ b/clang/unittests/Format/ConfigParseTest.cpp
@@ -183,7 +183,6 @@ TEST(ConfigParseTest, ParsesConfigurationBools) {
   CHECK_PARSE_BOOL(ObjCSpaceAfterProperty);
   CHECK_PARSE_BOOL(ObjCSpaceBeforeProtocolList);
   CHECK_PARSE_BOOL(Cpp11BracedListStyle);
-  CHECK_PARSE_BOOL(ReflowComments);
   CHECK_PARSE_BOOL(RemoveBracesLLVM);
   CHECK_PARSE_BOOL(RemoveSemicolon);
   CHECK_PARSE_BOOL(SkipMacroDefinitionBody);
@@ -381,6 +380,16 @@ TEST(ConfigParseTest, ParsesConfiguration) {
   CHECK_PARSE("PointerBindsToType: Middle", PointerAlignment,
               FormatStyle::PAS_Middle);
 
+  Style.ReflowComments = FormatStyle::RCS_Always;
+  CHECK_PARSE("ReflowComments: Never", ReflowComments, FormatStyle::RCS_Never);
+  CHECK_PARSE("ReflowComments: IndentOnly", ReflowComments,
+              FormatStyle::RCS_IndentOnly);
+  CHECK_PARSE("ReflowComments: Always", ReflowComments,
+              FormatStyle::RCS_Always);
+  // For backward compatibility:
+  CHECK_PARSE("ReflowComments: false", ReflowComments, FormatStyle::RCS_Never);
+  CHECK_PARSE("ReflowComments: true", ReflowComments, FormatStyle::RCS_Always);
+
   Style.Standard = FormatStyle::LS_Auto;
   CHECK_PARSE("Standard: c++03", Standard, FormatStyle::LS_Cpp03);
   CHECK_PARSE("Standard: c++11", Standard, FormatStyle::LS_Cpp11);
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 2c5e5857445c35738e29e0a47291c7d7a20df05a..43513f18321bc08c68b6bd2172cb11da9ddefbcd 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -18108,7 +18108,7 @@ TEST_F(FormatTest, AlignConsecutiveMacros) {
 
   // Test across comments
   Style.MaxEmptyLinesToKeep = 10;
-  Style.ReflowComments = false;
+  Style.ReflowComments = FormatStyle::RCS_Never;
   Style.AlignConsecutiveMacros.AcrossComments = true;
   verifyFormat("#define a    3\n"
                "// line comment\n"
@@ -18855,7 +18855,7 @@ TEST_F(FormatTest, AlignConsecutiveAssignmentsAcrossEmptyLinesAndComments) {
                "y = 1;",
                Alignment);
 
-  Alignment.ReflowComments = true;
+  Alignment.ReflowComments = FormatStyle::RCS_Always;
   Alignment.ColumnLimit = 50;
   verifyFormat("int x   = 0;\n"
                "int yy  = 1; /// specificlennospace\n"
@@ -19253,7 +19253,7 @@ TEST_F(FormatTest, AlignConsecutiveAssignments) {
                "y = 1;",
                Alignment);
 
-  EXPECT_EQ(Alignment.ReflowComments, true);
+  EXPECT_EQ(Alignment.ReflowComments, FormatStyle::RCS_Always);
   Alignment.ColumnLimit = 50;
   verifyFormat("int x   = 0;\n"
                "int yy  = 1; /// specificlennospace\n"
diff --git a/clang/unittests/Format/FormatTestComments.cpp b/clang/unittests/Format/FormatTestComments.cpp
index 4eea14282c32563c6d0ab9e373fe6dc54f15c666..5eefd767706a3eebfb757ce3c64b5654985add69 100644
--- a/clang/unittests/Format/FormatTestComments.cpp
+++ b/clang/unittests/Format/FormatTestComments.cpp
@@ -520,9 +520,36 @@ TEST_F(FormatTestComments, AlignsBlockComments) {
 
 TEST_F(FormatTestComments, CommentReflowingCanBeTurnedOff) {
   FormatStyle Style = getLLVMStyleWithColumns(20);
-  Style.ReflowComments = false;
+  Style.ReflowComments = FormatStyle::RCS_Never;
   verifyFormat("// aaaaaaaaa aaaaaaaaaa aaaaaaaaaa", Style);
   verifyFormat("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa */", Style);
+  verifyNoChange("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa\n"
+                 "aaaaaaaaa*/",
+                 Style);
+  verifyNoChange("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa\n"
+                 "    aaaaaaaaa*/",
+                 Style);
+  verifyNoChange("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa\n"
+                 " *    aaaaaaaaa*/",
+                 Style);
+}
+
+TEST_F(FormatTestComments, CommentReflowingCanApplyOnlyToIndents) {
+  FormatStyle Style = getLLVMStyleWithColumns(20);
+  Style.ReflowComments = FormatStyle::RCS_IndentOnly;
+  verifyFormat("// aaaaaaaaa aaaaaaaaaa aaaaaaaaaa", Style);
+  verifyFormat("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa */", Style);
+  verifyNoChange("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa\n"
+                 "aaaaaaaaa*/",
+                 Style);
+  verifyNoChange("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa\n"
+                 "    aaaaaaaaa*/",
+                 Style);
+  verifyFormat("/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa\n"
+               " * aaaaaaaaa*/",
+               "/* aaaaaaaaa aaaaaaaaaa aaaaaaaaaa\n"
+               "      * aaaaaaaaa*/",
+               Style);
 }
 
 TEST_F(FormatTestComments, CorrectlyHandlesLengthOfBlockComments) {
diff --git a/clang/unittests/Format/FormatTestVerilog.cpp b/clang/unittests/Format/FormatTestVerilog.cpp
index fbaf289fbc4d6df677686dec90f14d908bd487d9..49d276fc78d81b4f18e1023f99f01c2578579cf6 100644
--- a/clang/unittests/Format/FormatTestVerilog.cpp
+++ b/clang/unittests/Format/FormatTestVerilog.cpp
@@ -964,6 +964,7 @@ TEST_F(FormatTestVerilog, Instantiation) {
                "        .qbar(out1),\n"
                "        .clear(in1),\n"
                "        .preset(in2));");
+  verifyNoCrash(", ff1();");
   // With breaking between instance ports disabled.
   auto Style = getDefaultStyle();
   Style.VerilogBreakBetweenInstancePorts = false;
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index b78d0ab989e408eb6536ecc6093cb0c9cc4f5050..00776dac28a14b000e224ba0e92d2dd8b6a4c4c1 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -308,11 +308,6 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
 
-  Tokens = annotate("if (Foo *&foo{a})");
-  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
-  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
-  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
-
   FormatStyle Style = getLLVMStyle();
   Style.TypeNames.push_back("MYI");
   Tokens = annotate("if (MYI *p{nullptr})", Style);
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index e5da7f3c87a51967d7d8ecd7068cef0076ac6b85..c896b6124b6ab2e0c2244c8466d3bb5a7f856d2d 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -167,6 +167,87 @@ conformance.</p>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3274.pdf">N3274</a></td>
       <td class="full" align="center">Yes</td>
     </tr>
+    <!-- Oct 2024 Papers -->
+    <tr>
+      <td>Floating-point exceptions for macro replacements</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3286.pdf">N3286</a></td>
+      <td class="full" align="center">Yes</td>
+    </tr>
+    <tr>
+      <td>Obsolete implicitly octal literals and add delimited escape sequences</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3353.htm">N3353</a></td>
+      <td class="none" align="center">No</td>
+    </tr>
+    <tr>
+      <td>'if' declarations, v2</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3356.htm">N3356</a></td>
+      <td class="none" align="center">No</td>
+    </tr>
+    <tr>
+      <td>Allowing stricter alignment for atomic types</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3312.pdf">N3312</a></td>
+      <td class="full" align="center">Yes</td>
+    </tr>
+    <tr>
+      <td>Slay Some Earthly Demons III</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3341.pdf">N3341</a></td>
+      <td class="unknown" align="center">Unknown</td>
+    </tr>
+    <tr>
+      <td>Slay Some Earthly Demons IV</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3342.pdf">N3342</a></td>
+      <td class="unknown" align="center">Unknown</td>
+    </tr>
+    <tr>
+      <td>Slay Some Earthly Demons VI</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3344.pdf">N3344</a></td>
+      <td class="unknown" align="center">Unknown</td>
+    </tr>
+    <tr>
+      <td>Slay Some Earthly Demons VII</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3345.pdf">N3345</a></td>
+      <td class="full" align="center">Yes</td>
+    </tr>
+    <tr>
+      <td>Slay Some Earthly Demons VIII</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3346.pdf">N3346</a></td>
+      <td class="unknown" align="center">Unknown</td>
+    </tr>
+    <tr>
+      <td>Introduce complex literals v. 2</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3298.htm">N3298</a></td>
+      <td class="full" align="center">Yes</td>
+    </tr>
+    <tr>
+      <td>Allow zero length operations on null pointers</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3322.pdf">N3322</a></td>
+      <td class="full" align="center">Yes</td>
+    </tr>
+    <tr>
+      <td>How do you add one to something?</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3323.pdf">N3323</a></td>
+      <td class="full" align="center">Yes</td>
+    </tr>
+    <tr>
+      <td>Give consistent wording for SNAN initialization v3</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3364.pdf">N3364</a></td>
+      <td class="unknown" align="center">Unknown</td>
+    </tr>
+    <tr>
+      <td>Case range expressions v3.1</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3370.htm">N3370</a></td>
+      <td class="unknown" align="center">Unknown</td>
+    </tr>
+    <tr>
+      <td>New _Lengthof() operator (v4)</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3369.pdf">N3369</a></td>
+      <td class="none" align="center">No</td>
+    </tr>
+    <tr>
+      <td>Named loops, v3</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3355.htm">N3355</a></td>
+      <td class="none" align="center">No</td>
+    </tr>
 </table>
 </details>
 
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 1a67b6103cf43e6c3b0739812393371fb3c18f26..6f3cc8247d2e2d3e93d98654d904d9ea0d590b97 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -17128,11 +17128,11 @@ objects</td>
     <td>Undesired outcomes with <TT>const_cast</TT></td>
     <td align="center">Not resolved</td>
   </tr>
-  <tr class="open" id="2880">
+  <tr id="2880">
     <td><a href="https://cplusplus.github.io/CWG/issues/2880.html">2880</a></td>
-    <td>open</td>
+    <td>accepted</td>
     <td>Accessibility check for destructor of incomplete class type</td>
-    <td align="center">Not resolved</td>
+    <td class="unknown" align="center">Unknown</td>
   </tr>
   <tr id="2881">
     <td><a href="https://cplusplus.github.io/CWG/issues/2881.html">2881</a></td>
@@ -17260,7 +17260,7 @@ objects</td>
   </tr>
   <tr class="open" id="2901">
     <td><a href="https://cplusplus.github.io/CWG/issues/2901.html">2901</a></td>
-    <td>review</td>
+    <td>tentatively ready</td>
     <td>Unclear semantics for near-match aliased access</td>
     <td align="center">Not resolved</td>
   </tr>
@@ -17408,31 +17408,31 @@ objects</td>
   </tr>
   <tr class="open" id="2923">
     <td><a href="https://cplusplus.github.io/CWG/issues/2923.html">2923</a></td>
-    <td>open</td>
+    <td>tentatively ready</td>
     <td>Note about infinite loops and execution steps</td>
     <td align="center">Not resolved</td>
   </tr>
   <tr class="open" id="2924">
     <td><a href="https://cplusplus.github.io/CWG/issues/2924.html">2924</a></td>
-    <td>open</td>
+    <td>review</td>
     <td>Undefined behavior during constant evaluation</td>
     <td align="center">Not resolved</td>
   </tr>
-  <tr class="open" id="2925">
+  <tr id="2925">
     <td><a href="https://cplusplus.github.io/CWG/issues/2925.html">2925</a></td>
-    <td>open</td>
+    <td>NAD</td>
     <td>Deleting a pointer to an incomplete enumeration type</td>
-    <td align="center">Not resolved</td>
+    <td class="unknown" align="center">Unknown</td>
   </tr>
   <tr class="open" id="2926">
     <td><a href="https://cplusplus.github.io/CWG/issues/2926.html">2926</a></td>
-    <td>open</td>
+    <td>tentatively ready</td>
     <td>Lookup context for dependent qualified names</td>
     <td align="center">Not resolved</td>
   </tr>
   <tr class="open" id="2927">
     <td><a href="https://cplusplus.github.io/CWG/issues/2927.html">2927</a></td>
-    <td>open</td>
+    <td>review</td>
     <td>Unclear status of translation unit with <TT>module</TT> keyword</td>
     <td align="center">Not resolved</td>
   </tr>
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index deb6994f481854a996bbc019a14d2971b7d1a697..f6072aa40fad0d56e427d947f38afc20d7e945a4 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -12,6 +12,11 @@ endif()
 include(${LLVM_COMMON_CMAKE_UTILS}/Modules/CMakePolicy.cmake
   NO_POLICY_SCOPE)
 
+# TODO(CMake 3.22): remove
+if(POLICY CMP0128)
+  cmake_policy(SET CMP0128 NEW)
+endif()
+
 # Check if compiler-rt is built as a standalone project.
 if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR OR COMPILER_RT_STANDALONE_BUILD)
   project(CompilerRT C CXX ASM)
@@ -616,7 +621,7 @@ if(COMPILER_RT_USE_LLVM_UNWINDER)
   if (COMPILER_RT_ENABLE_STATIC_UNWINDER)
     list(APPEND COMPILER_RT_UNWINDER_LINK_LIBS "$<TARGET_LINKER_FILE:unwind_static>")
   else()
-    list(APPEND COMPILER_RT_UNWINDER_LINK_LIBS "$<TARGET_LINKER_FILE:$<IF:$<TARGET_EXISTS:unwind_shared>,unwind_shared,unwind_static>>")
+    list(APPEND COMPILER_RT_UNWINDER_LINK_LIBS "$<TARGET_LINKER_FILE:$<IF:$<BOOL:${LIBUNWIND_ENABLE_SHARED}>,unwind_shared,unwind_static>>")
   endif()
 endif()
 
@@ -629,7 +634,7 @@ if (COMPILER_RT_CXX_LIBRARY STREQUAL "libcxx")
   if (COMPILER_RT_STATIC_CXX_LIBRARY)
     set(COMPILER_RT_CXX_LINK_LIBS "$<TARGET_LINKER_FILE:cxx_static>")
   else()
-    set(COMPILER_RT_CXX_LINK_LIBS "$<TARGET_LINKER_FILE:$<IF:$<TARGET_EXISTS:cxx_shared>,cxx_shared,cxx_static>>")
+    set(COMPILER_RT_CXX_LINK_LIBS "$<TARGET_LINKER_FILE:$<IF:$<BOOL:${LIBCXX_ENABLE_SHARED}>,cxx_shared,cxx_static>>")
   endif()
 elseif (COMPILER_RT_CXX_LIBRARY STREQUAL "none")
   # We aren't using any C++ standard library so avoid including the default one.
@@ -671,7 +676,7 @@ if (SANITIZER_TEST_CXX_LIBNAME STREQUAL "libc++")
     if (SANITIZER_USE_STATIC_TEST_CXX)
       list(APPEND SANITIZER_TEST_CXX_LIBRARIES "$<TARGET_LINKER_FILE:cxx_static>")
     else()
-      list(APPEND SANITIZER_TEST_CXX_LIBRARIES "$<TARGET_LINKER_FILE:$<IF:$<TARGET_EXISTS:cxx_shared>,cxx_shared,cxx_static>>")
+      list(APPEND SANITIZER_TEST_CXX_LIBRARIES "$<TARGET_LINKER_FILE:$<IF:$<BOOL:${LIBCXX_ENABLE_SHARED}>,cxx_shared,cxx_static>>")
     endif()
     # We are using the in tree libc++ so avoid including the default one.
     append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ COMPILER_RT_UNITTEST_CFLAGS)
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index 6962b733733a6ab8d07554409849a49f7706d84c..e3d81d241b10542df96e2e0ec7a69248113d074e 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -161,7 +161,8 @@ endmacro()
 #                         LINK_LIBS <linked libraries> (only for shared library)
 #                         OBJECT_LIBS <object libraries to use as sources>
 #                         PARENT_TARGET <convenience parent target>
-#                         ADDITIONAL_HEADERS <header files>)
+#                         ADDITIONAL_HEADERS <header files>
+#                         EXTENSIONS <boolean>)
 function(add_compiler_rt_runtime name type)
   if(NOT type MATCHES "^(OBJECT|STATIC|SHARED|MODULE)$")
     message(FATAL_ERROR
@@ -171,7 +172,7 @@ function(add_compiler_rt_runtime name type)
   cmake_parse_arguments(LIB
     ""
     "PARENT_TARGET"
-    "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;DEPS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS"
+    "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;DEPS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS;EXTENSIONS"
     ${ARGN})
   set(libnames)
   # Until we support this some other way, build compiler-rt runtime without LTO
@@ -435,6 +436,10 @@ function(add_compiler_rt_runtime name type)
     if(type STREQUAL "SHARED")
       rt_externalize_debuginfo(${libname})
     endif()
+
+    if(DEFINED LIB_EXTENSIONS)
+      set_target_properties(${libname} PROPERTIES C_EXTENSIONS ${LIB_EXTENSIONS})
+    endif()
   endforeach()
   if(LIB_PARENT_TARGET)
     add_dependencies(${LIB_PARENT_TARGET} ${libnames})
diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index 809e9277156912e7100d4cc933dfb6543c6304dd..d00d39518104bf5de3252e11dc327260b87db3f8 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -77,7 +77,7 @@ set(ALL_HWASAN_SUPPORTED_ARCH ${X86_64} ${ARM64} ${RISCV64})
 set(ALL_MEMPROF_SUPPORTED_ARCH ${X86_64})
 set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC32} ${PPC64}
     ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON}
-    ${RISCV32} ${RISCV64} ${LOONGARCH64})
+    ${RISCV32} ${RISCV64} ${LOONGARCH64} ${WASM32})
 set(ALL_CTX_PROFILE_SUPPORTED_ARCH ${X86_64})
 if (OS_NAME MATCHES "FreeBSD")
   set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64})
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index a93a88a9205001fe09c49637ccf3b2dc464da130..a494e0532a50bca78e0cdd817b2ffe30a97ec64f 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -822,7 +822,7 @@ else()
 endif()
 
 if (PROFILE_SUPPORTED_ARCH AND NOT LLVM_USE_SANITIZER AND
-    OS_NAME MATCHES "Darwin|Linux|FreeBSD|Windows|Android|Fuchsia|SunOS|NetBSD|AIX")
+    OS_NAME MATCHES "Darwin|Linux|FreeBSD|Windows|Android|Fuchsia|SunOS|NetBSD|AIX|WASI")
   set(COMPILER_RT_HAS_PROFILE TRUE)
 else()
   set(COMPILER_RT_HAS_PROFILE FALSE)
diff --git a/compiler-rt/include/sanitizer/common_interface_defs.h b/compiler-rt/include/sanitizer/common_interface_defs.h
index f9fce595b37bb89764d90b087313385df56a5146..57313f9bc80e67467006402c954cd182de1d6367 100644
--- a/compiler-rt/include/sanitizer/common_interface_defs.h
+++ b/compiler-rt/include/sanitizer/common_interface_defs.h
@@ -193,6 +193,43 @@ void SANITIZER_CDECL __sanitizer_annotate_double_ended_contiguous_container(
     const void *old_container_beg, const void *old_container_end,
     const void *new_container_beg, const void *new_container_end);
 
+/// Copies memory annotations from a source storage region to a destination
+/// storage region. After the operation, the destination region has the same
+/// memory annotations as the source region, as long as sanitizer limitations
+/// allow it (more bytes may be unpoisoned than in the source region, resulting
+/// in more false negatives, but never false positives). If the source and
+/// destination regions overlap, only the minimal required changes are made to
+/// preserve the correct annotations. Old storage bytes that are not in the new
+/// storage should have the same annotations, as long as sanitizer limitations
+/// allow it.
+///
+/// This function is primarily designed to be used when moving trivially
+/// relocatable objects that may have poisoned memory, making direct copying
+/// problematic under sanitizer. However, this function does not move memory
+/// content itself, only annotations.
+///
+/// A contiguous container is a container that keeps all of its elements in a
+/// contiguous region of memory. The container owns the region of memory
+/// <c>[src_begin, src_end)</c> and <c>[dst_begin, dst_end)</c>. The memory
+/// within these regions may be alternately poisoned and non-poisoned, with
+/// possibly smaller poisoned and unpoisoned regions.
+///
+/// If this function fully poisons a granule, it is marked as "container
+/// overflow".
+///
+/// Argument requirements: The destination container must have the same size as
+/// the source container, which is inferred from the beginning and end of the
+/// source region. Addresses may be granule-unaligned, but this may affect
+/// performance.
+///
+/// \param src_begin Begin of the source container region.
+/// \param src_end End of the source container region.
+/// \param dst_begin Begin of the destination container region.
+/// \param dst_end End of the destination container region.
+void SANITIZER_CDECL __sanitizer_copy_contiguous_container_annotations(
+    const void *src_begin, const void *src_end, const void *dst_begin,
+    const void *dst_end);
+
 /// Returns true if the contiguous container <c>[beg, end)</c> is properly
 /// poisoned.
 ///
diff --git a/compiler-rt/lib/asan/asan_errors.cpp b/compiler-rt/lib/asan/asan_errors.cpp
index 6f2fd28bfdf11a667e99f7466fd9cb3f5ea58407..4f112cc5d1bca5514680ab4cf6b492237500b872 100644
--- a/compiler-rt/lib/asan/asan_errors.cpp
+++ b/compiler-rt/lib/asan/asan_errors.cpp
@@ -348,6 +348,20 @@ void ErrorBadParamsToAnnotateDoubleEndedContiguousContainer::Print() {
   ReportErrorSummary(scariness.GetDescription(), stack);
 }
 
+void ErrorBadParamsToCopyContiguousContainerAnnotations::Print() {
+  Report(
+      "ERROR: AddressSanitizer: bad parameters to "
+      "__sanitizer_copy_contiguous_container_annotations:\n"
+      "      src_storage_beg : %p\n"
+      "      src_storage_end : %p\n"
+      "      dst_storage_beg : %p\n"
+      "      new_storage_end : %p\n",
+      (void *)old_storage_beg, (void *)old_storage_end, (void *)new_storage_beg,
+      (void *)new_storage_end);
+  stack->Print();
+  ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
 void ErrorODRViolation::Print() {
   Decorator d;
   Printf("%s", d.Error());
diff --git a/compiler-rt/lib/asan/asan_errors.h b/compiler-rt/lib/asan/asan_errors.h
index 634f6da544355233e05ef0948f93461f96f1fce7..b3af655e66639a90079f59397a4d542eff50d0e6 100644
--- a/compiler-rt/lib/asan/asan_errors.h
+++ b/compiler-rt/lib/asan/asan_errors.h
@@ -353,6 +353,24 @@ struct ErrorBadParamsToAnnotateDoubleEndedContiguousContainer : ErrorBase {
   void Print();
 };
 
+struct ErrorBadParamsToCopyContiguousContainerAnnotations : ErrorBase {
+  const BufferedStackTrace *stack;
+  uptr old_storage_beg, old_storage_end, new_storage_beg, new_storage_end;
+
+  ErrorBadParamsToCopyContiguousContainerAnnotations() = default;  // (*)
+  ErrorBadParamsToCopyContiguousContainerAnnotations(
+      u32 tid, BufferedStackTrace *stack_, uptr old_storage_beg_,
+      uptr old_storage_end_, uptr new_storage_beg_, uptr new_storage_end_)
+      : ErrorBase(tid, 10,
+                  "bad-__sanitizer_annotate_double_ended_contiguous_container"),
+        stack(stack_),
+        old_storage_beg(old_storage_beg_),
+        old_storage_end(old_storage_end_),
+        new_storage_beg(new_storage_beg_),
+        new_storage_end(new_storage_end_) {}
+  void Print();
+};
+
 struct ErrorODRViolation : ErrorBase {
   __asan_global global1, global2;
   u32 stack_id1, stack_id2;
@@ -421,6 +439,7 @@ struct ErrorGeneric : ErrorBase {
   macro(StringFunctionSizeOverflow)                        \
   macro(BadParamsToAnnotateContiguousContainer)            \
   macro(BadParamsToAnnotateDoubleEndedContiguousContainer) \
+  macro(BadParamsToCopyContiguousContainerAnnotations)     \
   macro(ODRViolation)                                      \
   macro(InvalidPointerPair)                                \
   macro(Generic)
diff --git a/compiler-rt/lib/asan/asan_fuchsia.cpp b/compiler-rt/lib/asan/asan_fuchsia.cpp
index dbc4342e83388cfb3504b5dc109881c9dfd5f104..96c41e9d42ba6a2b65b25fe3841cfea4a05c5b5e 100644
--- a/compiler-rt/lib/asan/asan_fuchsia.cpp
+++ b/compiler-rt/lib/asan/asan_fuchsia.cpp
@@ -121,8 +121,7 @@ static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
   // In lieu of AsanThread::Create.
   AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__);
 
-  AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
-  u32 tid = asanThreadRegistry().CreateThread(0, detached, parent_tid, &args);
+  u32 tid = asanThreadRegistry().CreateThread(0, detached, parent_tid, thread);
   asanThreadRegistry().SetThreadName(tid, name);
 
   return thread;
diff --git a/compiler-rt/lib/asan/asan_poisoning.cpp b/compiler-rt/lib/asan/asan_poisoning.cpp
index d600b1a0c241fa2fec53592dd8b05461a3c95ba3..762670632f4e0fbd9cce5db883fc06ee42646c3c 100644
--- a/compiler-rt/lib/asan/asan_poisoning.cpp
+++ b/compiler-rt/lib/asan/asan_poisoning.cpp
@@ -16,6 +16,7 @@
 #include "asan_report.h"
 #include "asan_stack.h"
 #include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_interface_internal.h"
 #include "sanitizer_common/sanitizer_libc.h"
@@ -576,6 +577,185 @@ void __sanitizer_annotate_double_ended_contiguous_container(
   }
 }
 
+// Marks the specified number of bytes in a granule as accessible or
+// poisones the whole granule with kAsanContiguousContainerOOBMagic value.
+static void SetContainerGranule(uptr ptr, u8 n) {
+  constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+  u8 s = (n == granularity) ? 0 : (n ? n : kAsanContiguousContainerOOBMagic);
+  *(u8 *)MemToShadow(ptr) = s;
+}
+
+// Performs a byte-by-byte copy of ASan annotations (shadow memory values).
+// Result may be different due to ASan limitations, but result cannot lead
+// to false positives (more memory than requested may get unpoisoned).
+static void SlowCopyContainerAnnotations(uptr src_beg, uptr src_end,
+                                         uptr dst_beg, uptr dst_end) {
+  constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+  uptr dst_end_down = RoundDownTo(dst_end, granularity);
+  uptr src_ptr = src_beg;
+  uptr dst_ptr = dst_beg;
+
+  while (dst_ptr < dst_end) {
+    uptr granule_beg = RoundDownTo(dst_ptr, granularity);
+    uptr granule_end = granule_beg + granularity;
+    uptr unpoisoned_bytes = 0;
+
+    uptr end = Min(granule_end, dst_end);
+    for (; dst_ptr != end; ++dst_ptr, ++src_ptr)
+      if (!AddressIsPoisoned(src_ptr))
+        unpoisoned_bytes = dst_ptr - granule_beg + 1;
+
+    if (dst_ptr == dst_end && dst_end != dst_end_down &&
+        !AddressIsPoisoned(dst_end))
+      continue;
+
+    if (unpoisoned_bytes != 0 || granule_beg >= dst_beg)
+      SetContainerGranule(granule_beg, unpoisoned_bytes);
+    else if (!AddressIsPoisoned(dst_beg))
+      SetContainerGranule(granule_beg, dst_beg - granule_beg);
+  }
+}
+
+// Performs a byte-by-byte copy of ASan annotations (shadow memory values),
+// going through bytes in reversed order, but not reversing annotations.
+// Result may be different due to ASan limitations, but result cannot lead
+// to false positives (more memory than requested may get unpoisoned).
+static void SlowReversedCopyContainerAnnotations(uptr src_beg, uptr src_end,
+                                                 uptr dst_beg, uptr dst_end) {
+  constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+  uptr dst_end_down = RoundDownTo(dst_end, granularity);
+  uptr src_ptr = src_end;
+  uptr dst_ptr = dst_end;
+
+  while (dst_ptr > dst_beg) {
+    uptr granule_beg = RoundDownTo(dst_ptr - 1, granularity);
+    uptr unpoisoned_bytes = 0;
+
+    uptr end = Max(granule_beg, dst_beg);
+    for (; dst_ptr != end; --dst_ptr, --src_ptr)
+      if (unpoisoned_bytes == 0 && !AddressIsPoisoned(src_ptr - 1))
+        unpoisoned_bytes = dst_ptr - granule_beg;
+
+    if (dst_ptr >= dst_end_down && !AddressIsPoisoned(dst_end))
+      continue;
+
+    if (granule_beg == dst_ptr || unpoisoned_bytes != 0)
+      SetContainerGranule(granule_beg, unpoisoned_bytes);
+    else if (!AddressIsPoisoned(dst_beg))
+      SetContainerGranule(granule_beg, dst_beg - granule_beg);
+  }
+}
+
+// A helper function for __sanitizer_copy_contiguous_container_annotations,
+// has assumption about begin and end of the container.
+// Should not be used stand alone.
+static void CopyContainerFirstGranuleAnnotation(uptr src_beg, uptr dst_beg) {
+  constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+  // First granule
+  uptr src_beg_down = RoundDownTo(src_beg, granularity);
+  uptr dst_beg_down = RoundDownTo(dst_beg, granularity);
+  if (dst_beg_down == dst_beg)
+    return;
+  if (!AddressIsPoisoned(src_beg))
+    *(u8 *)MemToShadow(dst_beg_down) = *(u8 *)MemToShadow(src_beg_down);
+  else if (!AddressIsPoisoned(dst_beg))
+    SetContainerGranule(dst_beg_down, dst_beg - dst_beg_down);
+}
+
+// A helper function for __sanitizer_copy_contiguous_container_annotations,
+// has assumption about begin and end of the container.
+// Should not be used stand alone.
+static void CopyContainerLastGranuleAnnotation(uptr src_end, uptr dst_end) {
+  constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+  // Last granule
+  uptr src_end_down = RoundDownTo(src_end, granularity);
+  uptr dst_end_down = RoundDownTo(dst_end, granularity);
+  if (dst_end_down == dst_end || !AddressIsPoisoned(dst_end))
+    return;
+  if (AddressIsPoisoned(src_end))
+    *(u8 *)MemToShadow(dst_end_down) = *(u8 *)MemToShadow(src_end_down);
+  else
+    SetContainerGranule(dst_end_down, src_end - src_end_down);
+}
+
+// This function copies ASan memory annotations (poisoned/unpoisoned states)
+// from one buffer to another.
+// It's main purpose is to help with relocating trivially relocatable objects,
+// which memory may be poisoned, without calling copy constructor.
+// However, it does not move memory content itself, only annotations.
+// If the buffers aren't aligned (the distance between buffers isn't
+// granule-aligned)
+//     // src_beg % granularity != dst_beg % granularity
+// the function handles this by going byte by byte, slowing down performance.
+// The old buffer annotations are not removed. If necessary,
+// user can unpoison old buffer with __asan_unpoison_memory_region.
+void __sanitizer_copy_contiguous_container_annotations(const void *src_beg_p,
+                                                       const void *src_end_p,
+                                                       const void *dst_beg_p,
+                                                       const void *dst_end_p) {
+  if (!flags()->detect_container_overflow)
+    return;
+
+  VPrintf(3, "contiguous_container_src: %p %p\n", src_beg_p, src_end_p);
+  VPrintf(3, "contiguous_container_dst: %p %p\n", dst_beg_p, dst_end_p);
+
+  uptr src_beg = reinterpret_cast<uptr>(src_beg_p);
+  uptr src_end = reinterpret_cast<uptr>(src_end_p);
+  uptr dst_beg = reinterpret_cast<uptr>(dst_beg_p);
+  uptr dst_end = reinterpret_cast<uptr>(dst_end_p);
+
+  constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+
+  if (src_beg > src_end || (dst_end - dst_beg) != (src_end - src_beg)) {
+    GET_STACK_TRACE_FATAL_HERE;
+    ReportBadParamsToCopyContiguousContainerAnnotations(
+        src_beg, src_end, dst_beg, dst_end, &stack);
+  }
+
+  if (src_beg == src_end || src_beg == dst_beg)
+    return;
+  // Due to support for overlapping buffers, we may have to copy elements
+  // in reversed order, when destination buffer starts in the middle of
+  // the source buffer (or shares first granule with it).
+  //
+  // When buffers are not granule-aligned (or distance between them,
+  // to be specific), annotatios have to be copied byte by byte.
+  //
+  // The only remaining edge cases involve edge granules,
+  // when the container starts or ends within a granule.
+  uptr src_beg_up = RoundUpTo(src_beg, granularity);
+  uptr src_end_up = RoundUpTo(src_end, granularity);
+  bool copy_in_reversed_order = src_beg < dst_beg && dst_beg <= src_end_up;
+  if (src_beg % granularity != dst_beg % granularity ||
+      RoundDownTo(dst_end - 1, granularity) <= dst_beg) {
+    if (copy_in_reversed_order)
+      SlowReversedCopyContainerAnnotations(src_beg, src_end, dst_beg, dst_end);
+    else
+      SlowCopyContainerAnnotations(src_beg, src_end, dst_beg, dst_end);
+    return;
+  }
+
+  // As buffers are granule-aligned, we can just copy annotations of granules
+  // from the middle.
+  uptr dst_beg_up = RoundUpTo(dst_beg, granularity);
+  uptr dst_end_down = RoundDownTo(dst_end, granularity);
+  if (copy_in_reversed_order)
+    CopyContainerLastGranuleAnnotation(src_end, dst_end);
+  else
+    CopyContainerFirstGranuleAnnotation(src_beg, dst_beg);
+
+  if (dst_beg_up < dst_end_down) {
+    internal_memmove((u8 *)MemToShadow(dst_beg_up),
+                     (u8 *)MemToShadow(src_beg_up),
+                     (dst_end_down - dst_beg_up) / granularity);
+  }
+
+  if (copy_in_reversed_order)
+    CopyContainerFirstGranuleAnnotation(src_beg, dst_beg);
+  else
+    CopyContainerLastGranuleAnnotation(src_end, dst_end);
+}
+
 static const void *FindBadAddress(uptr begin, uptr end, bool poisoned) {
   CHECK_LE(begin, end);
   constexpr uptr kMaxRangeToCheck = 32;
diff --git a/compiler-rt/lib/asan/asan_report.cpp b/compiler-rt/lib/asan/asan_report.cpp
index fd590e401f67fe879e0aae0dcef59ed5af0fded2..45aa607dcda07fd9e0da254345aeff7262942bbb 100644
--- a/compiler-rt/lib/asan/asan_report.cpp
+++ b/compiler-rt/lib/asan/asan_report.cpp
@@ -367,6 +367,16 @@ void ReportBadParamsToAnnotateDoubleEndedContiguousContainer(
   in_report.ReportError(error);
 }
 
+void ReportBadParamsToCopyContiguousContainerAnnotations(
+    uptr old_storage_beg, uptr old_storage_end, uptr new_storage_beg,
+    uptr new_storage_end, BufferedStackTrace *stack) {
+  ScopedInErrorReport in_report;
+  ErrorBadParamsToCopyContiguousContainerAnnotations error(
+      GetCurrentTidOrInvalid(), stack, old_storage_beg, old_storage_end,
+      new_storage_beg, new_storage_end);
+  in_report.ReportError(error);
+}
+
 void ReportODRViolation(const __asan_global *g1, u32 stack_id1,
                         const __asan_global *g2, u32 stack_id2) {
   ScopedInErrorReport in_report;
diff --git a/compiler-rt/lib/asan/asan_report.h b/compiler-rt/lib/asan/asan_report.h
index 3540b3b4b1bfe0d7361730793939455dc47f6719..3143d83abe390dacd16b1318206d3452d39e6c94 100644
--- a/compiler-rt/lib/asan/asan_report.h
+++ b/compiler-rt/lib/asan/asan_report.h
@@ -88,6 +88,9 @@ void ReportBadParamsToAnnotateDoubleEndedContiguousContainer(
     uptr storage_beg, uptr storage_end, uptr old_container_beg,
     uptr old_container_end, uptr new_container_beg, uptr new_container_end,
     BufferedStackTrace *stack);
+void ReportBadParamsToCopyContiguousContainerAnnotations(
+    uptr old_storage_beg, uptr old_storage_end, uptr new_storage_beg,
+    uptr new_storage_end, BufferedStackTrace *stack);
 
 void ReportODRViolation(const __asan_global *g1, u32 stack_id1,
                         const __asan_global *g2, u32 stack_id2);
diff --git a/compiler-rt/lib/asan/asan_thread.cpp b/compiler-rt/lib/asan/asan_thread.cpp
index c1a804b9fcccd33c5849896f1f369d946812ba64..37fb6f2b07f276eff388dd72ad3024affac807d0 100644
--- a/compiler-rt/lib/asan/asan_thread.cpp
+++ b/compiler-rt/lib/asan/asan_thread.cpp
@@ -21,6 +21,7 @@
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_thread_history.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 namespace __asan {
@@ -28,10 +29,7 @@ namespace __asan {
 // AsanThreadContext implementation.
 
 void AsanThreadContext::OnCreated(void *arg) {
-  CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs *>(arg);
-  if (args->stack)
-    stack_id = StackDepotPut(*args->stack);
-  thread = args->thread;
+  thread = static_cast<AsanThread *>(arg);
   thread->set_context(this);
 }
 
@@ -106,8 +104,8 @@ AsanThread *AsanThread::Create(const void *start_data, uptr data_size,
     CHECK_LE(data_size, availible_size);
     internal_memcpy(thread->start_data_, start_data, data_size);
   }
-  AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
-  asanThreadRegistry().CreateThread(0, detached, parent_tid, &args);
+  asanThreadRegistry().CreateThread(0, detached, parent_tid,
+                                    stack ? StackDepotPut(*stack) : 0, thread);
 
   return thread;
 }
@@ -558,6 +556,12 @@ void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {
       threads);
 }
 
+void PrintThreads() {
+  InternalScopedString out;
+  PrintThreadHistory(__asan::asanThreadRegistry(), out);
+  Report("%s\n", out.data());
+}
+
 }  // namespace __lsan
 
 // ---------------------- Interface ---------------- {{{1
diff --git a/compiler-rt/lib/asan/asan_thread.h b/compiler-rt/lib/asan/asan_thread.h
index 62f1b5337fe4bf3b57a68a694743e2a39638c180..ad9e03d68fe96a0bd5bf64ca498d72a34cc8976b 100644
--- a/compiler-rt/lib/asan/asan_thread.h
+++ b/compiler-rt/lib/asan/asan_thread.h
@@ -36,21 +36,16 @@ class AsanThread;
 class AsanThreadContext final : public ThreadContextBase {
  public:
   explicit AsanThreadContext(int tid)
-      : ThreadContextBase(tid), announced(false),
-        destructor_iterations(GetPthreadDestructorIterations()), stack_id(0),
+      : ThreadContextBase(tid),
+        announced(false),
+        destructor_iterations(GetPthreadDestructorIterations()),
         thread(nullptr) {}
   bool announced;
   u8 destructor_iterations;
-  u32 stack_id;
   AsanThread *thread;
 
   void OnCreated(void *arg) override;
   void OnFinished() override;
-
-  struct CreateThreadContextArgs {
-    AsanThread *thread;
-    StackTrace *stack;
-  };
 };
 
 // AsanThreadContext objects are never freed, so we need many of them.
diff --git a/compiler-rt/lib/builtins/cpu_model/riscv.c b/compiler-rt/lib/builtins/cpu_model/riscv.c
index 85b24ab8f43bc1875c9685e39469d246316e2193..052124fdde447ef18ca05989d49bb8b362ca9fa5 100644
--- a/compiler-rt/lib/builtins/cpu_model/riscv.c
+++ b/compiler-rt/lib/builtins/cpu_model/riscv.c
@@ -22,8 +22,8 @@ struct {
 
 struct {
   unsigned mvendorid;
-  unsigned marchid;
-  unsigned mimpid;
+  unsigned long long marchid;
+  unsigned long long mimpid;
 } __riscv_cpu_model __attribute__((visibility("hidden"), nocommon));
 
 // NOTE: Should sync-up with RISCVFeatures.td
diff --git a/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h b/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
index d92b51052194275d41bcfdbb25a2a78a43cf7561..e8011014c2331d7ce9de7832daa1a5a84a2de04e 100644
--- a/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
+++ b/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
@@ -200,6 +200,9 @@
 #undef SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID
 #define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID 0
 
+#undef SANITIZER_INTERCEPT_TIMER_CREATE
+#define SANITIZER_INTERCEPT_TIMER_CREATE 0
+
 #undef SANITIZER_INTERCEPT_GETITIMER
 #define SANITIZER_INTERCEPT_GETITIMER 0
 
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp
index 3e14a718513d7f3d1c26cb2a59bfd6168a16930c..8b32e4e760e2fa3df37bf18b49e09a446da4c377 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp
@@ -218,6 +218,11 @@ void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) {
   __hwasan::hwasanThreadArgRetval().GetAllPtrsLocked(ptrs);
 }
 
-void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {}
+void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {
+  // TODO: implement.
+}
+void PrintThreads() {
+  // TODO: implement.
+}
 
 }  // namespace __lsan
diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp
index 6776598651ae9b85aed1e8724e3003b6bfa96d1d..c05e0dd0a9332da405de368ca19fcdb416278c48 100644
--- a/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/compiler-rt/lib/lsan/lsan_common.cpp
@@ -771,14 +771,15 @@ static bool PrintResults(LeakReport &report) {
   }
   if (common_flags()->print_suppressions)
     GetSuppressionContext()->PrintMatchedSuppressions();
-  if (unsuppressed_count > 0) {
+  if (unsuppressed_count)
     report.PrintSummary();
-    return true;
-  }
-  return false;
+  if ((unsuppressed_count && common_flags()->verbosity >= 2) ||
+      flags()->log_threads)
+    PrintThreads();
+  return unsuppressed_count;
 }
 
-static bool CheckForLeaks() {
+static bool CheckForLeaksOnce() {
   if (&__lsan_is_turned_off && __lsan_is_turned_off()) {
     VReport(1, "LeakSanitizer is disabled\n");
     return false;
@@ -830,6 +831,12 @@ static bool CheckForLeaks() {
   }
 }
 
+static bool CheckForLeaks() {
+  int leaking_tries = 0;
+  for (int i = 0; i < flags()->tries; ++i) leaking_tries += CheckForLeaksOnce();
+  return leaking_tries == flags()->tries;
+}
+
 static bool has_reported_leaks = false;
 bool HasReportedLeaks() { return has_reported_leaks; }
 
diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h
index c598b62105873e36bde865f71f2c510cbbff431b..f990c7850497a53f6b5585d863c87c3d93da7d93 100644
--- a/compiler-rt/lib/lsan/lsan_common.h
+++ b/compiler-rt/lib/lsan/lsan_common.h
@@ -111,6 +111,7 @@ void GetThreadExtraStackRangesLocked(tid_t os_id,
                                      InternalMmapVector<Range> *ranges);
 void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs);
 void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads);
+void PrintThreads();
 
 //// --------------------------------------------------------------------------
 //// Allocator prototypes.
diff --git a/compiler-rt/lib/lsan/lsan_flags.inc b/compiler-rt/lib/lsan/lsan_flags.inc
index b7f28223b8189b76883632ea4a682c8a52b88bc0..c97b021ba5c02fc3a0924b32238f5a52c24317b3 100644
--- a/compiler-rt/lib/lsan/lsan_flags.inc
+++ b/compiler-rt/lib/lsan/lsan_flags.inc
@@ -43,6 +43,7 @@ LSAN_FLAG(bool, use_poisoned, false,
           "Consider pointers found in poisoned memory to be valid.")
 LSAN_FLAG(bool, log_pointers, false, "Debug logging")
 LSAN_FLAG(bool, log_threads, false, "Debug logging")
+LSAN_FLAG(int, tries, 1, "Debug option to repeat leak checking multiple times")
 LSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
 LSAN_FLAG(int, thread_suspend_fail, 1,
           "Behaviour if thread suspendion all thread (0 - "
diff --git a/compiler-rt/lib/lsan/lsan_thread.cpp b/compiler-rt/lib/lsan/lsan_thread.cpp
index 07c7b923623fa95326682fc95bb8abb70dbb6e6e..b66ea61a2de4e54ac5316565481221c33b5ae505 100644
--- a/compiler-rt/lib/lsan/lsan_thread.cpp
+++ b/compiler-rt/lib/lsan/lsan_thread.cpp
@@ -18,6 +18,7 @@
 #include "lsan_common.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_thread_history.h"
 #include "sanitizer_common/sanitizer_thread_registry.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
@@ -109,6 +110,12 @@ void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {
       threads);
 }
 
+void PrintThreads() {
+  InternalScopedString out;
+  PrintThreadHistory(*thread_registry, out);
+  Report("%s\n", out.data());
+}
+
 void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) {
   GetThreadArgRetval().GetAllPtrsLocked(ptrs);
 }
diff --git a/compiler-rt/lib/memprof/memprof_thread.cpp b/compiler-rt/lib/memprof/memprof_thread.cpp
index 50072bb91ee74cc8f84104f723acf7015fe6bd4b..4b9665ffc3fcea58d4151e87ad595730fd8f2850 100644
--- a/compiler-rt/lib/memprof/memprof_thread.cpp
+++ b/compiler-rt/lib/memprof/memprof_thread.cpp
@@ -25,10 +25,7 @@ namespace __memprof {
 // MemprofThreadContext implementation.
 
 void MemprofThreadContext::OnCreated(void *arg) {
-  CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs *>(arg);
-  if (args->stack)
-    stack_id = StackDepotPut(*args->stack);
-  thread = args->thread;
+  thread = static_cast<MemprofThread *>(arg);
   thread->set_context(this);
 }
 
@@ -79,8 +76,8 @@ MemprofThread *MemprofThread::Create(thread_callback_t start_routine, void *arg,
   MemprofThread *thread = (MemprofThread *)MmapOrDie(size, __func__);
   thread->start_routine_ = start_routine;
   thread->arg_ = arg;
-  MemprofThreadContext::CreateThreadContextArgs args = {thread, stack};
-  memprofThreadRegistry().CreateThread(0, detached, parent_tid, &args);
+  memprofThreadRegistry().CreateThread(
+      0, detached, parent_tid, stack ? StackDepotPut(*stack) : 0, thread);
 
   return thread;
 }
diff --git a/compiler-rt/lib/memprof/memprof_thread.h b/compiler-rt/lib/memprof/memprof_thread.h
index 4c9313fcb369eb702bbcd578e98fa251bb61ca9d..fb90dbf328a430dc2fc875d230c4e996f78d3521 100644
--- a/compiler-rt/lib/memprof/memprof_thread.h
+++ b/compiler-rt/lib/memprof/memprof_thread.h
@@ -34,20 +34,14 @@ class MemprofThread;
 struct MemprofThreadContext final : public ThreadContextBase {
   explicit MemprofThreadContext(int tid)
       : ThreadContextBase(tid), announced(false),
-        destructor_iterations(GetPthreadDestructorIterations()), stack_id(0),
+        destructor_iterations(GetPthreadDestructorIterations()),
         thread(nullptr) {}
   bool announced;
   u8 destructor_iterations;
-  u32 stack_id;
   MemprofThread *thread;
 
   void OnCreated(void *arg) override;
   void OnFinished() override;
-
-  struct CreateThreadContextArgs {
-    MemprofThread *thread;
-    StackTrace *stack;
-  };
 };
 
 // MemprofThreadContext objects are never freed, so we need many of them.
diff --git a/compiler-rt/lib/msan/tests/msan_test.cpp b/compiler-rt/lib/msan/tests/msan_test.cpp
index 41b99fabe84f4789d3ae179ed3f3bb4aefb812c5..ad265acf4c1e39a703c32f730c67b283efc29664 100644
--- a/compiler-rt/lib/msan/tests/msan_test.cpp
+++ b/compiler-rt/lib/msan/tests/msan_test.cpp
@@ -4881,4 +4881,27 @@ TEST(MemorySanitizer, throw_catch) {
     // pass
   }
 }
+
+#if defined(__linux__)
+TEST(MemorySanitizer, timer_create) {
+  timer_t timer;
+  EXPECT_POISONED(timer);
+  int res = timer_create(CLOCK_REALTIME, nullptr, &timer);
+  ASSERT_EQ(0, res);
+  EXPECT_NOT_POISONED(timer);
+
+  // Make sure the timer is usable.
+  struct itimerspec cur_value {};
+  cur_value.it_value.tv_sec = 1;
+  EXPECT_EQ(0, timer_settime(timer, 0, &cur_value, nullptr));
+
+  timer_t timer2;
+  EXPECT_POISONED(timer2);
+  // Use an invalid clock_id to make timer_create fail.
+  res = timer_create(INT_MAX, nullptr, &timer2);
+  ASSERT_EQ(-1, res);
+  EXPECT_POISONED(timer2);
+  timer_delete(timer);
+}
+#endif
 } // namespace
diff --git a/compiler-rt/lib/orc/common.h b/compiler-rt/lib/orc/common.h
index 73c5c4a2bd8d47417e6511eee0d0a2402eb5ca4f..f34229a615341df8e8eb94d1cf7f0c901a8578a8 100644
--- a/compiler-rt/lib/orc/common.h
+++ b/compiler-rt/lib/orc/common.h
@@ -19,9 +19,9 @@
 
 /// This macro should be used to define tags that will be associated with
 /// handlers in the JIT process, and call can be used to define tags f
-#define ORC_RT_JIT_DISPATCH_TAG(X) \
-extern "C" char X; \
-char X = 0;
+#define ORC_RT_JIT_DISPATCH_TAG(X)                                             \
+  ORC_RT_INTERFACE char X;                                                     \
+  char X = 0;
 
 /// Opaque struct for external symbols.
 struct __orc_rt_Opaque {};
diff --git a/compiler-rt/lib/orc/elfnix_platform.cpp b/compiler-rt/lib/orc/elfnix_platform.cpp
index 57673f088f77cb03ddc92863c28afb95fba55146..24cc6e1ef117781f68e1b0a84b0a0cb478444dc0 100644
--- a/compiler-rt/lib/orc/elfnix_platform.cpp
+++ b/compiler-rt/lib/orc/elfnix_platform.cpp
@@ -30,6 +30,7 @@ using namespace orc_rt;
 using namespace orc_rt::elfnix;
 
 // Declare function tags for functions in the JIT process.
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_reoptimize_tag)
 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag)
 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
 
diff --git a/compiler-rt/lib/profile/CMakeLists.txt b/compiler-rt/lib/profile/CMakeLists.txt
index ef23492514898b351870b223d124cfa075000f68..ac1451c8ceed18081b9a9800a3d40928ab954a8c 100644
--- a/compiler-rt/lib/profile/CMakeLists.txt
+++ b/compiler-rt/lib/profile/CMakeLists.txt
@@ -38,6 +38,17 @@ int main() {
 
 " COMPILER_RT_TARGET_HAS_FCNTL_LCK)
 
+CHECK_CXX_SOURCE_COMPILES("
+#include <sys/file.h>
+
+int fd;
+int main() {
+  flock(fd, LOCK_EX);
+  return 0;
+}
+
+" COMPILER_RT_TARGET_HAS_FLOCK)
+
 CHECK_CXX_SOURCE_COMPILES("
 #include <sys/utsname.h>
 int main() {
@@ -93,6 +104,13 @@ if(FUCHSIA OR UNIX)
      -Wno-pedantic)
 endif()
 
+if(CMAKE_SYSTEM_NAME STREQUAL "WASI")
+  set(EXTRA_FLAGS
+      ${EXTRA_FLAGS}
+      -D_WASI_EMULATED_MMAN
+      -D_WASI_EMULATED_GETPID)
+endif()
+
 if(COMPILER_RT_TARGET_HAS_ATOMICS)
  set(EXTRA_FLAGS
      ${EXTRA_FLAGS}
@@ -105,6 +123,12 @@ if(COMPILER_RT_TARGET_HAS_FCNTL_LCK)
      -DCOMPILER_RT_HAS_FCNTL_LCK=1)
 endif()
 
+if(COMPILER_RT_TARGET_HAS_FLOCK)
+  set(EXTRA_FLAGS
+      ${EXTRA_FLAGS}
+      -DCOMPILER_RT_HAS_FLOCK=1)
+endif()
+
 if(COMPILER_RT_TARGET_HAS_UNAME)
  set(EXTRA_FLAGS
      ${EXTRA_FLAGS}
@@ -138,7 +162,8 @@ if(APPLE)
     CFLAGS ${EXTRA_FLAGS}
     SOURCES ${PROFILE_SOURCES}
     ADDITIONAL_HEADERS ${PROFILE_HEADERS}
-    PARENT_TARGET profile)
+    PARENT_TARGET profile
+    EXTENSIONS ON)
 else()
   add_compiler_rt_runtime(clang_rt.profile
     STATIC
@@ -146,5 +171,6 @@ else()
     CFLAGS ${EXTRA_FLAGS}
     SOURCES ${PROFILE_SOURCES}
     ADDITIONAL_HEADERS ${PROFILE_HEADERS}
-    PARENT_TARGET profile)
+    PARENT_TARGET profile
+    EXTENSIONS ON)
 endif()
diff --git a/compiler-rt/lib/profile/GCDAProfiling.c b/compiler-rt/lib/profile/GCDAProfiling.c
index d6e2175169e4a525b5aaa5e3d8d801fe3fd2ef20..f67d95d21a7b541c1b709fd4a944ef10b94a11a7 100644
--- a/compiler-rt/lib/profile/GCDAProfiling.c
+++ b/compiler-rt/lib/profile/GCDAProfiling.c
@@ -584,7 +584,7 @@ void llvm_reset_counters(void) {
   }
 }
 
-#if !defined(_WIN32)
+#if !defined(_WIN32) && !defined(__wasm__)
 COMPILER_RT_VISIBILITY
 pid_t __gcov_fork() {
   pid_t parent_pid = getpid();
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index b766436497b741f68fb618ef97195d5ea995e1d0..02f23379ce98bfeacd2f408e3435fd79c30f1e49 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -6,11 +6,11 @@
 |*
 \*===----------------------------------------------------------------------===*/
 
-#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
-    (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \
-    defined(_AIX)
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) ||      \
+    (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) ||          \
+    defined(_AIX) || defined(__wasm__)
 
-#if !defined(_AIX)
+#if !defined(_AIX) && !defined(__wasm__)
 #include <elf.h>
 #include <link.h>
 #endif
diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
index aa79a5641ceca6192946935af5d82f2156b54aa1..52e82273f8aade4f9abad8bddd7ef2cb1c875e55 100644
--- a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
+++ b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
@@ -8,7 +8,8 @@
 
 #if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) &&     \
     !defined(__Fuchsia__) && !(defined(__sun__) && defined(__svr4__)) &&       \
-    !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX)
+    !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX) &&              \
+    !defined(__wasm__)
 
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/compiler-rt/lib/profile/InstrProfilingPort.h b/compiler-rt/lib/profile/InstrProfilingPort.h
index f77699ee8d59cf68eef40497b5561b60441bfca8..66d697885eaeeee2e36b24c5de460e7bea0b1091 100644
--- a/compiler-rt/lib/profile/InstrProfilingPort.h
+++ b/compiler-rt/lib/profile/InstrProfilingPort.h
@@ -54,7 +54,7 @@
 #endif
 
 #define COMPILER_RT_MAX_HOSTLEN 128
-#ifdef __ORBIS__
+#if defined(__ORBIS__) || defined(__wasi__)
 #define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1))
 #else
 #define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len)
diff --git a/compiler-rt/lib/profile/InstrProfilingUtil.c b/compiler-rt/lib/profile/InstrProfilingUtil.c
index 642393d432d7ea3e2451f5686144908c7f3dcdef..95ec4080ba2504a9ea1a424557b61883e594d36c 100644
--- a/compiler-rt/lib/profile/InstrProfilingUtil.c
+++ b/compiler-rt/lib/profile/InstrProfilingUtil.c
@@ -152,9 +152,11 @@ COMPILER_RT_VISIBILITY int lprofLockFd(int fd) {
     }
   }
   return 0;
-#else
+#elif defined(COMPILER_RT_HAS_FLOCK)
   flock(fd, LOCK_EX);
   return 0;
+#else
+  return 0;
 #endif
 }
 
@@ -177,9 +179,11 @@ COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) {
     }
   }
   return 0;
-#else
+#elif defined(COMPILER_RT_HAS_FLOCK)
   flock(fd, LOCK_UN);
   return 0;
+#else
+  return 0;
 #endif
 }
 
@@ -353,8 +357,8 @@ COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) {
 
 COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin,
                                                        uintptr_t End) {
-#if defined(__ve__)
-  // VE doesn't support madvise.
+#if defined(__ve__) || defined(__wasi__)
+  // VE and WASI doesn't support madvise.
   return 0;
 #else
   size_t PageSize = getpagesize();
diff --git a/compiler-rt/lib/rtsan/CMakeLists.txt b/compiler-rt/lib/rtsan/CMakeLists.txt
index af34fb63cf53ccd0a9e33e2b2daf9dc5a3c585ed..f8dd4d735bc2a300a68afe67faac4d7c5a113124 100644
--- a/compiler-rt/lib/rtsan/CMakeLists.txt
+++ b/compiler-rt/lib/rtsan/CMakeLists.txt
@@ -7,6 +7,7 @@ set(RTSAN_CXX_SOURCES
   rtsan_flags.cpp
   rtsan_interceptors.cpp
   rtsan_stats.cpp
+  rtsan_suppressions.cpp
   )
 
 set(RTSAN_PREINIT_SOURCES
@@ -14,12 +15,14 @@ set(RTSAN_PREINIT_SOURCES
 
 set(RTSAN_HEADERS
   rtsan.h
+  rtsan_checks.inc
   rtsan_assertions.h
   rtsan_context.h
   rtsan_diagnostics.h
   rtsan_flags.h
   rtsan_flags.inc
   rtsan_stats.h
+  rtsan_suppressions.h
   )
 
 set(RTSAN_DEPS)
diff --git a/compiler-rt/lib/rtsan/rtsan.cpp b/compiler-rt/lib/rtsan/rtsan.cpp
index 7691815fd5c1dc9ed701cdf972ce15d6518ac9c9..e9f42d3760aa824cb74c47373faeeef58967966e 100644
--- a/compiler-rt/lib/rtsan/rtsan.cpp
+++ b/compiler-rt/lib/rtsan/rtsan.cpp
@@ -14,12 +14,12 @@
 #include "rtsan/rtsan_flags.h"
 #include "rtsan/rtsan_interceptors.h"
 #include "rtsan/rtsan_stats.h"
+#include "rtsan/rtsan_suppressions.h"
 
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
-#include "sanitizer_common/sanitizer_stacktrace.h"
 
 using namespace __rtsan;
 using namespace __sanitizer;
@@ -85,6 +85,8 @@ SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init() {
   InitializeFlags();
   InitializeInterceptors();
 
+  InitializeSuppressions();
+
   if (flags().print_stats_on_exit)
     Atexit(PrintStatisticsSummary);
 
diff --git a/compiler-rt/lib/rtsan/rtsan_assertions.h b/compiler-rt/lib/rtsan/rtsan_assertions.h
index 745cbea0eb3a296351d9cde6b4121f05092e308a..8183a8202478ffcb99735e753d99b41526d7f18c 100644
--- a/compiler-rt/lib/rtsan/rtsan_assertions.h
+++ b/compiler-rt/lib/rtsan/rtsan_assertions.h
@@ -15,6 +15,7 @@
 #include "rtsan/rtsan.h"
 #include "rtsan/rtsan_context.h"
 #include "rtsan/rtsan_diagnostics.h"
+#include "rtsan/rtsan_suppressions.h"
 
 #include "sanitizer_common/sanitizer_stacktrace.h"
 
@@ -34,6 +35,9 @@ void ExpectNotRealtime(Context &context, const DiagnosticsInfo &info,
     stack.Unwind(info.pc, info.bp, nullptr,
                  __sanitizer::common_flags()->fast_unwind_on_fatal);
 
+    if (IsStackTraceSuppressed(stack))
+      return;
+
     OnViolation(stack, info);
   }
 }
diff --git a/compiler-rt/lib/rtsan/rtsan_checks.inc b/compiler-rt/lib/rtsan/rtsan_checks.inc
new file mode 100644
index 0000000000000000000000000000000000000000..f5f23e044bd5d7d74da84d9cce8619fbba0032fb
--- /dev/null
+++ b/compiler-rt/lib/rtsan/rtsan_checks.inc
@@ -0,0 +1,19 @@
+//===-- rtsan_checks.inc ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// List of suppression checks handled by RTSan runtime.
+//
+//===----------------------------------------------------------------------===//
+#ifndef RTSAN_CHECK
+#error "Define RTSAN_CHECK prior to including this file!"
+#endif
+
+// RTSAN_CHECK(Name, SummaryKind)
+// SummaryKind should be a string literal.
+
+RTSAN_CHECK(CallStackContains, "call-stack-contains")
diff --git a/compiler-rt/lib/rtsan/rtsan_flags.inc b/compiler-rt/lib/rtsan/rtsan_flags.inc
index 1df71127d19d3790db7891a9bdb44fc2dd93c949..5c3eb3f53a5eb429c35fd8778886d1d8ebd858a3 100644
--- a/compiler-rt/lib/rtsan/rtsan_flags.inc
+++ b/compiler-rt/lib/rtsan/rtsan_flags.inc
@@ -18,3 +18,4 @@
 
 RTSAN_FLAG(bool, halt_on_error, true, "Exit after first reported error.")
 RTSAN_FLAG(bool, print_stats_on_exit, false, "Print stats on exit.")
+RTSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
diff --git a/compiler-rt/lib/rtsan/rtsan_suppressions.cpp b/compiler-rt/lib/rtsan/rtsan_suppressions.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c5051dd19102362da80f859f3b2965921529322b
--- /dev/null
+++ b/compiler-rt/lib/rtsan/rtsan_suppressions.cpp
@@ -0,0 +1,94 @@
+//===--- rtsan_suppressions.cpp - Realtime Sanitizer ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the RTSan runtime, providing support for suppressions
+//
+//===----------------------------------------------------------------------===//
+
+#include "rtsan/rtsan_suppressions.h"
+
+#include "rtsan/rtsan_flags.h"
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_suppressions.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+#include <new>
+
+using namespace __sanitizer;
+using namespace __rtsan;
+
+namespace {
+enum class ErrorType {
+#define RTSAN_CHECK(Name, FSanitizeFlagName) Name,
+#include "rtsan_checks.inc"
+#undef RTSAN_CHECK
+};
+} // namespace
+
+alignas(64) static char suppression_placeholder[sizeof(SuppressionContext)];
+static SuppressionContext *suppression_ctx = nullptr;
+
+static const char *kSuppressionTypes[] = {
+#define RTSAN_CHECK(Name, FSanitizeFlagName) FSanitizeFlagName,
+#include "rtsan_checks.inc"
+#undef RTSAN_CHECK
+};
+
+static const char *ConvertTypeToFlagName(ErrorType Type) {
+  switch (Type) {
+#define RTSAN_CHECK(Name, FSanitizeFlagName)                                   \
+  case ErrorType::Name:                                                        \
+    return FSanitizeFlagName;
+#include "rtsan_checks.inc"
+#undef RTSAN_CHECK
+  }
+  UNREACHABLE("unknown ErrorType!");
+}
+
+void __rtsan::InitializeSuppressions() {
+  CHECK_EQ(nullptr, suppression_ctx);
+
+  // We will use suppression_ctx == nullptr as an early out
+  if (flags().suppressions[0] == '\0')
+    return;
+
+  suppression_ctx = new (suppression_placeholder)
+      SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
+  suppression_ctx->ParseFromFile(flags().suppressions);
+}
+
+bool __rtsan::IsStackTraceSuppressed(const StackTrace &stack) {
+  if (suppression_ctx == nullptr)
+    return false;
+
+  const char *call_stack_flag =
+      ConvertTypeToFlagName(ErrorType::CallStackContains);
+  if (!suppression_ctx->HasSuppressionType(call_stack_flag))
+    return false;
+
+  Symbolizer *symbolizer = Symbolizer::GetOrInit();
+  for (uptr i = 0; i < stack.size && stack.trace[i]; i++) {
+    const uptr addr = stack.trace[i];
+
+    SymbolizedStackHolder symbolized_stack(symbolizer->SymbolizePC(addr));
+    const SymbolizedStack *frames = symbolized_stack.get();
+    CHECK(frames);
+    for (const SymbolizedStack *cur = frames; cur; cur = cur->next) {
+      const char *function_name = cur->info.function;
+      if (!function_name)
+        continue;
+
+      Suppression *s;
+      if (suppression_ctx->Match(function_name, call_stack_flag, &s))
+        return true;
+    }
+  }
+  return false;
+}
diff --git a/compiler-rt/lib/rtsan/rtsan_suppressions.h b/compiler-rt/lib/rtsan/rtsan_suppressions.h
new file mode 100644
index 0000000000000000000000000000000000000000..45545f8c0e0b65c71817ca52960616cfada7e57a
--- /dev/null
+++ b/compiler-rt/lib/rtsan/rtsan_suppressions.h
@@ -0,0 +1,22 @@
+//===--- rtsan_suppressions.h - Realtime Sanitizer --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the RTSan runtime, providing support for suppressions
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "sanitizer_common/sanitizer_stacktrace.h"
+
+namespace __rtsan {
+
+void InitializeSuppressions();
+bool IsStackTraceSuppressed(const __sanitizer::StackTrace &stack);
+
+} // namespace __rtsan
diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
index 556a64f3017488c0513d230f7b1e2c5189cd898b..09391e4f5f3704ba657e57ff060c2b1462f3a5bc 100644
--- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt
+++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
@@ -94,6 +94,7 @@ set(SANITIZER_SYMBOLIZER_SOURCES
   sanitizer_symbolizer_report.cpp
   sanitizer_symbolizer_report_fuchsia.cpp
   sanitizer_symbolizer_win.cpp
+  sanitizer_thread_history.cpp
   sanitizer_unwind_linux_libcdep.cpp
   sanitizer_unwind_fuchsia.cpp
   sanitizer_unwind_win.cpp
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index b8627f8557afe29c10a37211fbc7f7cf6fc8221b..211f9f70d7e4c6c54ac4b8778f97c61bdee55b98 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -2289,6 +2289,24 @@ INTERCEPTOR(int, pthread_getcpuclockid, uptr thread,
 #define INIT_CLOCK_GETCPUCLOCKID
 #endif
 
+#if SANITIZER_INTERCEPT_TIMER_CREATE
+INTERCEPTOR(int, timer_create, __sanitizer_clockid_t clockid, void *sevp,
+            __sanitizer_timer_t *timer) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timer_create, clockid, sevp, timer);
+  int res = REAL(timer_create)(clockid, sevp, timer);
+  if (!res && timer) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, timer, sizeof *timer);
+  }
+  return res;
+}
+
+#  define INIT_TIMER_CREATE \
+    COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_create, "GLIBC_2.3.3");
+#else
+#  define INIT_TIMER_CREATE
+#endif
+
 #if SANITIZER_INTERCEPT_GETITIMER
 INTERCEPTOR(int, getitimer, int which, void *curr_value) {
   void *ctx;
@@ -10266,6 +10284,7 @@ static void InitializeCommonInterceptors() {
   INIT_SETPWENT;
   INIT_CLOCK_GETTIME;
   INIT_CLOCK_GETCPUCLOCKID;
+  INIT_TIMER_CREATE;
   INIT_GETITIMER;
   INIT_TIME;
   INIT_GLOB;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
index 66744aa021e62b09b24bf8b4de569db1664a151d..4ea75cdd67cb93820af0a3b621bd54bf828eefac 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
@@ -10,6 +10,7 @@
 INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
 INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
 INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)
+INTERFACE_FUNCTION(__sanitizer_copy_contiguous_container_annotations)
 INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
 INTERFACE_FUNCTION(
     __sanitizer_double_ended_contiguous_container_find_bad_address)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h b/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
index c424ab1cecf94f376a13391298037261096bcfe6..387a4d87d97bf9468e01655a72f6e2dbeb933d12 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
@@ -76,6 +76,11 @@ void __sanitizer_annotate_double_ended_contiguous_container(
     const void *old_container_beg, const void *old_container_end,
     const void *new_container_beg, const void *new_container_end);
 SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_copy_contiguous_container_annotations(const void *src_begin,
+                                                       const void *src_end,
+                                                       const void *dst_begin,
+                                                       const void *dst_end);
+SANITIZER_INTERFACE_ATTRIBUTE
 int __sanitizer_verify_contiguous_container(const void *beg, const void *mid,
                                             const void *end);
 SANITIZER_INTERFACE_ATTRIBUTE
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 6959a6d52d604e01b82b561733057e598e1f4c2e..36fafdc642642bf758afc2879f36ea7d977116d9 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -237,6 +237,9 @@
   (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \
   (SI_LINUX || SI_FREEBSD || SI_NETBSD)
+// TODO: This should be SI_POSIX, adding Linux first until I have time
+// to verify all timer_t typedefs on other platforms.
+#define SANITIZER_INTERCEPT_TIMER_CREATE SI_LINUX
 #define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
 #define SANITIZER_INTERCEPT_TIME SI_POSIX
 #define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index e8c81aa8e281637b74f7fff5660e8d07b0089412..b4ccf7b3d7bef4877ec82debbfd62ed0d4dffb0d 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -1517,6 +1517,10 @@ extern const int si_SEGV_ACCERR;
 
 #define SIGACTION_SYMNAME sigaction
 
+#  if SANITIZER_LINUX
+typedef void *__sanitizer_timer_t;
+#  endif
+
 #endif  // SANITIZER_LINUX || SANITIZER_APPLE
 
 #endif
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
index eeb49e2afe34d492cbe54fbf74fcd2a59d3a0778..80b8158f43db9e9d7b47171374d7dd8871a055fc 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
@@ -11,6 +11,10 @@
 
 // Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
 #undef _FILE_OFFSET_BITS
+
+// Avoid conflict between `_TIME_BITS` defined vs. `_FILE_OFFSET_BITS`
+// undefined in some Linux configurations.
+#undef _TIME_BITS
 #include "sanitizer_platform.h"
 #if SANITIZER_SOLARIS
 #  include <fcntl.h>
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_history.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_thread_history.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0f5bec3ca083e372813fc5e68064824ac315cae8
--- /dev/null
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_history.cpp
@@ -0,0 +1,72 @@
+//===-- sanitizer_thread_history.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_thread_history.h"
+
+#include "sanitizer_stackdepot.h"
+namespace __sanitizer {
+
+void PrintThreadHistory(ThreadRegistry &registry, InternalScopedString &out) {
+  ThreadRegistryLock l(&registry);
+  // Stack traces are largest part of printout and they often the same for
+  // multiple threads, so we will deduplicate them.
+  InternalMmapVector<const ThreadContextBase *> stacks;
+
+  registry.RunCallbackForEachThreadLocked(
+      [](ThreadContextBase *context, void *arg) {
+        static_cast<decltype(&stacks)>(arg)->push_back(context);
+      },
+      &stacks);
+
+  Sort(stacks.data(), stacks.size(),
+       [](const ThreadContextBase *a, const ThreadContextBase *b) {
+         if (a->stack_id < b->stack_id)
+           return true;
+         if (a->stack_id > b->stack_id)
+           return false;
+         return a->unique_id < b->unique_id;
+       });
+
+  auto describe_thread = [&](const ThreadContextBase *context) {
+    if (!context) {
+      out.Append("T-1");
+      return;
+    }
+    out.AppendF("T%llu/%llu", context->unique_id, context->os_id);
+    if (internal_strlen(context->name))
+      out.AppendF(" (%s)", context->name);
+  };
+
+  auto get_parent =
+      [&](const ThreadContextBase *context) -> const ThreadContextBase * {
+    if (!context)
+      return nullptr;
+    ThreadContextBase *parent = registry.GetThreadLocked(context->parent_tid);
+    if (!parent)
+      return nullptr;
+    if (parent->unique_id >= context->unique_id)
+      return nullptr;
+    return parent;
+  };
+
+  const ThreadContextBase *prev = nullptr;
+  for (const ThreadContextBase *context : stacks) {
+    if (prev && prev->stack_id != context->stack_id)
+      StackDepotGet(prev->stack_id).PrintTo(&out);
+    prev = context;
+    out.Append("Thread ");
+    describe_thread(context);
+    out.Append(" was created by ");
+    describe_thread(get_parent(context));
+    out.Append("\n");
+  }
+  if (prev)
+    StackDepotGet(prev->stack_id).PrintTo(&out);
+}
+
+}  // namespace __sanitizer
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_history.h b/compiler-rt/lib/sanitizer_common/sanitizer_thread_history.h
new file mode 100644
index 0000000000000000000000000000000000000000..2995f6015fe50e7c3b40b8b4ea4c5123ee2ad862
--- /dev/null
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_history.h
@@ -0,0 +1,24 @@
+//===-- sanitizer_thread_history.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utility to print thread histroy from ThreadRegistry.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_THREAD_HISTORY_H
+#define SANITIZER_THREAD_HISTORY_H
+
+#include "sanitizer_thread_registry.h"
+
+namespace __sanitizer {
+
+void PrintThreadHistory(ThreadRegistry& registry, InternalScopedString& out);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_THREAD_HISTORY_H
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
index df04822b28851cde3ed2f625a50673fdba2c39aa..cdc24f4a8869c7e5d020ee540e46c20173fca5ee 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
@@ -27,6 +27,7 @@ ThreadContextBase::ThreadContextBase(u32 tid)
       detached(false),
       thread_type(ThreadType::Regular),
       parent_tid(0),
+      stack_id(0),
       next(0) {
   name[0] = '\0';
   atomic_store(&thread_destroyed, 0, memory_order_release);
@@ -88,14 +89,17 @@ void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type,
 }
 
 void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id,
-                                   bool _detached, u32 _parent_tid, void *arg) {
+                                   bool _detached, u32 _parent_tid,
+                                   u32 _stack_tid, void *arg) {
   status = ThreadStatusCreated;
   user_id = _user_id;
   unique_id = _unique_id;
   detached = _detached;
   // Parent tid makes no sense for the main thread.
-  if (tid != kMainTid)
+  if (tid != kMainTid) {
     parent_tid = _parent_tid;
+    stack_id = _stack_tid;
+  }
   OnCreated(arg);
 }
 
@@ -143,7 +147,7 @@ uptr ThreadRegistry::GetMaxAliveThreads() {
 }
 
 u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
-                                 void *arg) {
+                                 u32 stack_tid, void *arg) {
   ThreadRegistryLock l(this);
   u32 tid = kInvalidTid;
   ThreadContextBase *tctx = QuarantinePop();
@@ -181,7 +185,8 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
     // positives later (e.g. if we join a wrong thread).
     CHECK(live_.try_emplace(user_id, tid).second);
   }
-  tctx->SetCreated(user_id, total_threads_++, detached, parent_tid, arg);
+  tctx->SetCreated(user_id, total_threads_++, detached, parent_tid, stack_tid,
+                   arg);
   return tid;
 }
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
index bf492c17f7e107e2f91f59f4fc294e0684f49041..e06abb3932da5cfd5fef4d18567e1d14667fa64e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
@@ -52,6 +52,7 @@ class ThreadContextBase {
   ThreadType thread_type;
 
   u32 parent_tid;
+  u32 stack_id;
   ThreadContextBase *next;  // For storing thread contexts in a list.
 
   atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished
@@ -63,7 +64,7 @@ class ThreadContextBase {
   void SetFinished();
   void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg);
   void SetCreated(uptr _user_id, u64 _unique_id, bool _detached,
-                  u32 _parent_tid, void *arg);
+                  u32 _parent_tid, u32 _stack_tid, void *arg);
   void Reset();
 
   void SetDestroyed();
@@ -106,7 +107,11 @@ class SANITIZER_MUTEX ThreadRegistry {
 
   u32 NumThreadsLocked() const { return threads_.size(); }
 
-  u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
+  u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, u32 stack_tid,
+                   void *arg);
+  u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg) {
+    return CreateThread(user_id, detached, parent_tid, 0, arg);
+  }
 
   typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg);
   // Invokes callback with a specified arg for each thread context.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
index 7cee571314868e93afee36040cb5892fd66dce13..6fb947aa6d6c26f98ad15438ac9cdfceb4b06e31 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
@@ -1033,7 +1033,52 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
 }
 
 void SignalContext::DumpAllRegisters(void *context) {
-  // FIXME: Implement this.
+  CONTEXT *ctx = (CONTEXT *)context;
+#  if defined(_M_X64)
+  Report("Register values:\n");
+  Printf("rax = %llx  ", ctx->Rax);
+  Printf("rbx = %llx  ", ctx->Rbx);
+  Printf("rcx = %llx  ", ctx->Rcx);
+  Printf("rdx = %llx  ", ctx->Rdx);
+  Printf("\n");
+  Printf("rdi = %llx  ", ctx->Rdi);
+  Printf("rsi = %llx  ", ctx->Rsi);
+  Printf("rbp = %llx  ", ctx->Rbp);
+  Printf("rsp = %llx  ", ctx->Rsp);
+  Printf("\n");
+  Printf("r8  = %llx  ", ctx->R8);
+  Printf("r9  = %llx  ", ctx->R9);
+  Printf("r10 = %llx  ", ctx->R10);
+  Printf("r11 = %llx  ", ctx->R11);
+  Printf("\n");
+  Printf("r12 = %llx  ", ctx->R12);
+  Printf("r13 = %llx  ", ctx->R13);
+  Printf("r14 = %llx  ", ctx->R14);
+  Printf("r15 = %llx  ", ctx->R15);
+  Printf("\n");
+#  elif defined(_M_IX86)
+  Report("Register values:\n");
+  Printf("eax = %lx  ", ctx->Eax);
+  Printf("ebx = %lx  ", ctx->Ebx);
+  Printf("ecx = %lx  ", ctx->Ecx);
+  Printf("edx = %lx  ", ctx->Edx);
+  Printf("\n");
+  Printf("edi = %lx  ", ctx->Edi);
+  Printf("esi = %lx  ", ctx->Esi);
+  Printf("ebp = %lx  ", ctx->Ebp);
+  Printf("esp = %lx  ", ctx->Esp);
+  Printf("\n");
+#  elif defined(_M_ARM64)
+  Report("Register values:\n");
+  for (int i = 0; i <= 30; i++) {
+    Printf("x%d%s = %llx", i < 10 ? " " : "", ctx->X[i]);
+    if (i % 4 == 3)
+      Printf("\n");
+  }
+#  else
+  // TODO
+  (void)ctx;
+#  endif
 }
 
 int SignalContext::GetType() const {
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cpp
index 8c4b4ba5c58f7d94c6a3518599d2703ef2d25ddc..6e84ecdfeba683f63cb659de15c07f051cb1b813 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_thread_registry_test.cpp
@@ -11,11 +11,19 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_thread_registry.h"
 
+#include <iostream>
 #include <vector>
 
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_thread_history.h"
 #include "sanitizer_pthread_wrappers.h"
 
+using testing::HasSubstr;
+
 namespace __sanitizer {
 
 static Mutex tctx_allocator_lock;
@@ -64,11 +72,12 @@ static void MarkUidAsPresent(ThreadContextBase *tctx, void *arg) {
 
 static void TestRegistry(ThreadRegistry *registry, bool has_quarantine) {
   // Create and start a main thread.
-  EXPECT_EQ(0U, registry->CreateThread(get_uid(0), true, -1, 0));
+  EXPECT_EQ(0U, registry->CreateThread(get_uid(0), true, -1, 0, nullptr));
   registry->StartThread(0, 0, ThreadType::Regular, 0);
   // Create a bunch of threads.
   for (u32 i = 1; i <= 10; i++) {
-    EXPECT_EQ(i, registry->CreateThread(get_uid(i), is_detached(i), 0, 0));
+    EXPECT_EQ(i, registry->CreateThread(get_uid(i), is_detached(i), 100 + i,
+                                        200 + i, nullptr));
   }
   CheckThreadQuantity(registry, 11, 1, 11);
   // Start some of them.
@@ -88,19 +97,27 @@ static void TestRegistry(ThreadRegistry *registry, bool has_quarantine) {
   std::vector<u32> new_tids;
   for (u32 i = 11; i <= 15; i++) {
     new_tids.push_back(
-        registry->CreateThread(get_uid(i), is_detached(i), 0, 0));
+        registry->CreateThread(get_uid(i), is_detached(i), 0, 0, nullptr));
   }
   ASSERT_LE(kRegistryQuarantine, 5U);
-  u32 exp_total = 16 - (has_quarantine ? 5 - kRegistryQuarantine  : 0);
+  u32 exp_total = 16 - (has_quarantine ? 5 - kRegistryQuarantine : 0);
   CheckThreadQuantity(registry, exp_total, 6, 11);
   // Test SetThreadName and FindThread.
   registry->SetThreadName(6, "six");
   registry->SetThreadName(7, "seven");
-  EXPECT_EQ(7U, registry->FindThread(HasName, (void*)"seven"));
+  EXPECT_EQ(7U, registry->FindThread(HasName, (void *)"seven"));
   EXPECT_EQ(kInvalidTid, registry->FindThread(HasName, (void *)"none"));
-  EXPECT_EQ(0U, registry->FindThread(HasUid, (void*)get_uid(0)));
-  EXPECT_EQ(10U, registry->FindThread(HasUid, (void*)get_uid(10)));
+  EXPECT_EQ(0U, registry->FindThread(HasUid, (void *)get_uid(0)));
+  EXPECT_EQ(10U, registry->FindThread(HasUid, (void *)get_uid(10)));
   EXPECT_EQ(kInvalidTid, registry->FindThread(HasUid, (void *)0x1234));
+  EXPECT_EQ(7U,
+            registry->FindThread([](ThreadContextBase *tctx,
+                                    void *) { return tctx->parent_tid == 107; },
+                                 nullptr));
+  EXPECT_EQ(8U,
+            registry->FindThread([](ThreadContextBase *tctx,
+                                    void *) { return tctx->stack_id == 208; },
+                                 nullptr));
   // Detach and finish and join remaining threads.
   for (u32 i = 6; i <= 10; i++) {
     registry->DetachThread(i, 0);
@@ -230,4 +247,56 @@ TEST(SanitizerCommon, ThreadRegistryThreadedTest) {
   ThreadedTestRegistry(&registry);
 }
 
+TEST(SanitizerCommon, PrintThreadHistory) {
+  ThreadRegistry registry(GetThreadContext<TestThreadContext>,
+                          kThreadsPerShard * kNumShards + 1, 10, 0);
+
+  UNINITIALIZED BufferedStackTrace stack1;
+  stack1.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, false,
+                /*max_depth=*/1);
+
+  UNINITIALIZED BufferedStackTrace stack2;
+  stack2.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, false,
+                /*max_depth=*/1);
+
+  EXPECT_EQ(0U, registry.CreateThread(0, true, -1, 0, nullptr));
+  for (int i = 0; i < 5; i++) {
+    registry.CreateThread(0, true, 0, StackDepotPut(stack1), nullptr);
+    registry.CreateThread(0, true, 0, StackDepotPut(stack2), nullptr);
+  }
+
+  InternalScopedString out;
+  PrintThreadHistory(registry, out);
+
+  std::string substrings[] = {
+      "Thread T0/0 was created by T-1",
+      "<empty stack>",
+      "",
+      "Thread T1/0 was created by T0/0",
+      "Thread T3/0 was created by T0/0",
+      "Thread T5/0 was created by T0/0",
+      "Thread T7/0 was created by T0/0",
+      "Thread T9/0 was created by T0/0",
+      "#0 0x",
+      "",
+      "Thread T2/0 was created by T0/0",
+      "Thread T4/0 was created by T0/0",
+      "Thread T6/0 was created by T0/0",
+      "Thread T8/0 was created by T0/0",
+      "Thread T10/0 was created by T0/0",
+      "#0 0x",
+      "",
+  };
+
+  std::stringstream ss(out.data());
+  std::string line;
+
+  for (auto substr : substrings) {
+    std::getline(ss, line);
+    EXPECT_THAT(line, HasSubstr(substr)) << line;
+  }
+
+  EXPECT_FALSE(std::getline(ss, line)) << "Unmatched line: " << line;
+}
+
 }  // namespace __sanitizer
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 97188a5ac235ccb14641a93452fe0813b32c91bb..e382e01f5d8e5408056411404ba63d9ae9eb285e 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -1492,6 +1492,11 @@ private:
       }
 
       const uptr PushedBytesDelta = BytesInBG - BG->BytesInBGAtLastCheckpoint;
+      if (PushedBytesDelta < getMinReleaseAttemptSize(BlockSize)) {
+        Prev = BG;
+        BG = BG->Next;
+        continue;
+      }
 
       // Given the randomness property, we try to release the pages only if the
       // bytes used by free blocks exceed certain proportion of group size. Note
diff --git a/compiler-rt/test/asan/TestCases/copy_container_annotations.cpp b/compiler-rt/test/asan/TestCases/copy_container_annotations.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ed20dc3e80d44e263dd5e4cb4fc9c9e9ca49b7ab
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/copy_container_annotations.cpp
@@ -0,0 +1,172 @@
+// RUN: %clangxx_asan -fexceptions -O %s -o %t && %env_asan_opts=detect_stack_use_after_return=0 %run %t
+//
+// Test __sanitizer_copy_contiguous_container_annotations.
+
+#include <algorithm>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <vector>
+
+#include <assert.h>
+#include <sanitizer/asan_interface.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static constexpr size_t kGranularity = 8;
+
+template <class T> static constexpr T RoundDown(T x) {
+  return reinterpret_cast<T>(reinterpret_cast<uintptr_t>(x) &
+                             ~(kGranularity - 1));
+}
+template <class T> static constexpr T RoundUp(T x) {
+  return reinterpret_cast<T>(
+      RoundDown(reinterpret_cast<uintptr_t>(x) + kGranularity - 1));
+}
+
+static std::vector<int> GetPoisonedState(char *begin, char *end) {
+  std::vector<int> result;
+  for (char *ptr = begin; ptr != end; ++ptr) {
+    result.push_back(__asan_address_is_poisoned(ptr));
+  }
+  return result;
+}
+
+static void RandomPoison(char *beg, char *end) {
+  assert(beg == RoundDown(beg));
+  assert(end == RoundDown(end));
+  __asan_poison_memory_region(beg, end - beg);
+  for (beg = RoundUp(beg); beg < end; beg += kGranularity) {
+    __asan_unpoison_memory_region(beg, rand() % (kGranularity + 1));
+  }
+}
+
+template <bool benchmark>
+static void Test(size_t capacity, size_t off_src, size_t off_dst,
+                 char *src_buffer_beg, char *src_buffer_end,
+                 char *dst_buffer_beg, char *dst_buffer_end) {
+  size_t dst_buffer_size = dst_buffer_end - dst_buffer_beg;
+  char *src_beg = src_buffer_beg + off_src;
+  char *src_end = src_beg + capacity;
+
+  char *dst_beg = dst_buffer_beg + off_dst;
+  char *dst_end = dst_beg + capacity;
+  if (benchmark) {
+    __sanitizer_copy_contiguous_container_annotations(src_beg, src_end, dst_beg,
+                                                      dst_end);
+    return;
+  }
+
+  std::vector<int> src_poison_states =
+      GetPoisonedState(src_buffer_beg, src_buffer_end);
+  std::vector<int> dst_poison_before =
+      GetPoisonedState(dst_buffer_beg, dst_buffer_end);
+  __sanitizer_copy_contiguous_container_annotations(src_beg, src_end, dst_beg,
+                                                    dst_end);
+  std::vector<int> dst_poison_after =
+      GetPoisonedState(dst_buffer_beg, dst_buffer_end);
+
+  // Create ideal copy of src over dst.
+  std::vector<int> dst_poison_exp = dst_poison_before;
+  for (size_t cur = 0; cur < capacity; ++cur)
+    dst_poison_exp[off_dst + cur] = src_poison_states[off_src + cur];
+
+  // Unpoison prefixes of Asan granules.
+  for (size_t cur = dst_buffer_size - 1; cur > 0; --cur) {
+    if (cur % kGranularity != 0 && !dst_poison_exp[cur])
+      dst_poison_exp[cur - 1] = 0;
+  }
+
+  if (dst_poison_after != dst_poison_exp) {
+    std::cerr << "[" << off_dst << ", " << off_dst + capacity << ")\n";
+    for (size_t i = 0; i < dst_poison_after.size(); ++i) {
+      std::cerr << i << ":\t" << dst_poison_before[i] << "\t"
+                << dst_poison_after[i] << "\t" << dst_poison_exp[i] << "\n";
+    }
+    std::cerr << "----------\n";
+
+    assert(dst_poison_after == dst_poison_exp);
+  }
+}
+
+template <bool benchmark>
+static void TestNonOverlappingContainers(size_t capacity, size_t off_src,
+                                         size_t off_dst) {
+  // Test will copy [off_src, off_src + capacity) to [off_dst, off_dst + capacity).
+  // Allocate buffers to have additional granule before and after tested ranges.
+  off_src += kGranularity;
+  off_dst += kGranularity;
+  size_t src_buffer_size = RoundUp(off_src + capacity) + kGranularity;
+  size_t dst_buffer_size = RoundUp(off_dst + capacity) + kGranularity;
+
+  std::unique_ptr<char[]> src_buffer =
+      std::make_unique<char[]>(src_buffer_size);
+  std::unique_ptr<char[]> dst_buffer =
+      std::make_unique<char[]>(dst_buffer_size);
+
+  char *src_buffer_beg = src_buffer.get();
+  char *src_buffer_end = src_buffer_beg + src_buffer_size;
+  assert(RoundDown(src_buffer_beg) == src_buffer_beg);
+
+  char *dst_buffer_beg = dst_buffer.get();
+  char *dst_buffer_end = dst_buffer_beg + dst_buffer_size;
+  assert(RoundDown(dst_buffer_beg) == dst_buffer_beg);
+
+  for (int i = 0; i < 35; i++) {
+    if (!benchmark || !i) {
+      RandomPoison(src_buffer_beg, src_buffer_end);
+      RandomPoison(dst_buffer_beg, dst_buffer_end);
+    }
+
+    Test<benchmark>(capacity, off_src, off_dst, src_buffer_beg, src_buffer_end,
+                    dst_buffer_beg, dst_buffer_end);
+  }
+
+  __asan_unpoison_memory_region(src_buffer_beg, src_buffer_size);
+  __asan_unpoison_memory_region(dst_buffer_beg, dst_buffer_size);
+}
+
+template <bool benchmark>
+static void TestOverlappingContainers(size_t capacity, size_t off_src,
+                                      size_t off_dst) {
+  // Test will copy [off_src, off_src + capacity) to [off_dst, off_dst + capacity).
+  // Allocate buffers to have additional granule before and after tested ranges.
+  off_src += kGranularity;
+  off_dst += kGranularity;
+  size_t buffer_size =
+      RoundUp(std::max(off_src, off_dst) + capacity) + kGranularity;
+
+  // Use unique_ptr with a custom deleter to manage the buffer
+  std::unique_ptr<char[]> buffer = std::make_unique<char[]>(buffer_size);
+
+  char *buffer_beg = buffer.get();
+  char *buffer_end = buffer_beg + buffer_size;
+  assert(RoundDown(buffer_beg) == buffer_beg);
+
+  for (int i = 0; i < 35; i++) {
+    if (!benchmark || !i)
+      RandomPoison(buffer_beg, buffer_end);
+    Test<benchmark>(capacity, off_src, off_dst, buffer_beg, buffer_end,
+                    buffer_beg, buffer_end);
+  }
+
+  __asan_unpoison_memory_region(buffer_beg, buffer_size);
+}
+
+int main(int argc, char **argv) {
+  int n = argc == 1 ? 64 : atoi(argv[1]);
+  for (size_t off_src = 0; off_src < kGranularity; off_src++) {
+    for (size_t off_dst = 0; off_dst < kGranularity; off_dst++) {
+      for (int capacity = 0; capacity <= n; capacity++) {
+        if (n < 1024) {
+          TestNonOverlappingContainers<false>(capacity, off_src, off_dst);
+          TestOverlappingContainers<false>(capacity, off_src, off_dst);
+        } else {
+          TestNonOverlappingContainers<true>(capacity, off_src, off_dst);
+          TestOverlappingContainers<true>(capacity, off_src, off_dst);
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/compiler-rt/test/asan/Unit/lit.site.cfg.py.in b/compiler-rt/test/asan/Unit/lit.site.cfg.py.in
index ac652b53dcb9da68c95b52267f7f0f0967551286..69313142ad58e468b38ef66aea444ebb45fb6d04 100644
--- a/compiler-rt/test/asan/Unit/lit.site.cfg.py.in
+++ b/compiler-rt/test/asan/Unit/lit.site.cfg.py.in
@@ -8,25 +8,6 @@ import shlex
 # Load common config for all compiler-rt unit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/unittests/lit.common.unit.configured")
 
-def push_ld_library_path(config, new_path):
-  new_ld_library_path = os.path.pathsep.join(
-    (new_path, config.environment.get('LD_LIBRARY_PATH', '')))
-  config.environment['LD_LIBRARY_PATH'] = new_ld_library_path
-
-  if platform.system() == 'FreeBSD':
-    new_ld_32_library_path = os.path.pathsep.join(
-      (new_path, config.environment.get('LD_32_LIBRARY_PATH', '')))
-    config.environment['LD_32_LIBRARY_PATH'] = new_ld_32_library_path
-
-  if platform.system() == 'SunOS':
-    new_ld_library_path_32 = os.path.pathsep.join(
-      (new_path, config.environment.get('LD_LIBRARY_PATH_32', '')))
-    config.environment['LD_LIBRARY_PATH_32'] = new_ld_library_path_32
-
-    new_ld_library_path_64 = os.path.pathsep.join(
-      (new_path, config.environment.get('LD_LIBRARY_PATH_64', '')))
-    config.environment['LD_LIBRARY_PATH_64'] = new_ld_library_path_64
-
 # Setup config name.
 config.name = 'AddressSanitizer-Unit'
 
@@ -48,19 +29,5 @@ config.test_exec_root = os.path.join("@COMPILER_RT_BINARY_DIR@",
 
 config.test_source_root = config.test_exec_root
 
-# When LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=on, the initial value of
-# config.compiler_rt_libdir (COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR) has the
-# host triple as the trailing path component. The value is incorrect for i386
-# tests on x86_64 hosts and vice versa. Adjust config.compiler_rt_libdir
-# accordingly.
-if config.enable_per_target_runtime_dir:
-    if config.target_arch == 'i386':
-        config.compiler_rt_libdir = re.sub(r'/x86_64(?=-[^/]+$)', '/i386', config.compiler_rt_libdir)
-    elif config.target_arch == 'x86_64':
-        config.compiler_rt_libdir = re.sub(r'/i386(?=-[^/]+$)', '/x86_64', config.compiler_rt_libdir)
-
-# Set LD_LIBRARY_PATH to pick dynamic runtime up properly.
-push_ld_library_path(config, config.compiler_rt_libdir)
-
 if not config.parallelism_group:
   config.parallelism_group = 'shadow-memory'
diff --git a/compiler-rt/test/fuzzer/CMakeLists.txt b/compiler-rt/test/fuzzer/CMakeLists.txt
index b19f52e591fc0f2263b10bb2661ccaf2f4f6c380..51021e2dc2d9955e0408bb707d21d57b493f1195 100644
--- a/compiler-rt/test/fuzzer/CMakeLists.txt
+++ b/compiler-rt/test/fuzzer/CMakeLists.txt
@@ -73,10 +73,10 @@ endmacro()
 
 test_fuzzer("default")
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
-  if(TARGET cxx_shared)
+  if(LIBCXX_ENABLE_SHARED)
     test_fuzzer("libc++" DEPS cxx_shared)
   endif()
-  if(TARGET cxx_static)
+  if(LIBCXX_ENABLE_STATIC)
     test_fuzzer("static-libc++" DEPS cxx_static)
   endif()
 endif()
diff --git a/compiler-rt/test/fuzzer/strncmp.test b/compiler-rt/test/fuzzer/strncmp.test
index 91f38a35b9c9ce989502f02d6e200bf935b9219f..9a34c70720b123838a7c311617300f6412566386 100644
--- a/compiler-rt/test/fuzzer/strncmp.test
+++ b/compiler-rt/test/fuzzer/strncmp.test
@@ -1,4 +1,4 @@
 UNSUPPORTED: target={{.*freebsd.*}}
 RUN: %cpp_compiler %S/StrncmpTest.cpp -o %t-StrncmpTest
-RUN: not %run %t-StrncmpTest              -seed=2 -runs=10000000   2>&1 | FileCheck %s
+RUN: not %run %t-StrncmpTest              -seed=2 -runs=20000000   2>&1 | FileCheck %s
 CHECK: BINGO
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index c533c7e9a70476db062efa582c278c2f3e0571d5..c3435dd2c52a9e0aaa569fdc7e5da56b34af7a39 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -161,18 +161,26 @@ config.available_features.add(compiler_id)
 
 # When LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=on, the initial value of
 # config.compiler_rt_libdir (COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR) has the
-# triple as the trailing path component. The value is incorrect for -m32/-m64.
-# Adjust config.compiler_rt accordingly.
+# host triple as the trailing path component. The value is incorrect for 32-bit
+# tests on 64-bit hosts and vice versa. Adjust config.compiler_rt_libdir
+# accordingly.
 if config.enable_per_target_runtime_dir:
-    if "-m32" in shlex.split(config.target_cflags):
+    if config.target_arch == "i386":
         config.compiler_rt_libdir = re.sub(
             r"/x86_64(?=-[^/]+$)", "/i386", config.compiler_rt_libdir
         )
-    elif "-m64" in shlex.split(config.target_cflags):
+    elif config.target_arch == "x86_64":
         config.compiler_rt_libdir = re.sub(
             r"/i386(?=-[^/]+$)", "/x86_64", config.compiler_rt_libdir
         )
-
+    if config.target_arch == "sparc":
+        config.compiler_rt_libdir = re.sub(
+            r"/sparcv9(?=-[^/]+$)", "/sparc", config.compiler_rt_libdir
+        )
+    elif config.target_arch == "sparcv9":
+        config.compiler_rt_libdir = re.sub(
+            r"/sparc(?=-[^/]+$)", "/sparcv9", config.compiler_rt_libdir
+        )
 
 # Check if the test compiler resource dir matches the local build directory
 # (which happens with -DLLVM_ENABLE_PROJECTS=clang;compiler-rt) or if we are
diff --git a/compiler-rt/test/lsan/TestCases/flag_tries.c b/compiler-rt/test/lsan/TestCases/flag_tries.c
new file mode 100644
index 0000000000000000000000000000000000000000..d6af766d5ef282d09bbdddaaa991102ca0f1553f
--- /dev/null
+++ b/compiler-rt/test/lsan/TestCases/flag_tries.c
@@ -0,0 +1,23 @@
+// Test retries option of lsan.
+// RUN: %clang_lsan %s -o %t
+// RUN: %env_lsan_opts=use_stacks=0:use_registers=0:symbolize=0 %run %t foo 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK1
+// RUN: %env_lsan_opts=use_stacks=0:use_registers=0:symbolize=0:tries=12 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK12
+
+#include <assert.h>
+#include <sanitizer/lsan_interface.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+void *p;
+
+int main(int argc, char *argv[]) {
+  fprintf(stderr, "Test alloc: %p.\n", malloc(1337));
+  // CHECK: Test alloc:
+
+  assert(__lsan_do_recoverable_leak_check() == 1);
+  // CHECK1-COUNT-1: SUMMARY: {{.*}}Sanitizer: 1337 byte
+  // CHECK12-COUNT-12: SUMMARY: {{.*}}Sanitizer: 1337 byte
+
+  _exit(0);
+}
diff --git a/compiler-rt/test/lsan/TestCases/print_threads.c b/compiler-rt/test/lsan/TestCases/print_threads.c
new file mode 100644
index 0000000000000000000000000000000000000000..b3072da93fab621259ebf4737fa378e7490f2e39
--- /dev/null
+++ b/compiler-rt/test/lsan/TestCases/print_threads.c
@@ -0,0 +1,33 @@
+// RUN: %clang_lsan %s -o %t && %env_lsan_opts=log_threads=1 %run %t 2>&1 | FileCheck %s
+
+// XFAIL: hwasan
+
+#include <assert.h>
+#include <pthread.h>
+#include <sanitizer/lsan_interface.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+pthread_barrier_t bar;
+
+void *threadfn(void *arg) {
+  pthread_barrier_wait(&bar);
+  sleep(10000);
+  return 0;
+}
+
+int main(int argc, char *argv[]) {
+  pthread_t thread_id;
+  pthread_barrier_init(&bar, 0, 3);
+
+  pthread_create(&thread_id, 0, threadfn, 0);
+  pthread_create(&thread_id, 0, threadfn, 0);
+
+  pthread_barrier_wait(&bar);
+  return 0;
+}
+
+// CHECK: Thread T0/{{[0-9]+}} was created by T-1
+// CHECK: Thread T1/{{[0-9]+}} was created by T0/
+// CHECK: Thread T2/{{[0-9]+}} was created by T0/
diff --git a/compiler-rt/test/rtsan/stack_suppressions.cpp b/compiler-rt/test/rtsan/stack_suppressions.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2aceedbb313b11df282e2f1ea8534cc1d601af93
--- /dev/null
+++ b/compiler-rt/test/rtsan/stack_suppressions.cpp
@@ -0,0 +1,53 @@
+// RUN: %clangxx -fsanitize=realtime %s -o %t
+// RUN: %env_rtsan_opts=suppressions='%s.supp' not %run %t 2>&1 | FileCheck %s
+// UNSUPPORTED: ios
+
+// Intent: Ensure that suppressions work as intended
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <vector>
+
+void *MallocViolation() { return malloc(10); }
+
+void VectorViolations() {
+  // All of these should be suppressed by *vector*
+  std::vector<int> v(10);
+  v.resize(20);
+  v.clear();
+  v.resize(0);
+  v.push_back(1);
+  v.reserve(10);
+}
+
+void BlockFunc() [[clang::blocking]] { usleep(1); }
+
+void *process() [[clang::nonblocking]] {
+  void *ptr = MallocViolation();
+  VectorViolations();
+  BlockFunc();
+  free(ptr);
+
+  // This is the one that should abort the program
+  // Everything else is suppressed
+  usleep(1);
+
+  return ptr;
+}
+
+int main() {
+  process();
+  return 0;
+}
+
+// CHECK-NOT: failed to open suppressions file
+// CHECK: Intercepted call to real-time unsafe function
+// CHECK-SAME: usleep
+
+// CHECK-NOT: Intercepted call to real-time unsafe function
+// CHECK-NOT: malloc
+// CHECK-NOT: vector
+// CHECK-NOT: free
+// CHECK-NOT: BlockFunc
diff --git a/compiler-rt/test/rtsan/stack_suppressions.cpp.supp b/compiler-rt/test/rtsan/stack_suppressions.cpp.supp
new file mode 100644
index 0000000000000000000000000000000000000000..bec4db259a3e0eabf20211b2658daaa8bf567932
--- /dev/null
+++ b/compiler-rt/test/rtsan/stack_suppressions.cpp.supp
@@ -0,0 +1,4 @@
+call-stack-contains:MallocViolation
+call-stack-contains:std::*vector
+call-stack-contains:free
+call-stack-contains:BlockFunc
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_aarch64.cpp b/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_aarch64.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f4805640d682b0f24a1377a1ea63b8a1831f5cdf
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_aarch64.cpp
@@ -0,0 +1,23 @@
+// Check that sanitizer prints registers dump_registers on dump_registers=1
+// RUN: %clangxx  %s -o %t
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NODUMP
+// RUN: not %run %t 2>&1 | FileCheck %s --strict-whitespace --check-prefix=CHECK-DUMP
+//
+// REQUIRES: aarch64-pc-windows-msvc
+
+#include <windows.h>
+
+int main() {
+  RaiseException(EXCEPTION_ACCESS_VIOLATION, 0, 0, NULL);
+  // CHECK-DUMP: Register values
+  // CHECK-DUMP-NEXT: x0 = {{0x[0-9a-f]+}}   x1 = {{0x[0-9a-f]+}}   x2 = {{0x[0-9a-f]+}}   x3 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: x4 = {{0x[0-9a-f]+}}   x5 = {{0x[0-9a-f]+}}   x6 = {{0x[0-9a-f]+}}   x7 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: x8 = {{0x[0-9a-f]+}}   x9 = {{0x[0-9a-f]+}}  x10 = {{0x[0-9a-f]+}}  x11 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:x12 = {{0x[0-9a-f]+}}  x13 = {{0x[0-9a-f]+}}  x14 = {{0x[0-9a-f]+}}  x15 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:x16 = {{0x[0-9a-f]+}}  x17 = {{0x[0-9a-f]+}}  x18 = {{0x[0-9a-f]+}}  x19 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:x20 = {{0x[0-9a-f]+}}  x21 = {{0x[0-9a-f]+}}  x22 = {{0x[0-9a-f]+}}  x23 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:x24 = {{0x[0-9a-f]+}}  x25 = {{0x[0-9a-f]+}}  x26 = {{0x[0-9a-f]+}}  x27 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:x28 = {{0x[0-9a-f]+}}   fp = {{0x[0-9a-f]+}}   lr = {{0x[0-9a-f]+}}   sp = {{0x[0-9a-f]+}}
+  // CHECK-NODUMP-NOT: Register values
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_i386.cpp b/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_i386.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..76c7a901e34fdf71686ef4503b921b71081fe6ce
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_i386.cpp
@@ -0,0 +1,17 @@
+// Check that sanitizer prints registers dump_registers on dump_registers=1
+// RUN: %clangxx  %s -o %t
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-NODUMP --strict-whitespace
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-DUMP --strict-whitespace
+//
+// REQUIRES: i386-pc-windows-msvc
+
+#include <windows.h>
+
+int main() {
+  RaiseException(EXCEPTION_ACCESS_VIOLATION, 0, 0, NULL);
+  // CHECK-DUMP: Register values
+  // CHECK-DUMP-NEXT: eax = {{0x[0-9a-f]+}}  ebx = {{0x[0-9a-f]+}}  ecx = {{0x[0-9a-f]+}}  edx = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: edi = {{0x[0-9a-f]+}}  esi = {{0x[0-9a-f]+}}  ebp = {{0x[0-9a-f]+}}  esp = {{0x[0-9a-f]+}}
+  // CHECK-NODUMP-NOT: Register values
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_x86_64.cpp b/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_x86_64.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..17c51aa27066e2d40c73dcf755fea1669a8e76bf
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Windows/dump_registers_x86_64.cpp
@@ -0,0 +1,19 @@
+// Check that sanitizer prints registers dump_registers on dump_registers=1
+// RUN: %clangxx  %s -o %t
+// RUN: %env_tool_opts=dump_registers=0 not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-NODUMP --strict-whitespace
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK-DUMP --strict-whitespace
+//
+// REQUIRES: x86_64-pc-windows-msvc
+
+#include <windows.h>
+
+int main() {
+  RaiseException(EXCEPTION_ACCESS_VIOLATION, 0, 0, NULL);
+  // CHECK-DUMP: Register values
+  // CHECK-DUMP-NEXT: rax = {{0x[0-9a-f]+}}  rbx = {{0x[0-9a-f]+}}  rcx = {{0x[0-9a-f]+}}  rdx = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: rdi = {{0x[0-9a-f]+}}  rsi = {{0x[0-9a-f]+}}  rbp = {{0x[0-9a-f]+}}  rsp = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT:  r8 = {{0x[0-9a-f]+}}   r9 = {{0x[0-9a-f]+}}  r10 = {{0x[0-9a-f]+}}  r11 = {{0x[0-9a-f]+}}
+  // CHECK-DUMP-NEXT: r12 = {{0x[0-9a-f]+}}  r13 = {{0x[0-9a-f]+}}  r14 = {{0x[0-9a-f]+}}  r15 = {{0x[0-9a-f]+}}
+  // CHECK-NODUMP-NOT: Register values
+  return 0;
+}
diff --git a/compiler-rt/test/ubsan/TestCases/Misc/Linux/diag-stacktrace.cpp b/compiler-rt/test/ubsan/TestCases/Misc/Posix/diag-stacktrace.cpp
similarity index 100%
rename from compiler-rt/test/ubsan/TestCases/Misc/Linux/diag-stacktrace.cpp
rename to compiler-rt/test/ubsan/TestCases/Misc/Posix/diag-stacktrace.cpp
diff --git a/compiler-rt/test/ubsan/TestCases/Misc/Linux/lit.local.cfg.py b/compiler-rt/test/ubsan/TestCases/Misc/Posix/lit.local.cfg.py
similarity index 82%
rename from compiler-rt/test/ubsan/TestCases/Misc/Linux/lit.local.cfg.py
rename to compiler-rt/test/ubsan/TestCases/Misc/Posix/lit.local.cfg.py
index 603ca0365068f5e6917664eff43452bb1dc856de..63240c3962565618f7be49f1cd44ef2a1f7e986c 100644
--- a/compiler-rt/test/ubsan/TestCases/Misc/Linux/lit.local.cfg.py
+++ b/compiler-rt/test/ubsan/TestCases/Misc/Posix/lit.local.cfg.py
@@ -6,5 +6,5 @@ def getRoot(config):
 
 root = getRoot(config)
 
-if root.host_os not in ["Linux"]:
+if root.host_os in ["Windows"]:
     config.unsupported = True
diff --git a/compiler-rt/test/ubsan/TestCases/Misc/Linux/print_stack_trace.cpp b/compiler-rt/test/ubsan/TestCases/Misc/Posix/print_stack_trace.cpp
similarity index 100%
rename from compiler-rt/test/ubsan/TestCases/Misc/Linux/print_stack_trace.cpp
rename to compiler-rt/test/ubsan/TestCases/Misc/Posix/print_stack_trace.cpp
diff --git a/compiler-rt/test/ubsan/TestCases/Misc/Linux/sigaction.cpp b/compiler-rt/test/ubsan/TestCases/Misc/Posix/sigaction.cpp
similarity index 100%
rename from compiler-rt/test/ubsan/TestCases/Misc/Linux/sigaction.cpp
rename to compiler-rt/test/ubsan/TestCases/Misc/Posix/sigaction.cpp
diff --git a/compiler-rt/test/ubsan/TestCases/Misc/Linux/static-link.cpp b/compiler-rt/test/ubsan/TestCases/Misc/Posix/static-link.cpp
similarity index 100%
rename from compiler-rt/test/ubsan/TestCases/Misc/Linux/static-link.cpp
rename to compiler-rt/test/ubsan/TestCases/Misc/Posix/static-link.cpp
diff --git a/compiler-rt/test/ubsan/TestCases/Misc/Linux/ubsan_options.cpp b/compiler-rt/test/ubsan/TestCases/Misc/Posix/ubsan_options.cpp
similarity index 100%
rename from compiler-rt/test/ubsan/TestCases/Misc/Linux/ubsan_options.cpp
rename to compiler-rt/test/ubsan/TestCases/Misc/Posix/ubsan_options.cpp
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index 8fd0236608a6685068020888765879c28f8e368d..06ca12a492d29b042222c187ec1bbf0351129d9b 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -492,9 +492,8 @@ public:
   READ_FEATURE(OmpLinearModifier::Type)
   READ_FEATURE(OmpLoopDirective)
   READ_FEATURE(OmpMapClause)
-  READ_FEATURE(OmpMapType)
-  READ_FEATURE(OmpMapType::Always)
-  READ_FEATURE(OmpMapType::Type)
+  READ_FEATURE(OmpMapClause::TypeModifier)
+  READ_FEATURE(OmpMapClause::Type)
   READ_FEATURE(OmpObject)
   READ_FEATURE(OmpObjectList)
   READ_FEATURE(OmpOrderClause)
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
index 00acb14ca8bbd55fb7d722f4823f82f533941cda..5d3c5cd72eef04d217c4b8b8f4f4a2b677da6c98 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
@@ -226,8 +226,8 @@ void OpenMPCounterVisitor::Post(const OmpDependenceType::Type &c) {
   clauseDetails +=
       "type=" + std::string{OmpDependenceType::EnumToString(c)} + ";";
 }
-void OpenMPCounterVisitor::Post(const OmpMapType::Type &c) {
-  clauseDetails += "type=" + std::string{OmpMapType::EnumToString(c)} + ";";
+void OpenMPCounterVisitor::Post(const OmpMapClause::Type &c) {
+  clauseDetails += "type=" + std::string{OmpMapClause::EnumToString(c)} + ";";
 }
 void OpenMPCounterVisitor::Post(const OmpScheduleClause::ScheduleType &c) {
   clauseDetails +=
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
index e1882f7b43635229fed10c415ac8fc23377c487e..380534ebbfd70ac95b1efc48d3be83891c0aaccc 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
@@ -74,7 +74,7 @@ struct OpenMPCounterVisitor {
   void Post(const OmpScheduleModifierType::ModType &c);
   void Post(const OmpLinearModifier::Type &c);
   void Post(const OmpDependenceType::Type &c);
-  void Post(const OmpMapType::Type &c);
+  void Post(const OmpMapClause::Type &c);
   void Post(const OmpScheduleClause::ScheduleType &c);
   void Post(const OmpIfClause::DirectiveNameModifier &c);
   void Post(const OmpCancelType::Type &c);
diff --git a/flang/include/flang/Frontend/CodeGenOptions.h b/flang/include/flang/Frontend/CodeGenOptions.h
index ac7fcbcba4f74766a16c02575d1698b82b449238..f19943335737b91e0ba1d1a1088c9be69224e888 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.h
+++ b/flang/include/flang/Frontend/CodeGenOptions.h
@@ -63,6 +63,9 @@ public:
   /// The directory where temp files are stored if specified by -save-temps
   std::optional<std::string> SaveTempsDir;
 
+  /// The string containing the commandline for the llvm.commandline metadata.
+  std::optional<std::string> RecordCommandLine;
+
   /// The name of the file to which the backend should save YAML optimization
   /// records.
   std::string OptRecordFile;
diff --git a/flang/include/flang/Lower/Bridge.h b/flang/include/flang/Lower/Bridge.h
index 4379ed512cdf0a4c683f0b2c0bc6a9658a41de33..8ea5ed52e2821805a6f256aaa660f2a1be27cc61 100644
--- a/flang/include/flang/Lower/Bridge.h
+++ b/flang/include/flang/Lower/Bridge.h
@@ -14,6 +14,8 @@
 #define FORTRAN_LOWER_BRIDGE_H
 
 #include "flang/Common/Fortran.h"
+#include "flang/Frontend/CodeGenOptions.h"
+#include "flang/Frontend/TargetOptions.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/EnvironmentDefault.h"
 #include "flang/Lower/LoweringOptions.h"
@@ -65,11 +67,13 @@ public:
          const Fortran::lower::LoweringOptions &loweringOptions,
          const std::vector<Fortran::lower::EnvironmentDefault> &envDefaults,
          const Fortran::common::LanguageFeatureControl &languageFeatures,
-         const llvm::TargetMachine &targetMachine, llvm::StringRef tuneCPU) {
+         const llvm::TargetMachine &targetMachine,
+         const Fortran::frontend::TargetOptions &targetOptions,
+         const Fortran::frontend::CodeGenOptions &codeGenOptions) {
     return LoweringBridge(ctx, semanticsContext, defaultKinds, intrinsics,
                           targetCharacteristics, allCooked, triple, kindMap,
                           loweringOptions, envDefaults, languageFeatures,
-                          targetMachine, tuneCPU);
+                          targetMachine, targetOptions, codeGenOptions);
   }
 
   //===--------------------------------------------------------------------===//
@@ -148,7 +152,9 @@ private:
       const Fortran::lower::LoweringOptions &loweringOptions,
       const std::vector<Fortran::lower::EnvironmentDefault> &envDefaults,
       const Fortran::common::LanguageFeatureControl &languageFeatures,
-      const llvm::TargetMachine &targetMachine, const llvm::StringRef tuneCPU);
+      const llvm::TargetMachine &targetMachine,
+      const Fortran::frontend::TargetOptions &targetOptions,
+      const Fortran::frontend::CodeGenOptions &codeGenOptions);
   LoweringBridge() = delete;
   LoweringBridge(const LoweringBridge &) = delete;
 
diff --git a/flang/include/flang/Optimizer/CodeGen/Target.h b/flang/include/flang/Optimizer/CodeGen/Target.h
index a7161152a5c3233d269805dd2630b6891055d83a..3b38583511927ab6fa3401dfc517012ab8d17b47 100644
--- a/flang/include/flang/Optimizer/CodeGen/Target.h
+++ b/flang/include/flang/Optimizer/CodeGen/Target.h
@@ -126,6 +126,11 @@ public:
   structArgumentType(mlir::Location loc, fir::RecordType recTy,
                      const Marshalling &previousArguments) const = 0;
 
+  /// Type representation of a `fir.type<T>` type argument when returned by
+  /// value. Such value may need to be converted to a hidden reference argument.
+  virtual Marshalling structReturnType(mlir::Location loc,
+                                       fir::RecordType eleTy) const = 0;
+
   /// Type representation of a `boxchar<n>` type argument when passed by value.
   /// An argument value may need to be passed as a (safe) reference argument.
   ///
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index cdbefdb2341485f064d838779c09b8d0505c19c9..fb7b1d16f62f3ac85813d6adf7ba73ac3de93c1e 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -177,6 +177,27 @@ inline mlir::NamedAttribute getAdaptToByRefAttr(Builder &builder) {
 }
 
 bool isDummyArgument(mlir::Value v);
+
+template <fir::FortranProcedureFlagsEnum Flag>
+inline bool hasProcedureAttr(fir::FortranProcedureFlagsEnumAttr flags) {
+  return flags && bitEnumContainsAny(flags.getValue(), Flag);
+}
+
+template <fir::FortranProcedureFlagsEnum Flag>
+inline bool hasProcedureAttr(mlir::Operation *op) {
+  if (auto firCallOp = mlir::dyn_cast<fir::CallOp>(op))
+    return hasProcedureAttr<Flag>(firCallOp.getProcedureAttrsAttr());
+  if (auto firCallOp = mlir::dyn_cast<fir::DispatchOp>(op))
+    return hasProcedureAttr<Flag>(firCallOp.getProcedureAttrsAttr());
+  return hasProcedureAttr<Flag>(
+      op->getAttrOfType<fir::FortranProcedureFlagsEnumAttr>(
+          getFortranProcedureFlagsAttrName()));
+}
+
+inline bool hasBindcAttr(mlir::Operation *op) {
+  return hasProcedureAttr<fir::FortranProcedureFlagsEnum::bind_c>(op);
+}
+
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_DIALECT_FIROPSSUPPORT_H
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index e45011c8e02a33b21e38f49bea08afff04618b29..2df14f83c11e1759528f6829dad0402062962669 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -77,6 +77,12 @@ void setIdent(mlir::ModuleOp mod, llvm::StringRef ident);
 /// Get the compiler identifier from the Module.
 llvm::StringRef getIdent(mlir::ModuleOp mod);
 
+/// Set the command line used in this invocation.
+void setCommandline(mlir::ModuleOp mod, llvm::StringRef cmdLine);
+
+/// Get the command line used in this invocation.
+llvm::StringRef getCommandline(mlir::ModuleOp mod);
+
 /// Helper for determining the target from the host, etc. Tools may use this
 /// function to provide a consistent interpretation of the `--target=<string>`
 /// command-line option.
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index bf00e6b43d0668492d6e5d3fb8b76dd7c1684a18..5d243b4e5d3e9a522835f6809a8b41f0dfcb3070 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -531,9 +531,8 @@ public:
   NODE_ENUM(OmpLinearModifier, Type)
   NODE(parser, OmpLoopDirective)
   NODE(parser, OmpMapClause)
-  NODE(parser, OmpMapType)
-  NODE(OmpMapType, Always)
-  NODE_ENUM(OmpMapType, Type)
+  NODE_ENUM(OmpMapClause, TypeModifier)
+  NODE_ENUM(OmpMapClause, Type)
   static std::string GetNodeName(const llvm::omp::Clause &x) {
     return llvm::Twine(
         "llvm::omp::Clause = ", llvm::omp::getOpenMPClauseName(x))
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 7057ac2267aa15d4e0fbda8f3fa902901f1b0872..21b4a344dbc438c8683f0cf731b8b1523df5e101 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3448,18 +3448,18 @@ struct OmpObject {
 
 WRAPPER_CLASS(OmpObjectList, std::list<OmpObject>);
 
-// 2.15.5.1 map-type -> TO | FROM | TOFROM | ALLOC | RELEASE | DELETE
-struct OmpMapType {
-  TUPLE_CLASS_BOILERPLATE(OmpMapType);
-  EMPTY_CLASS(Always);
-  ENUM_CLASS(Type, To, From, Tofrom, Alloc, Release, Delete)
-  std::tuple<std::optional<Always>, Type> t;
-};
-
-// 2.15.5.1 map -> MAP ([ [ALWAYS[,]] map-type : ] variable-name-list)
+// 2.15.5.1 map ->
+//    MAP ([ [map-type-modifiers [,] ] map-type : ] variable-name-list)
+// map-type-modifiers -> map-type-modifier [,] [...]
+// map-type-modifier -> ALWAYS | CLOSE | PRESENT | OMPX_HOLD
+// map-type -> TO | FROM | TOFROM | ALLOC | RELEASE | DELETE
 struct OmpMapClause {
+  ENUM_CLASS(TypeModifier, Always, Close, Present, OmpxHold);
+  ENUM_CLASS(Type, To, From, Tofrom, Alloc, Release, Delete)
   TUPLE_CLASS_BOILERPLATE(OmpMapClause);
-  std::tuple<std::optional<OmpMapType>, OmpObjectList> t;
+  std::tuple<std::optional<std::list<TypeModifier>>, std::optional<Type>,
+      OmpObjectList>
+      t;
 };
 
 // 2.15.5.2 defaultmap -> DEFAULTMAP (implicit-behavior[:variable-category])
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 849c6f53614f632384e4347e8a93273eab24891e..4607a33ffda6ccd1402951cc975933bd9e949bf9 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -349,6 +349,12 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
   if (auto *a = args.getLastArg(clang::driver::options::OPT_save_temps_EQ))
     opts.SaveTempsDir = a->getValue();
 
+  // -record-command-line option.
+  if (const llvm::opt::Arg *a =
+          args.getLastArg(clang::driver::options::OPT_record_command_line)) {
+    opts.RecordCommandLine = a->getValue();
+  }
+
   // -mlink-builtin-bitcode
   for (auto *a :
        args.filtered(clang::driver::options::OPT_mlink_builtin_bitcode))
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 8f882bff170909fe2a86bc8241fb0dba61c4edbd..f2e460fc53a67f40e027a75a39424dda56a0baa3 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -301,7 +301,7 @@ bool CodeGenAction::beginSourceFileAction() {
       kindMap, ci.getInvocation().getLoweringOpts(),
       ci.getInvocation().getFrontendOpts().envDefaults,
       ci.getInvocation().getFrontendOpts().features, targetMachine,
-      ci.getInvocation().getTargetOpts().cpuToTuneFor);
+      ci.getInvocation().getTargetOpts(), ci.getInvocation().getCodeGenOpts());
 
   // Fetch module from lb, so we can set
   mlirModule = std::make_unique<mlir::ModuleOp>(lb.getModule());
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 0894a5903635e15e67385896bfe9d041a310b48b..877fe122265dd0d8efb2236aaa13ff5fbe82b7cb 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -6062,7 +6062,9 @@ Fortran::lower::LoweringBridge::LoweringBridge(
     const Fortran::lower::LoweringOptions &loweringOptions,
     const std::vector<Fortran::lower::EnvironmentDefault> &envDefaults,
     const Fortran::common::LanguageFeatureControl &languageFeatures,
-    const llvm::TargetMachine &targetMachine, const llvm::StringRef tuneCPU)
+    const llvm::TargetMachine &targetMachine,
+    const Fortran::frontend::TargetOptions &targetOpts,
+    const Fortran::frontend::CodeGenOptions &cgOpts)
     : semanticsContext{semanticsContext}, defaultKinds{defaultKinds},
       intrinsics{intrinsics}, targetCharacteristics{targetCharacteristics},
       cooked{&cooked}, context{context}, kindMap{kindMap},
@@ -6119,11 +6121,13 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setTargetTriple(*module.get(), triple);
   fir::setKindMapping(*module.get(), kindMap);
   fir::setTargetCPU(*module.get(), targetMachine.getTargetCPU());
-  fir::setTuneCPU(*module.get(), tuneCPU);
+  fir::setTuneCPU(*module.get(), targetOpts.cpuToTuneFor);
   fir::setTargetFeatures(*module.get(), targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module.get(),
                                   targetMachine.createDataLayout());
   fir::setIdent(*module.get(), Fortran::common::getFlangFullVersion());
+  if (cgOpts.RecordCommandLine)
+    fir::setCommandline(*module.get(), *cgOpts.RecordCommandLine);
 }
 
 void Fortran::lower::genCleanUpInRegionIfAny(
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index cf91b2638aecc3958e1148db47cb58836cfef7aa..88c443b4198ab00be8556651be0868acc8fc58eb 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -945,40 +945,42 @@ bool ClauseProcessor::processMap(
             llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
         // If the map type is specified, then process it else Tofrom is the
         // default.
-        if (mapType) {
-          switch (*mapType) {
-          case Map::MapType::To:
-            mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
-            break;
-          case Map::MapType::From:
-            mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
-            break;
-          case Map::MapType::Tofrom:
-            mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
-                           llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
-            break;
-          case Map::MapType::Alloc:
-          case Map::MapType::Release:
-            // alloc and release is the default map_type for the Target Data
-            // Ops, i.e. if no bits for map_type is supplied then alloc/release
-            // is implicitly assumed based on the target directive. Default
-            // value for Target Data and Enter Data is alloc and for Exit Data
-            // it is release.
-            break;
-          case Map::MapType::Delete:
-            mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
-          }
-
-          auto &modTypeMods =
-              std::get<std::optional<Map::MapTypeModifiers>>(clause.t);
-          if (modTypeMods) {
-            if (llvm::is_contained(*modTypeMods, Map::MapTypeModifier::Always))
-              mapTypeBits |=
-                  llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
-          }
-        } else {
+        Map::MapType type = mapType.value_or(Map::MapType::Tofrom);
+        switch (type) {
+        case Map::MapType::To:
+          mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
+          break;
+        case Map::MapType::From:
+          mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
+          break;
+        case Map::MapType::Tofrom:
           mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
                          llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
+          break;
+        case Map::MapType::Alloc:
+        case Map::MapType::Release:
+          // alloc and release is the default map_type for the Target Data
+          // Ops, i.e. if no bits for map_type is supplied then alloc/release
+          // is implicitly assumed based on the target directive. Default
+          // value for Target Data and Enter Data is alloc and for Exit Data
+          // it is release.
+          break;
+        case Map::MapType::Delete:
+          mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
+        }
+
+        auto &modTypeMods =
+            std::get<std::optional<Map::MapTypeModifiers>>(clause.t);
+        if (modTypeMods) {
+          if (llvm::is_contained(*modTypeMods, Map::MapTypeModifier::Always))
+            mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
+          // Diagnose unimplemented map-type-modifiers.
+          if (llvm::any_of(*modTypeMods, [](Map::MapTypeModifier m) {
+                return m != Map::MapTypeModifier::Always;
+              })) {
+            TODO(currentLocation, "Map type modifiers (other than 'ALWAYS')"
+                                  " are not implemented yet");
+          }
         }
         processMapObjects(stmtCtx, clauseLocation,
                           std::get<omp::ObjectList>(clause.t), mapTypeBits,
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 91c1b08dd8c2f8ade3f49292f5235c8b1e96cce7..812551de68574b228a8bc6238a6efd5478afa090 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -846,41 +846,49 @@ Link make(const parser::OmpClause::Link &inp,
 Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
+  using wrapped = parser::OmpMapClause;
 
   CLAUSET_ENUM_CONVERT( //
-      convert1, parser::OmpMapType::Type, Map::MapType,
+      convert1, parser::OmpMapClause::Type, Map::MapType,
       // clang-format off
-      MS(To,       To)
-      MS(From,     From)
-      MS(Tofrom,   Tofrom)
       MS(Alloc,    Alloc)
-      MS(Release,  Release)
       MS(Delete,   Delete)
+      MS(From,     From)
+      MS(Release,  Release)
+      MS(To,       To)
+      MS(Tofrom,   Tofrom)
       // clang-format on
   );
 
-  // No convert2: MapTypeModifier is not an enum in parser.
-
-  auto &t0 = std::get<std::optional<parser::OmpMapType>>(inp.v.t);
-  auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
+  CLAUSET_ENUM_CONVERT( //
+      convert2, parser::OmpMapClause::TypeModifier, Map::MapTypeModifier,
+      // clang-format off
+      MS(Always,   Always)
+      MS(Close,    Close)
+      MS(OmpxHold, OmpxHold)
+      MS(Present,  Present)
+      // clang-format on
+  );
 
-  if (!t0) {
-    return Map{{/*MapType=*/std::nullopt, /*MapTypeModifiers=*/std::nullopt,
-                /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
-                /*LocatorList=*/makeObjects(t1, semaCtx)}};
-  }
+  auto &t0 = std::get<std::optional<std::list<wrapped::TypeModifier>>>(inp.v.t);
+  auto &t1 = std::get<std::optional<wrapped::Type>>(inp.v.t);
+  auto &t2 = std::get<parser::OmpObjectList>(inp.v.t);
 
-  auto &s0 = std::get<std::optional<parser::OmpMapType::Always>>(t0->t);
-  auto &s1 = std::get<parser::OmpMapType::Type>(t0->t);
+  std::optional<Map::MapType> maybeType = maybeApply(convert1, t1);
 
-  std::optional<Map::MapTypeModifiers> maybeList;
-  if (s0)
-    maybeList = Map::MapTypeModifiers{Map::MapTypeModifier::Always};
+  std::optional<Map::MapTypeModifiers> maybeTypeMods = maybeApply(
+      [&](const std::list<wrapped::TypeModifier> &typeMods) {
+        Map::MapTypeModifiers mods;
+        for (wrapped::TypeModifier mod : typeMods)
+          mods.push_back(convert2(mod));
+        return mods;
+      },
+      t0);
 
-  return Map{{/*MapType=*/convert1(s1),
-              /*MapTypeModifiers=*/maybeList,
+  return Map{{/*MapType=*/maybeType,
+              /*MapTypeModifiers=*/maybeTypeMods,
               /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
-              /*LocatorList=*/makeObjects(t1, semaCtx)}};
+              /*LocatorList=*/makeObjects(t2, semaCtx)}};
 }
 
 // Match: incomplete
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 2369dcd32d5e50107ed1278640b816f0d53b5b01..709ac402cc702dafd62a14ae5818b4197cbfbf75 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -195,20 +195,19 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
       // Update the original variable just before exiting the worksharing
       // loop. Conversion as follows:
       //
-      // omp.wsloop / omp.simd {             omp.wsloop / omp.simd {
-      //   omp.loop_nest {          omp.loop_nest {
-      //     ...                      ...
-      //     store          ===>      store
-      //     omp.yield                %v = arith.addi %iv, %step
-      //   }                          %cmp = %step < 0 ? %v < %ub : %v > %ub
-      //   omp.terminator             fir.if %cmp {
-      // }                              fir.store %v to %loopIV
-      //                                ^%lpv_update_blk:
+      // omp.wsloop / omp.simd {    omp.wsloop / omp.simd {
+      //   omp.loop_nest {            omp.loop_nest {
+      //     ...                        ...
+      //     store          ===>        store
+      //     omp.yield                  %v = arith.addi %iv, %step
+      //   }                            %cmp = %step < 0 ? %v < %ub : %v > %ub
+      // }                              fir.if %cmp {
+      //                                  fir.store %v to %loopIV
+      //                                  ^%lpv_update_blk:
+      //                                }
+      //                                omp.yield
       //                              }
-      //                              omp.yield
       //                            }
-      //                            omp.terminator
-      //                          }
 
       // Only generate the compare once in presence of multiple LastPrivate
       // clauses.
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0735e40ea2ca7e8a76785773652fbc4293d37e0d..70d89f5e521a59933650cbc3c21b20b26db49585 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1094,9 +1094,6 @@ static OpTy genWrapperOp(lower::AbstractConverter &converter,
   // Create entry block with arguments.
   genEntryBlock(converter, args, op.getRegion());
 
-  firOpBuilder.setInsertionPoint(
-      lower::genOpenMPTerminator(firOpBuilder, op, loc));
-
   return op;
 }
 
@@ -1310,8 +1307,8 @@ static void genTaskClauses(lower::AbstractConverter &converter,
   cp.processUntied(clauseOps);
   // TODO Support delayed privatization.
 
-  cp.processTODO<clause::Affinity, clause::Detach, clause::InReduction>(
-      loc, llvm::omp::Directive::OMPD_task);
+  cp.processTODO<clause::Affinity, clause::Detach, clause::InReduction,
+                 clause::Mergeable>(loc, llvm::omp::Directive::OMPD_task);
 }
 
 static void genTaskgroupClauses(lower::AbstractConverter &converter,
@@ -2780,7 +2777,10 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
         !std::holds_alternative<clause::ThreadLimit>(clause.u) &&
         !std::holds_alternative<clause::Threads>(clause.u) &&
         !std::holds_alternative<clause::UseDeviceAddr>(clause.u) &&
-        !std::holds_alternative<clause::UseDevicePtr>(clause.u)) {
+        !std::holds_alternative<clause::UseDevicePtr>(clause.u) &&
+        !std::holds_alternative<clause::InReduction>(clause.u) &&
+        !std::holds_alternative<clause::Mergeable>(clause.u) &&
+        !std::holds_alternative<clause::TaskReduction>(clause.u)) {
       TODO(clauseLocation, "OpenMP Block construct clause");
     }
   }
diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index e88da5a8ebae19811203bcf090d2ce8d19b42d4d..969dc748e7c54014da4beee975aae05d20226408 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -13,6 +13,8 @@
 #include "flang/Optimizer/Dialect/FortranVariableInterface.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "mlir/Analysis/AliasAnalysis.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
@@ -255,6 +257,10 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
   while (defOp && !breakFromLoop) {
     ty = defOp->getResultTypes()[0];
     llvm::TypeSwitch<Operation *>(defOp)
+        .Case<hlfir::AsExprOp>([&](auto op) {
+          v = op.getVar();
+          defOp = v.getDefiningOp();
+        })
         .Case<fir::AllocaOp, fir::AllocMemOp>([&](auto op) {
           // Unique memory allocation.
           type = SourceKind::Allocate;
@@ -296,6 +302,17 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
             defOp = v.getDefiningOp();
             return;
           }
+          // If load is inside target and it points to mapped item,
+          // continue tracking.
+          Operation *loadMemrefOp = op.getMemref().getDefiningOp();
+          bool isDeclareOp = llvm::isa<fir::DeclareOp>(loadMemrefOp) ||
+                             llvm::isa<hlfir::DeclareOp>(loadMemrefOp);
+          if (isDeclareOp &&
+              llvm::isa<omp::TargetOp>(loadMemrefOp->getParentOp())) {
+            v = op.getMemref();
+            defOp = v.getDefiningOp();
+            return;
+          }
           // No further tracking for addresses loaded from memory for now.
           type = SourceKind::Indirect;
           breakFromLoop = true;
@@ -319,6 +336,22 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
           breakFromLoop = true;
         })
         .Case<hlfir::DeclareOp, fir::DeclareOp>([&](auto op) {
+          // If declare operation is inside omp target region,
+          // continue alias analysis outside the target region
+          if (auto targetOp =
+                  llvm::dyn_cast<omp::TargetOp>(op->getParentOp())) {
+            auto argIface = cast<omp::BlockArgOpenMPOpInterface>(*targetOp);
+            for (auto [opArg, blockArg] : llvm::zip_equal(
+                     targetOp.getMapVars(), argIface.getMapBlockArgs())) {
+              if (blockArg == op.getMemref()) {
+                omp::MapInfoOp mapInfo =
+                    llvm::cast<omp::MapInfoOp>(opArg.getDefiningOp());
+                v = mapInfo.getVarPtr();
+                defOp = v.getDefiningOp();
+                return;
+              }
+            }
+          }
           auto varIf = llvm::cast<fir::FortranVariableOpInterface>(defOp);
           // While going through a declare operation collect
           // the variable attributes from it. Right now, some
diff --git a/flang/lib/Optimizer/Analysis/CMakeLists.txt b/flang/lib/Optimizer/Analysis/CMakeLists.txt
index 436d4d3f18969c15e26dada0805e3e9c3980e845..c000a9da99f871e0f7fdf8b4fa38ab9655b67933 100644
--- a/flang/lib/Optimizer/Analysis/CMakeLists.txt
+++ b/flang/lib/Optimizer/Analysis/CMakeLists.txt
@@ -6,6 +6,7 @@ add_flang_library(FIRAnalysis
   FIRDialect
   HLFIRDialect
   MLIRIR
+  MLIROpenMPDialect
 
   LINK_LIBS
   FIRBuilder
@@ -14,5 +15,6 @@ add_flang_library(FIRAnalysis
   MLIRFuncDialect
   MLIRLLVMDialect
   MLIRMathTransforms
+  MLIROpenMPDialect
   FIRSupport
 )
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 15fcc09c6219a273701d9a3eadf53a8b8ce05bf0..9b624efa05381372d2095f83dbcb94eb8afb318b 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -2808,14 +2808,14 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
   matchAndRewrite(fir::GlobalOp global, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
 
-    mlir::LLVM::DIGlobalVariableExpressionAttr dbgExpr;
+    llvm::SmallVector<mlir::Attribute> dbgExprs;
 
     if (auto fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(global.getLoc())) {
       if (auto gvAttr =
               mlir::dyn_cast_or_null<mlir::LLVM::DIGlobalVariableAttr>(
                   fusedLoc.getMetadata())) {
-        dbgExpr = mlir::LLVM::DIGlobalVariableExpressionAttr::get(
-            global.getContext(), gvAttr, mlir::LLVM::DIExpressionAttr());
+        dbgExprs.push_back(mlir::LLVM::DIGlobalVariableExpressionAttr::get(
+            global.getContext(), gvAttr, mlir::LLVM::DIExpressionAttr()));
       }
     }
 
@@ -2831,7 +2831,7 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
     llvm::ArrayRef<mlir::NamedAttribute> attrs;
     auto g = rewriter.create<mlir::LLVM::GlobalOp>(
         loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0, 0,
-        false, false, comdat, attrs, dbgExpr);
+        false, false, comdat, attrs, dbgExprs);
 
     if (global.getAlignment() && *global.getAlignment() > 0)
       g.setAlignment(*global.getAlignment());
diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp
index a12b59413f4456c7e262935d6a26050beac47714..6c148dffb0e55a2538901dad71e8858ef47859a7 100644
--- a/flang/lib/Optimizer/CodeGen/Target.cpp
+++ b/flang/lib/Optimizer/CodeGen/Target.cpp
@@ -100,6 +100,11 @@ struct GenericTarget : public CodeGenSpecifics {
     TODO(loc, "passing VALUE BIND(C) derived type for this target");
   }
 
+  CodeGenSpecifics::Marshalling
+  structReturnType(mlir::Location loc, fir::RecordType ty) const override {
+    TODO(loc, "returning BIND(C) derived type for this target");
+  }
+
   CodeGenSpecifics::Marshalling
   integerArgumentType(mlir::Location loc,
                       mlir::IntegerType argTy) const override {
@@ -533,7 +538,8 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
   /// When \p recTy is a one field record type that can be passed
   /// like the field on its own, returns the field type. Returns
   /// a null type otherwise.
-  mlir::Type passAsFieldIfOneFieldStruct(fir::RecordType recTy) const {
+  mlir::Type passAsFieldIfOneFieldStruct(fir::RecordType recTy,
+                                         bool allowComplex = false) const {
     auto typeList = recTy.getTypeList();
     if (typeList.size() != 1)
       return {};
@@ -541,6 +547,8 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     if (mlir::isa<mlir::FloatType, mlir::IntegerType, fir::LogicalType>(
             fieldType))
       return fieldType;
+    if (allowComplex && mlir::isa<mlir::ComplexType>(fieldType))
+      return fieldType;
     if (mlir::isa<fir::CharacterType>(fieldType)) {
       // Only CHARACTER(1) are expected in BIND(C) contexts, which is the only
       // contexts where derived type may be passed in registers.
@@ -593,7 +601,7 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     postMerge(byteOffset, Lo, Hi);
     if (Lo == ArgClass::Memory || Lo == ArgClass::X87 ||
         Lo == ArgClass::ComplexX87)
-      return passOnTheStack(loc, recTy);
+      return passOnTheStack(loc, recTy, /*isResult=*/false);
     int neededIntRegisters = 0;
     int neededSSERegisters = 0;
     if (Lo == ArgClass::SSE)
@@ -609,7 +617,7 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     // all in registers or all on the stack).
     if (!hasEnoughRegisters(loc, neededIntRegisters, neededSSERegisters,
                             previousArguments))
-      return passOnTheStack(loc, recTy);
+      return passOnTheStack(loc, recTy, /*isResult=*/false);
 
     if (auto fieldType = passAsFieldIfOneFieldStruct(recTy)) {
       CodeGenSpecifics::Marshalling marshal;
@@ -641,9 +649,57 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     return marshal;
   }
 
+  CodeGenSpecifics::Marshalling
+  structReturnType(mlir::Location loc, fir::RecordType recTy) const override {
+    std::uint64_t byteOffset = 0;
+    ArgClass Lo, Hi;
+    Lo = Hi = ArgClass::NoClass;
+    byteOffset = classifyStruct(loc, recTy, byteOffset, Lo, Hi);
+    mlir::MLIRContext *context = recTy.getContext();
+    postMerge(byteOffset, Lo, Hi);
+    if (Lo == ArgClass::Memory)
+      return passOnTheStack(loc, recTy, /*isResult=*/true);
+
+    // Note that X87/ComplexX87 are passed in memory, but returned via %st0
+    // %st1 registers. Here, they are returned as fp80 or {fp80, fp80} by
+    // passAsFieldIfOneFieldStruct, and LLVM will use the expected registers.
+
+    // Note that {_Complex long double} is not 100% clear from an ABI
+    // perspective because the aggregate post merger rules say it should be
+    // passed in memory because it is bigger than 2 eight bytes. This has the
+    // funny effect of
+    // {_Complex long double} return to be dealt with differently than
+    // _Complex long double.
+
+    if (auto fieldType =
+            passAsFieldIfOneFieldStruct(recTy, /*allowComplex=*/true)) {
+      if (auto complexType = mlir::dyn_cast<mlir::ComplexType>(fieldType))
+        return complexReturnType(loc, complexType.getElementType());
+      CodeGenSpecifics::Marshalling marshal;
+      marshal.emplace_back(fieldType, AT{});
+      return marshal;
+    }
+
+    if (Hi == ArgClass::NoClass || Hi == ArgClass::SSEUp) {
+      // Return a single integer or floating point argument.
+      mlir::Type lowType = pickLLVMArgType(loc, context, Lo, byteOffset);
+      CodeGenSpecifics::Marshalling marshal;
+      marshal.emplace_back(lowType, AT{});
+      return marshal;
+    }
+    // Will be returned in two different registers. Generate {lowTy, HiTy} for
+    // the LLVM IR result type.
+    CodeGenSpecifics::Marshalling marshal;
+    mlir::Type lowType = pickLLVMArgType(loc, context, Lo, 8u);
+    mlir::Type hiType = pickLLVMArgType(loc, context, Hi, byteOffset - 8u);
+    marshal.emplace_back(mlir::TupleType::get(context, {lowType, hiType}),
+                         AT{});
+    return marshal;
+  }
+
   /// Marshal an argument that must be passed on the stack.
-  CodeGenSpecifics::Marshalling passOnTheStack(mlir::Location loc,
-                                               mlir::Type ty) const {
+  CodeGenSpecifics::Marshalling
+  passOnTheStack(mlir::Location loc, mlir::Type ty, bool isResult) const {
     CodeGenSpecifics::Marshalling marshal;
     auto sizeAndAlign =
         fir::getTypeSizeAndAlignmentOrCrash(loc, ty, getDataLayout(), kindMap);
@@ -651,7 +707,7 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     unsigned short align =
         std::max(sizeAndAlign.second, static_cast<unsigned short>(8));
     marshal.emplace_back(fir::ReferenceType::get(ty),
-                         AT{align, /*byval=*/true, /*sret=*/false});
+                         AT{align, /*byval=*/!isResult, /*sret=*/isResult});
     return marshal;
   }
 };
diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
index fd56fd6bf50f447049a9cb1d2d404aad895eacf4..04a3ea684642c82a5a2882a49e1b32978e8f5c7f 100644
--- a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
@@ -142,20 +142,16 @@ public:
 
   mlir::ModuleOp getModule() { return getOperation(); }
 
-  template <typename A, typename B, typename C>
+  template <typename Ty, typename Callback>
   std::optional<std::function<mlir::Value(mlir::Operation *)>>
-  rewriteCallComplexResultType(
-      mlir::Location loc, A ty, B &newResTys,
-      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs, C &newOpers,
-      mlir::Value &savedStackPtr) {
-    if (noComplexConversion) {
-      newResTys.push_back(ty);
-      return std::nullopt;
-    }
-    auto m = specifics->complexReturnType(loc, ty.getElementType());
-    // Currently targets mandate COMPLEX is a single aggregate or packed
-    // scalar, including the sret case.
-    assert(m.size() == 1 && "target of complex return not supported");
+  rewriteCallResultType(mlir::Location loc, mlir::Type originalResTy,
+                        Ty &newResTys,
+                        fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                        Callback &newOpers, mlir::Value &savedStackPtr,
+                        fir::CodeGenSpecifics::Marshalling &m) {
+    // Currently, targets mandate COMPLEX or STRUCT is a single aggregate or
+    // packed scalar, including the sret case.
+    assert(m.size() == 1 && "return type not supported on this target");
     auto resTy = std::get<mlir::Type>(m[0]);
     auto attr = std::get<fir::CodeGenSpecifics::Attributes>(m[0]);
     if (attr.isSRet()) {
@@ -170,7 +166,7 @@ public:
       newInTyAndAttrs.push_back(m[0]);
       newOpers.push_back(stack);
       return [=](mlir::Operation *) -> mlir::Value {
-        auto memTy = fir::ReferenceType::get(ty);
+        auto memTy = fir::ReferenceType::get(originalResTy);
         auto cast = rewriter->create<fir::ConvertOp>(loc, memTy, stack);
         return rewriter->create<fir::LoadOp>(loc, cast);
       };
@@ -180,11 +176,41 @@ public:
       // We are going to generate an alloca, so save the stack pointer.
       if (!savedStackPtr)
         savedStackPtr = genStackSave(loc);
-      return this->convertValueInMemory(loc, call->getResult(0), ty,
+      return this->convertValueInMemory(loc, call->getResult(0), originalResTy,
                                         /*inputMayBeBigger=*/true);
     };
   }
 
+  template <typename Ty, typename Callback>
+  std::optional<std::function<mlir::Value(mlir::Operation *)>>
+  rewriteCallComplexResultType(
+      mlir::Location loc, mlir::ComplexType ty, Ty &newResTys,
+      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs, Callback &newOpers,
+      mlir::Value &savedStackPtr) {
+    if (noComplexConversion) {
+      newResTys.push_back(ty);
+      return std::nullopt;
+    }
+    auto m = specifics->complexReturnType(loc, ty.getElementType());
+    return rewriteCallResultType(loc, ty, newResTys, newInTyAndAttrs, newOpers,
+                                 savedStackPtr, m);
+  }
+
+  template <typename Ty, typename Callback>
+  std::optional<std::function<mlir::Value(mlir::Operation *)>>
+  rewriteCallStructResultType(
+      mlir::Location loc, fir::RecordType recTy, Ty &newResTys,
+      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs, Callback &newOpers,
+      mlir::Value &savedStackPtr) {
+    if (noStructConversion) {
+      newResTys.push_back(recTy);
+      return std::nullopt;
+    }
+    auto m = specifics->structReturnType(loc, recTy);
+    return rewriteCallResultType(loc, recTy, newResTys, newInTyAndAttrs,
+                                 newOpers, savedStackPtr, m);
+  }
+
   void passArgumentOnStackOrWithNewType(
       mlir::Location loc, fir::CodeGenSpecifics::TypeAndAttr newTypeAndAttr,
       mlir::Type oldType, mlir::Value oper,
@@ -356,6 +382,11 @@ public:
                                                 newInTyAndAttrs, newOpers,
                                                 savedStackPtr);
           })
+          .template Case<fir::RecordType>([&](fir::RecordType recTy) {
+            wrap = rewriteCallStructResultType(loc, recTy, newResTys,
+                                               newInTyAndAttrs, newOpers,
+                                               savedStackPtr);
+          })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
     } else if (fnTy.getResults().size() > 1) {
       TODO(loc, "multiple results not supported yet");
@@ -562,6 +593,24 @@ public:
     }
   }
 
+  template <typename Ty>
+  void
+  lowerStructSignatureRes(mlir::Location loc, fir::RecordType recTy,
+                          Ty &newResTys,
+                          fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs) {
+    if (noComplexConversion) {
+      newResTys.push_back(recTy);
+      return;
+    } else {
+      for (auto &tup : specifics->structReturnType(loc, recTy)) {
+        if (std::get<fir::CodeGenSpecifics::Attributes>(tup).isSRet())
+          newInTyAndAttrs.push_back(tup);
+        else
+          newResTys.push_back(std::get<mlir::Type>(tup));
+      }
+    }
+  }
+
   void
   lowerStructSignatureArg(mlir::Location loc, fir::RecordType recTy,
                           fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs) {
@@ -595,6 +644,9 @@ public:
           .Case<mlir::ComplexType>([&](mlir::ComplexType ty) {
             lowerComplexSignatureRes(loc, ty, newResTys, newInTyAndAttrs);
           })
+          .Case<fir::RecordType>([&](fir::RecordType ty) {
+            lowerStructSignatureRes(loc, ty, newResTys, newInTyAndAttrs);
+          })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
     }
     llvm::SmallVector<mlir::Type> trailingInTys;
@@ -696,7 +748,8 @@ public:
     for (auto ty : func.getResults())
       if ((mlir::isa<fir::BoxCharType>(ty) && !noCharacterConversion) ||
           (fir::isa_complex(ty) && !noComplexConversion) ||
-          (mlir::isa<mlir::IntegerType>(ty) && hasCCallingConv)) {
+          (mlir::isa<mlir::IntegerType>(ty) && hasCCallingConv) ||
+          (mlir::isa<fir::RecordType>(ty) && !noStructConversion)) {
         LLVM_DEBUG(llvm::dbgs() << "rewrite " << signature << " for target\n");
         return false;
       }
@@ -770,6 +823,9 @@ public:
                                                 rewriter->getUnitAttr()));
             newResTys.push_back(retTy);
           })
+          .Case<fir::RecordType>([&](fir::RecordType recTy) {
+            doStructReturn(func, recTy, newResTys, newInTyAndAttrs, fixups);
+          })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
 
     // Saved potential shift in argument. Handling of result can add arguments
@@ -1062,21 +1118,12 @@ public:
     return false;
   }
 
-  /// Convert a complex return value. This can involve converting the return
-  /// value to a "hidden" first argument or packing the complex into a wide
-  /// GPR.
   template <typename Ty, typename FIXUPS>
-  void doComplexReturn(mlir::func::FuncOp func, mlir::ComplexType cmplx,
-                       Ty &newResTys,
-                       fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
-                       FIXUPS &fixups) {
-    if (noComplexConversion) {
-      newResTys.push_back(cmplx);
-      return;
-    }
-    auto m =
-        specifics->complexReturnType(func.getLoc(), cmplx.getElementType());
-    assert(m.size() == 1);
+  void doReturn(mlir::func::FuncOp func, Ty &newResTys,
+                fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                FIXUPS &fixups, fir::CodeGenSpecifics::Marshalling &m) {
+    assert(m.size() == 1 &&
+           "expect result to be turned into single argument or result so far");
     auto &tup = m[0];
     auto attr = std::get<fir::CodeGenSpecifics::Attributes>(tup);
     auto argTy = std::get<mlir::Type>(tup);
@@ -1117,6 +1164,36 @@ public:
     newResTys.push_back(argTy);
   }
 
+  /// Convert a complex return value. This can involve converting the return
+  /// value to a "hidden" first argument or packing the complex into a wide
+  /// GPR.
+  template <typename Ty, typename FIXUPS>
+  void doComplexReturn(mlir::func::FuncOp func, mlir::ComplexType cmplx,
+                       Ty &newResTys,
+                       fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                       FIXUPS &fixups) {
+    if (noComplexConversion) {
+      newResTys.push_back(cmplx);
+      return;
+    }
+    auto m =
+        specifics->complexReturnType(func.getLoc(), cmplx.getElementType());
+    doReturn(func, newResTys, newInTyAndAttrs, fixups, m);
+  }
+
+  template <typename Ty, typename FIXUPS>
+  void doStructReturn(mlir::func::FuncOp func, fir::RecordType recTy,
+                      Ty &newResTys,
+                      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                      FIXUPS &fixups) {
+    if (noStructConversion) {
+      newResTys.push_back(recTy);
+      return;
+    }
+    auto m = specifics->structReturnType(func.getLoc(), recTy);
+    doReturn(func, newResTys, newInTyAndAttrs, fixups, m);
+  }
+
   template <typename FIXUPS>
   void
   createFuncOpArgFixups(mlir::func::FuncOp func,
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 5bd8e2d4336361661b8372bfb8dd6ab10829e9c1..01c0be66d1ecc35f56c65ae3b3e2f65ec98f51d4 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -130,6 +130,22 @@ llvm::StringRef fir::getIdent(mlir::ModuleOp mod) {
   return {};
 }
 
+void fir::setCommandline(mlir::ModuleOp mod, llvm::StringRef cmdLine) {
+  if (cmdLine.empty())
+    return;
+
+  mlir::MLIRContext *ctx = mod.getContext();
+  mod->setAttr(mlir::LLVM::LLVMDialect::getCommandlineAttrName(),
+               mlir::StringAttr::get(ctx, cmdLine));
+}
+
+llvm::StringRef fir::getCommandline(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::StringAttr>(
+          mlir::LLVM::LLVMDialect::getCommandlineAttrName()))
+    return attr;
+  return {};
+}
+
 std::string fir::determineTargetTriple(llvm::StringRef triple) {
   // Treat "" or "default" as stand-ins for the default machine.
   if (triple.empty() || triple == "default")
diff --git a/flang/lib/Optimizer/Transforms/AbstractResult.cpp b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
index 7299ff80121e13539c22866cd951ae9e0f75f30e..e64280508755a42acc282fdc6cac555c8bc1b784 100644
--- a/flang/lib/Optimizer/Transforms/AbstractResult.cpp
+++ b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
@@ -32,6 +32,36 @@ using namespace mlir;
 namespace fir {
 namespace {
 
+// Helper to only build the symbol table if needed because its build time is
+// linear on the number of symbols in the module.
+struct LazySymbolTable {
+  LazySymbolTable(mlir::Operation *op)
+      : module{op->getParentOfType<mlir::ModuleOp>()} {}
+  void build() {
+    if (table)
+      return;
+    table = std::make_unique<mlir::SymbolTable>(module);
+  }
+
+  template <typename T>
+  T lookup(llvm::StringRef name) {
+    build();
+    return table->lookup<T>(name);
+  }
+
+private:
+  std::unique_ptr<mlir::SymbolTable> table;
+  mlir::ModuleOp module;
+};
+
+bool hasScalarDerivedResult(mlir::FunctionType funTy) {
+  // C_PTR/C_FUNPTR are results to void* in this pass, do not consider
+  // them as normal derived types.
+  return funTy.getNumResults() == 1 &&
+         mlir::isa<fir::RecordType>(funTy.getResult(0)) &&
+         !fir::isa_builtin_cptr_type(funTy.getResult(0));
+}
+
 static mlir::Type getResultArgumentType(mlir::Type resultType,
                                         bool shouldBoxResult) {
   return llvm::TypeSwitch<mlir::Type, mlir::Type>(resultType)
@@ -190,7 +220,15 @@ public:
   llvm::LogicalResult
   matchAndRewrite(fir::SaveResultOp op,
                   mlir::PatternRewriter &rewriter) const override {
-    rewriter.eraseOp(op);
+    mlir::Operation *call = op.getValue().getDefiningOp();
+    mlir::Type type = op.getValue().getType();
+    if (mlir::isa<fir::RecordType>(type) && call && fir::hasBindcAttr(call) &&
+        !fir::isa_builtin_cptr_type(type)) {
+      rewriter.replaceOpWithNewOp<fir::StoreOp>(op, op.getValue(),
+                                                op.getMemref());
+    } else {
+      rewriter.eraseOp(op);
+    }
     return mlir::success();
   }
 };
@@ -300,6 +338,12 @@ public:
     auto *context = &getContext();
     // Convert function type itself if it has an abstract result.
     auto funcTy = mlir::cast<mlir::FunctionType>(func.getFunctionType());
+    // Scalar derived result of BIND(C) function must be returned according
+    // to the C struct return ABI which is target dependent and implemented in
+    // the target-rewrite pass.
+    if (hasScalarDerivedResult(funcTy) &&
+        fir::hasBindcAttr(func.getOperation()))
+      return;
     if (hasAbstractResult(funcTy)) {
       if (fir::isa_builtin_cptr_type(funcTy.getResult(0))) {
         func.setType(getCPtrFunctionType(funcTy));
@@ -395,6 +439,8 @@ public:
       return;
     }
 
+    LazySymbolTable symbolTable(op);
+
     mlir::RewritePatternSet patterns(context);
     mlir::ConversionTarget target = *context;
     const bool shouldBoxResult = this->passResultAsBox.getValue();
@@ -409,14 +455,29 @@ public:
                            mlir::func::FuncDialect>();
     target.addIllegalOp<fir::SaveResultOp>();
     target.addDynamicallyLegalOp<fir::CallOp>([](fir::CallOp call) {
-      return !hasAbstractResult(call.getFunctionType());
+      mlir::FunctionType funTy = call.getFunctionType();
+      if (hasScalarDerivedResult(funTy) &&
+          fir::hasBindcAttr(call.getOperation()))
+        return true;
+      return !hasAbstractResult(funTy);
     });
-    target.addDynamicallyLegalOp<fir::AddrOfOp>([](fir::AddrOfOp addrOf) {
-      if (auto funTy = mlir::dyn_cast<mlir::FunctionType>(addrOf.getType()))
+    target.addDynamicallyLegalOp<fir::AddrOfOp>([&symbolTable](
+                                                    fir::AddrOfOp addrOf) {
+      if (auto funTy = mlir::dyn_cast<mlir::FunctionType>(addrOf.getType())) {
+        if (hasScalarDerivedResult(funTy)) {
+          auto func = symbolTable.lookup<mlir::func::FuncOp>(
+              addrOf.getSymbol().getRootReference().getValue());
+          return func && fir::hasBindcAttr(func.getOperation());
+        }
         return !hasAbstractResult(funTy);
+      }
       return true;
     });
     target.addDynamicallyLegalOp<fir::DispatchOp>([](fir::DispatchOp dispatch) {
+      mlir::FunctionType funTy = dispatch.getFunctionType();
+      if (hasScalarDerivedResult(funTy) &&
+          fir::hasBindcAttr(dispatch.getOperation()))
+        return true;
       return !hasAbstractResult(dispatch.getFunctionType());
     });
 
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index d9e7bd684d4600d6209cfc7b489b07a43af11676..0c474f463f09c15eb2d779cf81f44705064cb095 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -375,7 +375,7 @@ mlir::LogicalResult AllocationAnalysis::visitOperation(
     }
   } else if (mlir::isa<fir::ResultOp>(op)) {
     mlir::Operation *parent = op->getParentOp();
-    LatticePoint *parentLattice = getLattice(parent);
+    LatticePoint *parentLattice = getLattice(getProgramPointAfter(parent));
     assert(parentLattice);
     mlir::ChangeResult parentChanged = parentLattice->join(*after);
     propagateIfChanged(parentLattice, parentChanged);
@@ -396,28 +396,29 @@ void AllocationAnalysis::setToEntryState(LatticePoint *lattice) {
 /// Mostly a copy of AbstractDenseLattice::processOperation - the difference
 /// being that call operations are passed through to the transfer function
 mlir::LogicalResult AllocationAnalysis::processOperation(mlir::Operation *op) {
+  mlir::ProgramPoint *point = getProgramPointAfter(op);
   // If the containing block is not executable, bail out.
-  if (!getOrCreateFor<mlir::dataflow::Executable>(op, op->getBlock())->isLive())
+  if (op->getBlock() != nullptr &&
+      !getOrCreateFor<mlir::dataflow::Executable>(
+           point, getProgramPointBefore(op->getBlock()))
+           ->isLive())
     return mlir::success();
 
   // Get the dense lattice to update
-  mlir::dataflow::AbstractDenseLattice *after = getLattice(op);
+  mlir::dataflow::AbstractDenseLattice *after = getLattice(point);
 
   // If this op implements region control-flow, then control-flow dictates its
   // transfer function.
   if (auto branch = mlir::dyn_cast<mlir::RegionBranchOpInterface>(op)) {
-    visitRegionBranchOperation(op, branch, after);
+    visitRegionBranchOperation(point, branch, after);
     return mlir::success();
   }
 
   // pass call operations through to the transfer function
 
   // Get the dense state before the execution of the op.
-  const mlir::dataflow::AbstractDenseLattice *before;
-  if (mlir::Operation *prev = op->getPrevNode())
-    before = getLatticeFor(op, prev);
-  else
-    before = getLatticeFor(op, op->getBlock());
+  const mlir::dataflow::AbstractDenseLattice *before =
+      getLatticeFor(point, getProgramPointBefore(op));
 
   /// Invoke the operation transfer function
   return visitOperationImpl(op, *before, after);
@@ -452,9 +453,10 @@ StackArraysAnalysisWrapper::analyseFunction(mlir::Operation *func) {
     return mlir::failure();
   }
 
-  LatticePoint point{func};
+  LatticePoint point{solver.getProgramPointAfter(func)};
   auto joinOperationLattice = [&](mlir::Operation *op) {
-    const LatticePoint *lattice = solver.lookupState<LatticePoint>(op);
+    const LatticePoint *lattice =
+        solver.lookupState<LatticePoint>(solver.getProgramPointAfter(op));
     // there will be no lattice for an unreachable block
     if (lattice)
       (void)point.join(*lattice);
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 214d6b4a91087bda00a18d8c31939098e8246c27..8634c522cf343aacdd8459ac0c8172414bc9492e 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -23,6 +23,70 @@ namespace Fortran::parser {
 constexpr auto startOmpLine = skipStuffBeforeStatement >> "!$OMP "_sptok;
 constexpr auto endOmpLine = space >> endOfLine;
 
+// Map modifiers come from two categories: map-type-modifier and map-type.
+// There can be zero or more map-type-modifiers, and zero or one map-type.
+// Syntax-wise they look like a single list, where the last element could
+// be a map-type, and all elements in that list are comma-separated[1].
+// Only if there was at least one modifier (of any kind) specified, the
+// list must end with ":".
+// [1] Any of the commas are optional, but that syntax has been deprecated,
+// and the parsing code is intended to identify that. There are complications
+// coming from the fact that the comma separating the two kinds of modifiers
+// is only allowed if there is at least one modifier of each kind.
+// The MapModifiers parser parses the modifier list as a whole, and returns
+// a tuple with the (optional) map-type-modifier list, and the (optional)
+// type modifier as its members.
+// The list is then parsed, first with a mandatory separator, and if that
+// fails, with an optional one. If the latter succeeds, a deprecation
+// message is printed.
+template <typename Separator> struct MapModifiers {
+  constexpr MapModifiers(
+      Separator sep, std::optional<MessageFixedText> msg = std::nullopt)
+      : sep_(sep), msg_(msg) {}
+  constexpr MapModifiers(const MapModifiers &) = default;
+  constexpr MapModifiers(MapModifiers &&) = default;
+
+  using resultType =
+      std::tuple<std::optional<std::list<OmpMapClause::TypeModifier>>,
+          std::optional<OmpMapClause::Type>>;
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    auto pmod{Parser<OmpMapClause::TypeModifier>{}};
+    auto ptype{Parser<OmpMapClause::Type>{}};
+    auto startLoc{state.GetLocation()};
+
+    auto &&[mods, type] = [&]() -> resultType {
+      // The 'maybe' will return optional<optional<list>>, and the outer
+      // optional will never be nullopt.
+      if (auto mods{
+              *maybe(attempt(nonemptySeparated(pmod, sep_))).Parse(state)}) {
+        // mods = optional<list>, and the list is nonempty.
+        return attempt(sep_).Parse(state)
+            ? resultType(mods, *maybe(attempt(ptype)).Parse(state))
+            : resultType(mods, std::nullopt);
+      }
+      return {std::nullopt, *maybe(attempt(ptype)).Parse(state)};
+    }();
+    auto endLoc{state.GetLocation()};
+
+    // The above always "succeeds", i.e. even if the input is junk, it will
+    // return a tuple with two nullopts. If any of the components is not a
+    // nullopt, expect a ":".
+    if ((mods.has_value() || type.has_value()) &&
+        !attempt(":"_tok).Parse(state)) {
+      return std::nullopt;
+    }
+    if (msg_) {
+      state.Say(CharBlock{startLoc, endLoc}, *msg_);
+    }
+    return resultType(mods, type);
+  }
+
+private:
+  const Separator sep_;
+  std::optional<MessageFixedText> msg_;
+};
+
 // OpenMP Clauses
 // 2.15.3.1 DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
 TYPE_PARSER(construct<OmpDefaultClause>(
@@ -38,20 +102,39 @@ TYPE_PARSER(construct<OmpProcBindClause>(
     "PRIMARY" >> pure(OmpProcBindClause::Type::Primary) ||
     "SPREAD" >> pure(OmpProcBindClause::Type::Spread)))
 
-// 2.15.5.1 MAP ([ [ALWAYS[,]] map-type : ] variable-name-list)
-//          map-type -> TO | FROM | TOFROM | ALLOC | RELEASE | DELETE
-TYPE_PARSER(construct<OmpMapType>(
-    maybe("ALWAYS" >> construct<OmpMapType::Always>() / maybe(","_tok)),
-    ("TO"_id >> pure(OmpMapType::Type::To) ||
-        "FROM" >> pure(OmpMapType::Type::From) ||
-        "TOFROM" >> pure(OmpMapType::Type::Tofrom) ||
-        "ALLOC" >> pure(OmpMapType::Type::Alloc) ||
-        "RELEASE" >> pure(OmpMapType::Type::Release) ||
-        "DELETE" >> pure(OmpMapType::Type::Delete)) /
-        ":"))
-
-TYPE_PARSER(construct<OmpMapClause>(
-    maybe(Parser<OmpMapType>{}), Parser<OmpObjectList>{}))
+// 2.15.5.1 map ->
+//    MAP ([ [map-type-modifiers [,] ] map-type : ] variable-name-list)
+// map-type-modifiers -> map-type-modifier [,] [...]
+// map-type-modifier -> ALWAYS | CLOSE | OMPX_HOLD | PRESENT
+// map-type -> ALLOC | DELETE | FROM | RELEASE | TO | TOFROM
+TYPE_PARSER(construct<OmpMapClause::TypeModifier>(
+    "ALWAYS" >> pure(OmpMapClause::TypeModifier::Always) ||
+    "CLOSE" >> pure(OmpMapClause::TypeModifier::Close) ||
+    "OMPX_HOLD" >> pure(OmpMapClause::TypeModifier::OmpxHold) ||
+    "PRESENT" >> pure(OmpMapClause::TypeModifier::Present)))
+
+TYPE_PARSER(
+    construct<OmpMapClause::Type>("ALLOC" >> pure(OmpMapClause::Type::Alloc) ||
+        "DELETE" >> pure(OmpMapClause::Type::Delete) ||
+        "FROM" >> pure(OmpMapClause::Type::From) ||
+        "RELEASE" >> pure(OmpMapClause::Type::Release) ||
+        "TO"_id >> pure(OmpMapClause::Type::To) ||
+        "TOFROM" >> pure(OmpMapClause::Type::Tofrom)))
+
+static inline OmpMapClause makeMapClause(
+    std::tuple<std::optional<std::list<OmpMapClause::TypeModifier>>,
+        std::optional<OmpMapClause::Type>> &&mod,
+    OmpObjectList &&obj) {
+  return OmpMapClause{
+      std::move(std::get<0>(mod)), std::move(std::get<1>(mod)), std::move(obj)};
+}
+
+TYPE_PARSER(construct<OmpMapClause>(applyFunction<OmpMapClause>(makeMapClause,
+    (MapModifiers(","_tok) ||
+        MapModifiers(maybe(","_tok),
+            "the specification of modifiers without comma separators for the "
+            "'MAP' clause has been deprecated"_port_en_US)),
+    Parser<OmpObjectList>{})))
 
 // [OpenMP 5.0]
 // 2.19.7.2 defaultmap(implicit-behavior[:variable-category])
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ab01d82537c03f51241027ef0d7e471d1f7bcc2a..d1011fe58a0264baf8866c4f678dd508b2ecc44b 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2073,11 +2073,27 @@ public:
         ":");
     Walk(std::get<OmpObjectList>(x.t));
   }
-  void Unparse(const OmpMapType::Always &) { Word("ALWAYS,"); }
   void Unparse(const OmpMapClause &x) {
-    Walk(std::get<std::optional<OmpMapType>>(x.t), ":");
+    auto &typeMod =
+        std::get<std::optional<std::list<OmpMapClause::TypeModifier>>>(x.t);
+    auto &type = std::get<std::optional<OmpMapClause::Type>>(x.t);
+    Walk(typeMod);
+    if (typeMod.has_value() && type.has_value()) {
+      Put(", ");
+    }
+    Walk(type);
+    if (typeMod.has_value() || type.has_value()) {
+      Put(": ");
+    }
     Walk(std::get<OmpObjectList>(x.t));
   }
+  void Unparse(const OmpMapClause::TypeModifier &x) {
+    if (x == OmpMapClause::TypeModifier::OmpxHold) {
+      Word("OMPX_HOLD");
+    } else {
+      Word(OmpMapClause::EnumToString(x));
+    }
+  }
   void Unparse(const OmpScheduleModifier &x) {
     Walk(std::get<OmpScheduleModifier::Modifier1>(x.t));
     Walk(",", std::get<std::optional<OmpScheduleModifier::Modifier2>>(x.t));
@@ -2775,7 +2791,6 @@ public:
   WALK_NESTED_ENUM(OmpScheduleModifierType, ModType) // OMP schedule-modifier
   WALK_NESTED_ENUM(OmpLinearModifier, Type) // OMP linear-modifier
   WALK_NESTED_ENUM(OmpDependenceType, Type) // OMP dependence-type
-  WALK_NESTED_ENUM(OmpMapType, Type) // OMP map-type
   WALK_NESTED_ENUM(OmpScheduleClause, ScheduleType) // OMP schedule-type
   WALK_NESTED_ENUM(OmpDeviceClause, DeviceModifier) // OMP device modifier
   WALK_NESTED_ENUM(OmpDeviceTypeClause, Type) // OMP DEVICE_TYPE
@@ -2785,6 +2800,7 @@ public:
   WALK_NESTED_ENUM(OmpCancelType, Type) // OMP cancel-type
   WALK_NESTED_ENUM(OmpOrderClause, Type) // OMP order-type
   WALK_NESTED_ENUM(OmpOrderModifier, Kind) // OMP order-modifier
+  WALK_NESTED_ENUM(OmpMapClause, Type) // OMP map-type
 #undef WALK_NESTED_ENUM
   void Unparse(const ReductionOperator::Operator x) {
     switch (x) {
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 39c4258ceb56f408d52342743c851226eafcd3d9..5cdc8602cf4c082aba8c648968019ec4d6234dfe 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -3053,15 +3053,15 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
 }
 
 void OmpStructureChecker::CheckAllowedMapTypes(
-    const parser::OmpMapType::Type &type,
-    const std::list<parser::OmpMapType::Type> &allowedMapTypeList) {
+    const parser::OmpMapClause::Type &type,
+    const std::list<parser::OmpMapClause::Type> &allowedMapTypeList) {
   if (!llvm::is_contained(allowedMapTypeList, type)) {
     std::string commaSeparatedMapTypes;
     llvm::interleave(
         allowedMapTypeList.begin(), allowedMapTypeList.end(),
-        [&](const parser::OmpMapType::Type &mapType) {
+        [&](const parser::OmpMapClause::Type &mapType) {
           commaSeparatedMapTypes.append(parser::ToUpperCaseLetters(
-              parser::OmpMapType::EnumToString(mapType)));
+              parser::OmpMapClause::EnumToString(mapType)));
         },
         [&] { commaSeparatedMapTypes.append(", "); });
     context_.Say(GetContext().clauseSource,
@@ -3073,10 +3073,9 @@ void OmpStructureChecker::CheckAllowedMapTypes(
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_map);
+  using Type = parser::OmpMapClause::Type;
 
-  if (const auto &maptype{std::get<std::optional<parser::OmpMapType>>(x.v.t)}) {
-    using Type = parser::OmpMapType::Type;
-    const Type &type{std::get<Type>(maptype->t)};
+  if (const auto &mapType{std::get<std::optional<Type>>(x.v.t)}) {
     switch (GetContext().directive) {
     case llvm::omp::Directive::OMPD_target:
     case llvm::omp::Directive::OMPD_target_teams:
@@ -3086,13 +3085,13 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
     case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do_simd:
     case llvm::omp::Directive::OMPD_target_data:
       CheckAllowedMapTypes(
-          type, {Type::To, Type::From, Type::Tofrom, Type::Alloc});
+          *mapType, {Type::To, Type::From, Type::Tofrom, Type::Alloc});
       break;
     case llvm::omp::Directive::OMPD_target_enter_data:
-      CheckAllowedMapTypes(type, {Type::To, Type::Alloc});
+      CheckAllowedMapTypes(*mapType, {Type::To, Type::Alloc});
       break;
     case llvm::omp::Directive::OMPD_target_exit_data:
-      CheckAllowedMapTypes(type, {Type::From, Type::Release, Type::Delete});
+      CheckAllowedMapTypes(*mapType, {Type::From, Type::Release, Type::Delete});
       break;
     default:
       break;
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index a8e60b411e1846e99684b2fc7d0c98ec64de9120..cce9fa4e3016e1c460875214d3eb1d4085269935 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -158,8 +158,8 @@ private:
   // specific clause related
   bool ScheduleModifierHasType(const parser::OmpScheduleClause &,
       const parser::OmpScheduleModifierType::ModType &);
-  void CheckAllowedMapTypes(const parser::OmpMapType::Type &,
-      const std::list<parser::OmpMapType::Type> &);
+  void CheckAllowedMapTypes(const parser::OmpMapClause::Type &,
+      const std::list<parser::OmpMapClause::Type> &);
   llvm::StringRef getClauseName(llvm::omp::Clause clause) override;
   llvm::StringRef getDirectiveName(llvm::omp::Directive directive) override;
 
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 3d3630e8f388ca9b1ca6c37803bd0611f67d0d41..186b58bcc52c354cf14b74e20932175a86ea6b17 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -591,26 +591,25 @@ public:
 
   void Post(const parser::OmpMapClause &x) {
     Symbol::Flag ompFlag = Symbol::Flag::OmpMapToFrom;
-    if (const auto &maptype{std::get<std::optional<parser::OmpMapType>>(x.t)}) {
-      using Type = parser::OmpMapType::Type;
-      const Type &type{std::get<Type>(maptype->t)};
-      switch (type) {
-      case Type::To:
+    if (const auto &mapType{
+            std::get<std::optional<parser::OmpMapClause::Type>>(x.t)}) {
+      switch (*mapType) {
+      case parser::OmpMapClause::Type::To:
         ompFlag = Symbol::Flag::OmpMapTo;
         break;
-      case Type::From:
+      case parser::OmpMapClause::Type::From:
         ompFlag = Symbol::Flag::OmpMapFrom;
         break;
-      case Type::Tofrom:
+      case parser::OmpMapClause::Type::Tofrom:
         ompFlag = Symbol::Flag::OmpMapToFrom;
         break;
-      case Type::Alloc:
+      case parser::OmpMapClause::Type::Alloc:
         ompFlag = Symbol::Flag::OmpMapAlloc;
         break;
-      case Type::Release:
+      case parser::OmpMapClause::Type::Release:
         ompFlag = Symbol::Flag::OmpMapRelease;
         break;
-      case Type::Delete:
+      case parser::OmpMapClause::Type::Delete:
         ompFlag = Symbol::Flag::OmpMapDelete;
         break;
       }
@@ -2419,10 +2418,16 @@ void OmpAttributeVisitor::ResolveOmpObject(
                       Symbol::Flag::OmpLastPrivate, Symbol::Flag::OmpShared,
                       Symbol::Flag::OmpLinear};
 
-                  for (Symbol::Flag ompFlag1 : dataMappingAttributeFlags) {
-                    for (Symbol::Flag ompFlag2 : dataSharingAttributeFlags) {
-                      checkExclusivelists(
-                          hostAssocSym, ompFlag1, symbol, ompFlag2);
+                  // For OMP TARGET TEAMS directive some sharing attribute
+                  // flags and mapping attribute flags can co-exist.
+                  if (!(llvm::omp::allTeamsSet.test(GetContext().directive) ||
+                          llvm::omp::allParallelSet.test(
+                              GetContext().directive))) {
+                    for (Symbol::Flag ompFlag1 : dataMappingAttributeFlags) {
+                      for (Symbol::Flag ompFlag2 : dataSharingAttributeFlags) {
+                        checkExclusivelists(
+                            hostAssocSym, ompFlag1, symbol, ompFlag2);
+                      }
                     }
                   }
                 }
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir
new file mode 100644
index 0000000000000000000000000000000000000000..91616a414676f94fce4ee20be252d6f370044f89
--- /dev/null
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir
@@ -0,0 +1,48 @@
+// Use --mlir-disable-threading so that the AA queries are serialized
+// as well as its diagnostic output.
+// RUN: fir-opt %s -pass-pipeline='builtin.module(func.func(test-fir-alias-analysis))' -split-input-file --mlir-disable-threading 2>&1 | FileCheck %s
+
+// Fortran source code:
+// program TestTmpArrayAssignment
+//   integer :: a, b, c
+//   integer :: arrayD(3)
+//
+//   arrayD = [ a, b, c ]
+// end program TestTmpArrayAssignment
+
+// CHECK-LABEL: Testing : "_QPTestTmpArrayAssignment"
+// CHECK-DAG: ArrayD#0 <-> tmp_array#0: NoAlias
+func.func @_QPTestTmpArrayAssignment() attributes {fir.bindc_name = "testtmparrayassignment"} {
+  %0 = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
+  %1:2 = hlfir.declare %0 {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c3 = arith.constant 3 : index
+  %2 = fir.alloca !fir.array<3xi32> {bindc_name = "arrayd", uniq_name = "_QFEarrayd", test.ptr = "ArrayD" }
+  %3 = fir.shape %c3 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %2(%3) {uniq_name = "_QFEarrayd"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+  %5 = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
+  %6:2 = hlfir.declare %5 {uniq_name = "_QFEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = fir.alloca i32 {bindc_name = "c", uniq_name = "_QFEc"}
+  %8:2 = hlfir.declare %7 {uniq_name = "_QFEc"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c3_0 = arith.constant 3 : index
+  %c1 = arith.constant 1 : index
+  %c1_1 = arith.constant 1 : index
+  %9 = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp.arrayctor", uniq_name = ""}
+  %10 = fir.shape %c3_0 : (index) -> !fir.shape<1>
+  %11:2 = hlfir.declare %9(%10) {uniq_name = ".tmp.arrayctor"} : (!fir.heap<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
+  %12 = fir.load %1#0 : !fir.ref<i32>
+  %13 = arith.addi %c1, %c1_1 : index
+  %14 = hlfir.designate %11#0 (%c1)  : (!fir.heap<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+  hlfir.assign %12 to %14 : i32, !fir.ref<i32>
+  %15 = fir.load %6#0 : !fir.ref<i32>
+  %16 = arith.addi %13, %c1_1 : index
+  %17 = hlfir.designate %11#0 (%13)  : (!fir.heap<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+  hlfir.assign %15 to %17 : i32, !fir.ref<i32>
+  %18 = fir.load %8#0 : !fir.ref<i32>
+  %19 = hlfir.designate %11#0 (%16)  : (!fir.heap<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+  hlfir.assign %18 to %19 : i32, !fir.ref<i32>
+  %true = arith.constant true
+  %20 = hlfir.as_expr %11#0 move %true {test.ptr = "tmp_array"}: (!fir.heap<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+  hlfir.assign %20 to %4#0 : !hlfir.expr<3xi32>, !fir.ref<!fir.array<3xi32>>
+  hlfir.destroy %20 : !hlfir.expr<3xi32>
+  return
+}
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-target-1.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-target-1.fir
new file mode 100644
index 0000000000000000000000000000000000000000..88f411847172a03d41fa63c36bcf1d3822c33379
--- /dev/null
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-target-1.fir
@@ -0,0 +1,66 @@
+// Use --mlir-disable-threading so that the AA queries are serialized
+// as well as its diagnostic output.
+// RUN: fir-opt %s -pass-pipeline='builtin.module(func.func(test-fir-alias-analysis))' -split-input-file --mlir-disable-threading 2>&1 | FileCheck %s
+
+// Fortran source code:
+//
+// program TestAllocatableArray
+// real(kind=8),  allocatable :: A(:)
+// real(kind=8),  allocatable :: B(:)
+// !$omp target
+//    A(0) = B(0)
+// !$omp end target
+// end TestAllocatableArray
+
+// CHECK-LABEL: Testing : "_QPTestAllocatableArray"
+// CHECK-DAG: targetArrayB#0 <-> targetArrayA#0: NoAlias
+func.func @_QPTestAllocatableArray() {
+  %0 = fir.address_of(@_QFEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+  %1:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "ArrayA" } : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
+  %2 = fir.address_of(@_QFEb) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+  %3:2 = hlfir.declare %2 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "ArrayB" } : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
+  %4 = fir.load %1#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %5 = fir.load %1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+  %c0_0 = arith.constant 0 : index
+  %6:3 = fir.box_dims %5, %c0_0 : (!fir.box<!fir.heap<!fir.array<?xf64>>>, index) -> (index, index, index)
+  %7:3 = fir.box_dims %4, %c0 : (!fir.box<!fir.heap<!fir.array<?xf64>>>, index) -> (index, index, index)
+  %c0_1 = arith.constant 0 : index
+  %8 = arith.subi %7#1, %c1 : index
+  %9 = omp.map.bounds lower_bound(%c0_1 : index) upper_bound(%8 : index) extent(%7#1 : index) stride(%7#2 : index) start_idx(%6#0 : index) {stride_in_bytes = true}
+  %10 = fir.box_offset %1#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>
+  %11 = omp.map.info var_ptr(%1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.array<?xf64>) var_ptr_ptr(%10 : !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>) map_clauses(implicit, tofrom) capture(ByRef) bounds(%9) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>> {name = ""}
+  %12 = omp.map.info var_ptr(%1#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.box<!fir.heap<!fir.array<?xf64>>>) map_clauses(implicit, tofrom) capture(ByRef) members(%11 : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>> {name = "a"}
+  %13 = fir.load %3#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+  %c1_2 = arith.constant 1 : index
+  %c0_3 = arith.constant 0 : index
+  %14 = fir.load %3#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+  %c0_4 = arith.constant 0 : index
+  %15:3 = fir.box_dims %14, %c0_4 : (!fir.box<!fir.heap<!fir.array<?xf64>>>, index) -> (index, index, index)
+  %16:3 = fir.box_dims %13, %c0_3 : (!fir.box<!fir.heap<!fir.array<?xf64>>>, index) -> (index, index, index)
+  %c0_5 = arith.constant 0 : index
+  %17 = arith.subi %16#1, %c1_2 : index
+  %18 = omp.map.bounds lower_bound(%c0_5 : index) upper_bound(%17 : index) extent(%16#1 : index) stride(%16#2 : index) start_idx(%15#0 : index) {stride_in_bytes = true}
+  %19 = fir.box_offset %3#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>
+  %20 = omp.map.info var_ptr(%3#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.array<?xf64>) var_ptr_ptr(%19 : !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>) map_clauses(implicit, tofrom) capture(ByRef) bounds(%18) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>> {name = ""}
+  %21 = omp.map.info var_ptr(%3#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.box<!fir.heap<!fir.array<?xf64>>>) map_clauses(implicit, tofrom) capture(ByRef) members(%20 : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>> {name = "b"}
+  omp.target map_entries(%11 -> %arg0, %12 -> %arg1, %20 -> %arg2, %21 -> %arg3 : !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.llvm_ptr<!fir.ref<!fir.array<?xf64>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) {
+    %22:2 = hlfir.declare %arg1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
+    %23:2 = hlfir.declare %arg3 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>)
+    %24 = fir.load %23#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+    %c0_6 = arith.constant 0 : index
+    %25 = hlfir.designate %24 (%c0_6) {test.ptr = "targetArrayB"} : (!fir.box<!fir.heap<!fir.array<?xf64>>>, index) -> !fir.ref<f64>
+    %26 = fir.load %25 : !fir.ref<f64>
+    %27 = fir.load %22#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>
+    %c0_7 = arith.constant 0 : index
+    %28 = hlfir.designate %27 (%c0_7) {test.ptr = "targetArrayA"} : (!fir.box<!fir.heap<!fir.array<?xf64>>>, index) -> !fir.ref<f64>
+    hlfir.assign %26 to %28 : f64, !fir.ref<f64>
+    omp.terminator
+  }
+  return
+}
+fir.global internal @_QFEa : !fir.box<!fir.heap<!fir.array<?xf64>>> {
+}
+fir.global internal @_QFEb : !fir.box<!fir.heap<!fir.array<?xf64>>> {
+}
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-target-2.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-target-2.fir
new file mode 100644
index 0000000000000000000000000000000000000000..c6b2e29a7188a99e2378a5a54c696a1bf4739f60
--- /dev/null
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-omp-target-2.fir
@@ -0,0 +1,96 @@
+// Use --mlir-disable-threading so that the AA queries are serialized
+// as well as its diagnostic output.
+// RUN: fir-opt %s -pass-pipeline='builtin.module(func.func(test-fir-alias-analysis))' -split-input-file --mlir-disable-threading 2>&1 | FileCheck %s
+
+// Fortran source code:
+//
+// subroutine TestTargetData(p, a, b)
+//   real    ::  p(10), a(10), b(10)
+//   !$omp target data map(from: p)
+//      !$omp target map(to: a )
+//            p(1) = a(1)
+//      !$omp end target
+//      !$omp target map(to: b )
+//            p(1) = b(1)
+//      !$omp end target
+//   !$omp end target data
+// end subroutine
+
+// CHECK-LABEL: Testing : "_QPTestTargetData"
+
+// CHECK-DAG: targetArrayA#0 <-> targetArrayP#0: NoAlias
+// CHECK-DAG: targetArrayA#0 <-> targetArrayB#0: NoAlias
+// CHECK-DAG: targetArrayP#0 <-> targetArrayB#0: NoAlias
+
+func.func @_QPTestTargetData(%arg0: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "p"}, %arg1: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "a"}, %arg2: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "b"}) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %c10 = arith.constant 10 : index
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2:2 = hlfir.declare %arg1(%1) dummy_scope %0 {uniq_name = "_QFtest_target_dataEa"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+  %c10_0 = arith.constant 10 : index
+  %3 = fir.shape %c10_0 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %arg2(%3) dummy_scope %0 {uniq_name = "_QFtest_target_dataEb"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+  %c10_1 = arith.constant 10 : index
+  %5 = fir.shape %c10_1 : (index) -> !fir.shape<1>
+  %6:2 = hlfir.declare %arg0(%5) dummy_scope %0 {uniq_name = "_QFtest_target_dataEp"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %7 = arith.subi %c10_1, %c1 : index
+  %8 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%7 : index) extent(%c10_1 : index) stride(%c1 : index) start_idx(%c1 : index)
+  %9 = omp.map.info var_ptr(%6#1 : !fir.ref<!fir.array<10xf32>>, !fir.array<10xf32>) map_clauses(from) capture(ByRef) bounds(%8) -> !fir.ref<!fir.array<10xf32>> {name = "p"}
+  omp.target_data map_entries(%9 : !fir.ref<!fir.array<10xf32>>) {
+    %c1_2 = arith.constant 1 : index
+    %c0_3 = arith.constant 0 : index
+    %10 = arith.subi %c10, %c1_2 : index
+    %11 = omp.map.bounds lower_bound(%c0_3 : index) upper_bound(%10 : index) extent(%c10 : index) stride(%c1_2 : index) start_idx(%c1_2 : index)
+    %12 = omp.map.info var_ptr(%2#1 : !fir.ref<!fir.array<10xf32>>, !fir.array<10xf32>) map_clauses(to) capture(ByRef) bounds(%11) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
+    %c1_4 = arith.constant 1 : index
+    %c0_5 = arith.constant 0 : index
+    %13 = arith.subi %c10_1, %c1_4 : index
+    %14 = omp.map.bounds lower_bound(%c0_5 : index) upper_bound(%13 : index) extent(%c10_1 : index) stride(%c1_4 : index) start_idx(%c1_4 : index)
+    %15 = omp.map.info var_ptr(%6#1 : !fir.ref<!fir.array<10xf32>>, !fir.array<10xf32>) map_clauses(implicit, tofrom) capture(ByRef) bounds(%14) -> !fir.ref<!fir.array<10xf32>> {name = "p"}
+    omp.target map_entries(%12 -> %arg3, %15 -> %arg4 : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+      %c10_10 = arith.constant 10 : index
+      %22 = fir.shape %c10_10 : (index) -> !fir.shape<1>
+      %23:2 = hlfir.declare %arg3(%22) {uniq_name = "_QFtest_target_dataEa"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+      %c10_11 = arith.constant 10 : index
+      %24 = fir.shape %c10_11 : (index) -> !fir.shape<1>
+      %25:2 = hlfir.declare %arg4(%24) {uniq_name = "_QFtest_target_dataEp"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+      %c1_12 = arith.constant 1 : index
+      %26 = hlfir.designate %23#0 (%c1_12) {test.ptr = "targetArrayA"}  : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+      %27 = fir.load %26 : !fir.ref<f32>
+      %c1_13 = arith.constant 1 : index
+      %28 = hlfir.designate %25#0 (%c1_13) {test.ptr = "targetArrayP"}  : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+      hlfir.assign %27 to %28 : f32, !fir.ref<f32>
+      omp.terminator
+    }
+    %c1_6 = arith.constant 1 : index
+    %c0_7 = arith.constant 0 : index
+    %16 = arith.subi %c10_0, %c1_6 : index
+    %17 = omp.map.bounds lower_bound(%c0_7 : index) upper_bound(%16 : index) extent(%c10_0 : index) stride(%c1_6 : index) start_idx(%c1_6 : index)
+    %18 = omp.map.info var_ptr(%4#1 : !fir.ref<!fir.array<10xf32>>, !fir.array<10xf32>) map_clauses(to) capture(ByRef) bounds(%17) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
+    %c1_8 = arith.constant 1 : index
+    %c0_9 = arith.constant 0 : index
+    %19 = arith.subi %c10_1, %c1_8 : index
+    %20 = omp.map.bounds lower_bound(%c0_9 : index) upper_bound(%19 : index) extent(%c10_1 : index) stride(%c1_8 : index) start_idx(%c1_8 : index)
+    %21 = omp.map.info var_ptr(%6#1 : !fir.ref<!fir.array<10xf32>>, !fir.array<10xf32>) map_clauses(implicit, tofrom) capture(ByRef) bounds(%20) -> !fir.ref<!fir.array<10xf32>> {name = "p"}
+    omp.target map_entries(%18 -> %arg3, %21 -> %arg4 : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+      %c10_10 = arith.constant 10 : index
+      %22 = fir.shape %c10_10 : (index) -> !fir.shape<1>
+      %23:2 = hlfir.declare %arg3(%22) {uniq_name = "_QFtest_target_dataEb"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+      %c10_11 = arith.constant 10 : index
+      %24 = fir.shape %c10_11 : (index) -> !fir.shape<1>
+      %25:2 = hlfir.declare %arg4(%24) {uniq_name = "_QFtest_target_dataEp"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+      %c1_12 = arith.constant 1 : index
+      %26 = hlfir.designate %23#0 (%c1_12) {test.ptr = "targetArrayB"}  : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+      %27 = fir.load %26 : !fir.ref<f32>
+      %c1_13 = arith.constant 1 : index
+      %28 = hlfir.designate %25#0 (%c1_13) {test.ptr = "targetArrayP"}  : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+      hlfir.assign %27 to %28 : f32, !fir.ref<f32>
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
diff --git a/flang/test/Driver/frecord-command-line.f90 b/flang/test/Driver/frecord-command-line.f90
new file mode 100644
index 0000000000000000000000000000000000000000..bc4ce79e4a51c3755f210b610391e4b09713eefb
--- /dev/null
+++ b/flang/test/Driver/frecord-command-line.f90
@@ -0,0 +1,16 @@
+! This only checks that the command line is correctly passed on to the
+! -record-command-line option FC1 option and that the latter does not complain
+! about anything. The correct lowering to a module attribute and beyond will
+! be checked in other tests.
+!
+! RUN: %flang -### -target x86_64-unknown-linux -frecord-command-line %s 2>&1 | FileCheck --check-prefix=CHECK-RECORD %s
+! RUN: %flang -### -target x86_64-unknown-macosx -frecord-command-line %s 2>&1 | FileCheck --check-prefix=CHECK-RECORD %s
+! RUN: not %flang -### -target x86_64-unknown-windows -frecord-command-line %s 2>&1 | FileCheck --check-prefix=CHECK-RECORD-ERROR %s
+
+! RUN: %flang -### -target x86_64-unknown-linux -fno-record-command-line %s 2>&1 | FileCheck --check-prefix=CHECK-NO-RECORD %s
+! RUN: %flang -### -target x86_64-unknown-macosx -fno-record-command-line %s 2>&1 | FileCheck --check-prefix=CHECK-NO-RECORD %s
+! RUN: %flang -### -target x86_64-unknown-windows -fno-record-command-line %s 2>&1 | FileCheck --check-prefix=CHECK-NO-RECORD %s
+
+! CHECK-RECORD: "-record-command-line"
+! CHECK-NO-RECORD-NOT: "-record-command-line"
+! CHECK-RECORD-ERROR: error: unsupported option '-frecord-command-line' for target
diff --git a/flang/test/Fir/abstract-results-bindc.fir b/flang/test/Fir/abstract-results-bindc.fir
new file mode 100644
index 0000000000000000000000000000000000000000..b2efffea31fb153723cfdd7a43c293a2baaa81de
--- /dev/null
+++ b/flang/test/Fir/abstract-results-bindc.fir
@@ -0,0 +1,60 @@
+// Test that bind_c derived type results are not moved to a hidden argument
+// by the abstract-result pass. They will be dealt with according to the C
+// struct returning ABI for the target in the target-rewrite pass.
+// RUN: fir-opt %s --abstract-result | FileCheck %s
+
+!t = !fir.type<t{i:f32, j: i32, k: f32}>
+!cptr = !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+
+func.func private @foo() -> !t attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+func.func @test_call(%x: !fir.ref<!t>) {
+  %0 = fir.call @foo() proc_attrs<bind_c> : () -> !t
+  fir.save_result %0 to %x : !t, !fir.ref<!t>
+  return
+}
+
+func.func @test_addr_of() -> (() -> !t) {
+  %0 = fir.address_of(@foo) : () -> !t
+  return %0 : () -> !t
+}
+
+func.func @test_dispatch(%x: !fir.ref<!t>, %y : !fir.class<!fir.type<somet>>) {
+  %0 = fir.dispatch "bar"(%y : !fir.class<!fir.type<somet>>) (%y : !fir.class<!fir.type<somet>>) -> !t proc_attrs<bind_c> {pass_arg_pos = 0 : i32}
+  fir.save_result %0 to %x : !t, !fir.ref<!t>
+  return
+}
+
+
+func.func private @return_cptr() -> !cptr attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+func.func @test_return_cptr(%x: !fir.ref<!cptr>) {
+  %0 = fir.call @return_cptr() proc_attrs<bind_c> : () -> !cptr
+  fir.save_result %0 to %x : !cptr, !fir.ref<!cptr>
+  return
+}
+
+
+// CHECK-LABEL:   func.func @test_call(
+// CHECK-SAME:                         %[[VAL_0:.*]]: !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>) {
+// CHECK:           %[[VAL_1:.*]] = fir.call @foo() proc_attrs<bind_c> : () -> !fir.type<t{i:f32,j:i32,k:f32}>
+// CHECK:           fir.store %[[VAL_1]] to %[[VAL_0]] : !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+// CHECK-LABEL:   func.func @test_addr_of() -> (() -> !fir.type<t{i:f32,j:i32,k:f32}>) {
+// CHECK:           %[[VAL_0:.*]] = fir.address_of(@foo) : () -> !fir.type<t{i:f32,j:i32,k:f32}>
+// CHECK:           return %[[VAL_0]] : () -> !fir.type<t{i:f32,j:i32,k:f32}>
+// CHECK:         }
+// CHECK-LABEL:   func.func @test_dispatch(
+// CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>,
+// CHECK-SAME:                             %[[VAL_1:.*]]: !fir.class<!fir.type<somet>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.dispatch "bar"(%[[VAL_1]] : !fir.class<!fir.type<somet>>) (%[[VAL_1]] : !fir.class<!fir.type<somet>>) -> !fir.type<t{i:f32,j:i32,k:f32}> proc_attrs <bind_c> {pass_arg_pos = 0 : i32}
+// CHECK:           fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+// CHECK-LABEL:   func.func @test_return_cptr(
+// CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) {
+// CHECK:           %[[VAL_1:.*]] = fir.call @return_cptr() : () -> !fir.ref<none>
+// CHECK:           %[[VAL_2:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+// CHECK:           %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_2]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<none>) -> i64
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_3]] : !fir.ref<i64>
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index aa2f416063fced90dd584298c7f9ad9568f36de5..335877e7c9a87252b41bf4db941f71d370c25e93 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -17,7 +17,6 @@ func.func @_QPsb1(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!
         fir.store %3 to %6 : !fir.ref<i32>
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -43,7 +42,6 @@ func.func @_QPsb1(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!
 // CHECK:   llvm.store %[[I1]], %[[ARR_I_REF]] : i32, !llvm.ptr
 // CHECK: omp.yield
 // CHECK: }
-// CHECK: omp.terminator
 // CHECK: }
 // CHECK: omp.terminator
 // CHECK: }
@@ -93,7 +91,6 @@ func.func @_QPsb(%arr: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "arr"}) {
         fir.store %indx to %3 : !fir.ref<i32>
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -110,7 +107,6 @@ func.func @_QPsb(%arr: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "arr"}) {
 // CHECK-NEXT:   omp.loop_nest (%[[INDX:.*]]) : i32 = (%[[C1]]) to (%[[C50]]) inclusive step (%[[C1]]) {
 // CHECK:          llvm.store %[[INDX]], %{{.*}} : i32, !llvm.ptr
 // CHECK:          omp.yield
-// CHECK:        omp.terminator
 // CHECK:      omp.terminator
 // CHECK:    llvm.return
 
@@ -200,7 +196,6 @@ func.func @_QPsimd1(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref
         fir.store %3 to %6 : !fir.ref<i32>
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -226,7 +221,6 @@ func.func @_QPsimd1(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref
 // CHECK:   llvm.store %[[I1]], %[[ARR_I_REF]] : i32, !llvm.ptr
 // CHECK: omp.yield
 // CHECK: }
-// CHECK: omp.terminator
 // CHECK: }
 // CHECK: omp.terminator
 // CHECK: }
@@ -519,7 +513,6 @@ func.func @_QPsimd_with_nested_loop() {
       fir.store %7 to %3 : !fir.ref<i32>
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -540,7 +533,6 @@ func.func @_QPsimd_with_nested_loop() {
 // CHECK:             ^bb3:
 // CHECK:               omp.yield
 // CHECK:             }
-// CHECK:             omp.terminator
 // CHECK:           }
 // CHECK:           llvm.return
 // CHECK:         }
@@ -736,7 +728,6 @@ func.func @_QPsb() {
 // CHECK:          %[[RES_EXT:.*]] = llvm.zext %[[RES]] : i1 to i32
 // CHECK:          llvm.store %[[RES_EXT]], %[[PRV]] : i32, !llvm.ptr
 // CHECK:          omp.yield
-// CHECK:        omp.terminator
 // CHECK:      omp.terminator
 // CHECK:    llvm.return
 
@@ -781,7 +772,6 @@ func.func @_QPsimple_reduction(%arg0: !fir.ref<!fir.array<100x!fir.logical<4>>>
         fir.store %11 to %prv : !fir.ref<!fir.logical<4>>
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
diff --git a/flang/test/Fir/struct-return-x86-64.fir b/flang/test/Fir/struct-return-x86-64.fir
new file mode 100644
index 0000000000000000000000000000000000000000..f4c2add69ff7e9015b9b00256dee4034aefeae34
--- /dev/null
+++ b/flang/test/Fir/struct-return-x86-64.fir
@@ -0,0 +1,120 @@
+// Test X86-64 ABI rewrite of struct returned by value (BIND(C), VALUE derived types).
+// REQUIRES: x86-registered-target
+// RUN: fir-opt --target-rewrite %s | FileCheck %s
+
+!fits_in_reg = !fir.type<t1{i:f32,j:i32,k:f32}>
+!too_big = !fir.type<t2{i:!fir.array<5xf32>}>
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+
+  func.func private @test_inreg() -> !fits_in_reg
+  func.func @test_call_inreg(%arg0: !fir.ref<!fits_in_reg>) {
+    %0 = fir.call @test_inreg() : () -> !fits_in_reg
+    fir.store %0 to %arg0 : !fir.ref<!fits_in_reg>
+    return
+  }
+  func.func @test_addr_of_inreg() -> (() -> ()) {
+    %0 = fir.address_of(@test_inreg) : () -> !fits_in_reg
+    %1 = fir.convert %0 : (() -> !fits_in_reg) -> (() -> ())
+    return %1 : () -> ()
+  }
+  func.func @test_dispatch_inreg(%arg0: !fir.ref<!fits_in_reg>, %arg1: !fir.class<!fir.type<somet>>) {
+    %0 = fir.dispatch "bar"(%arg1 : !fir.class<!fir.type<somet>>) (%arg1 : !fir.class<!fir.type<somet>>) -> !fits_in_reg {pass_arg_pos = 0 : i32}
+    fir.store %0 to %arg0 : !fir.ref<!fits_in_reg>
+    return
+  }
+
+  func.func private @test_sret() -> !too_big
+  func.func @test_call_sret(%arg0: !fir.ref<!too_big>) {
+    %0 = fir.call @test_sret() : () -> !too_big
+    fir.store %0 to %arg0 : !fir.ref<!too_big>
+    return
+  }
+  func.func @test_addr_of_sret() -> (() -> ()) {
+    %0 = fir.address_of(@test_sret) : () -> !too_big
+    %1 = fir.convert %0 : (() -> !too_big) -> (() -> ())
+    return %1 : () -> ()
+  }
+  func.func @test_dispatch_sret(%arg0: !fir.ref<!too_big>, %arg1: !fir.class<!fir.type<somet>>) {
+    %0 = fir.dispatch "bar"(%arg1 : !fir.class<!fir.type<somet>>) (%arg1 : !fir.class<!fir.type<somet>>) -> !too_big {pass_arg_pos = 0 : i32}
+    fir.store %0 to %arg0 : !fir.ref<!too_big>
+    return
+  }
+  func.func private @test_fp_80() -> !fir.type<t3{i:f80}>
+  func.func private @test_complex_80() -> !fir.type<t4{i:complex<f80>}>
+  func.func private @test_two_fp_80() -> !fir.type<t5{i:f80,j:f80}>
+  func.func private @test_fp128() -> !fir.type<t6{i:f128}>
+}
+
+// CHECK-LABEL:   func.func private @test_inreg() -> tuple<i64, f32>
+
+// CHECK-LABEL:   func.func @test_call_inreg(
+// CHECK-SAME:                               %[[VAL_0:.*]]: !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>) {
+// CHECK:           %[[VAL_1:.*]] = fir.call @test_inreg() : () -> tuple<i64, f32>
+// CHECK:           %[[VAL_2:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_3:.*]] = fir.alloca tuple<i64, f32>
+// CHECK:           fir.store %[[VAL_1]] to %[[VAL_3]] : !fir.ref<tuple<i64, f32>>
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<tuple<i64, f32>>) -> !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_2]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_0]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_addr_of_inreg() -> (() -> ()) {
+// CHECK:           %[[VAL_0:.*]] = fir.address_of(@test_inreg) : () -> tuple<i64, f32>
+// CHECK:           %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (() -> tuple<i64, f32>) -> (() -> ())
+// CHECK:           return %[[VAL_1]] : () -> ()
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_dispatch_inreg(
+// CHECK-SAME:                                   %[[VAL_0:.*]]: !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>,
+// CHECK-SAME:                                   %[[VAL_1:.*]]: !fir.class<!fir.type<somet>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.dispatch "bar"(%[[VAL_1]] : !fir.class<!fir.type<somet>>) (%[[VAL_1]] : !fir.class<!fir.type<somet>>) -> tuple<i64, f32> {pass_arg_pos = 0 : i32}
+// CHECK:           %[[VAL_3:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_4:.*]] = fir.alloca tuple<i64, f32>
+// CHECK:           fir.store %[[VAL_2]] to %[[VAL_4]] : !fir.ref<tuple<i64, f32>>
+// CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<tuple<i64, f32>>) -> !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_3]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_6]] to %[[VAL_0]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+// CHECK:         func.func private @test_sret(!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t2{i:!fir.array<5xf32>}>})
+
+// CHECK-LABEL:   func.func @test_call_sret(
+// CHECK-SAME:                              %[[VAL_0:.*]]: !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) {
+// CHECK:           %[[VAL_1:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.type<t2{i:!fir.array<5xf32>}>
+// CHECK:           fir.call @test_sret(%[[VAL_2]]) : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> ()
+// CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_1]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_0]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           return
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_addr_of_sret() -> (() -> ()) {
+// CHECK:           %[[VAL_0:.*]] = fir.address_of(@test_sret) : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> ()
+// CHECK:           %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : ((!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> ()) -> (() -> ())
+// CHECK:           return %[[VAL_1]] : () -> ()
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_dispatch_sret(
+// CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>,
+// CHECK-SAME:                                  %[[VAL_1:.*]]: !fir.class<!fir.type<somet>>) {
+// CHECK:           %[[VAL_2:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<t2{i:!fir.array<5xf32>}>
+// CHECK:           fir.dispatch "bar"(%[[VAL_1]] : !fir.class<!fir.type<somet>>) (%[[VAL_3]], %[[VAL_1]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>, !fir.class<!fir.type<somet>>) {pass_arg_pos = 1 : i32}
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_2]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_0]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           return
+// CHECK:         }
+
+
+// CHECK: func.func private @test_fp_80() -> f80
+// CHECK: func.func private @test_complex_80(!fir.ref<!fir.type<t4{i:complex<f80>}>> {llvm.align = 16 : i32, llvm.sret = !fir.type<t4{i:complex<f80>}>})
+// CHECK: func.func private @test_two_fp_80(!fir.ref<!fir.type<t5{i:f80,j:f80}>> {llvm.align = 16 : i32, llvm.sret = !fir.type<t5{i:f80,j:f80}>})
+// CHECK: func.func private @test_fp128() -> f128
diff --git a/flang/test/Lower/OpenMP/Todo/map-modifiers-close.f90 b/flang/test/Lower/OpenMP/Todo/map-modifiers-close.f90
new file mode 100644
index 0000000000000000000000000000000000000000..c30f45d83864be21a69d907ae3c602f449f4913d
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/map-modifiers-close.f90
@@ -0,0 +1,10 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+!CHECK: Map type modifiers (other than 'ALWAYS') are not implemented yet
+subroutine f00()
+  integer :: x
+  !$omp target map(close: x)
+  x = x + 1
+  !$omp end target
+end
diff --git a/flang/test/Lower/OpenMP/Todo/map-modifiers-ompxhold.f90 b/flang/test/Lower/OpenMP/Todo/map-modifiers-ompxhold.f90
new file mode 100644
index 0000000000000000000000000000000000000000..0b5f2f5ca5e243e7107481895af912c46efeb3fc
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/map-modifiers-ompxhold.f90
@@ -0,0 +1,11 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+!CHECK: Map type modifiers (other than 'ALWAYS') are not implemented yet
+subroutine f01()
+  integer :: x
+  !$omp target map(ompx_hold: x)
+  x = x + 1
+  !$omp end target
+end
+
diff --git a/flang/test/Lower/OpenMP/Todo/map-modifiers-present.f90 b/flang/test/Lower/OpenMP/Todo/map-modifiers-present.f90
new file mode 100644
index 0000000000000000000000000000000000000000..836b3a0a84bec107da4d49db1cda820cf3d0e82e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/map-modifiers-present.f90
@@ -0,0 +1,11 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+!CHECK: Map type modifiers (other than 'ALWAYS') are not implemented yet
+subroutine f02()
+  integer :: x
+  !$omp target map(present: x)
+  x = x + 1
+  !$omp end target
+end
+
diff --git a/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90 b/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
index 42ebd37d1c431353d9efd7cd773c464cc94a5bb7..77a1304f39a48818cf729a89ac933014b0192850 100644
--- a/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
+++ b/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
@@ -24,7 +24,6 @@
 ! CHECK: fir.store %[[result]] to %{{.*}} : !fir.ref<i32>
 ! CHECK: omp.yield
 ! CHECK: }
-! CHECK: omp.terminator
 ! CHECK: }
 ! CHECK: omp.terminator
 ! CHECK: }
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-inscan.f90 b/flang/test/Lower/OpenMP/Todo/reduction-inscan.f90
new file mode 100644
index 0000000000000000000000000000000000000000..c5f196fe09693a4ab8b1f4796bf6fb86e553d95c
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/reduction-inscan.f90
@@ -0,0 +1,14 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Reduction modifiers are not supported
+subroutine reduction_inscan()
+  integer :: i,j
+  i = 0
+
+  !$omp do reduction(inscan, +:i)
+  do j=1,10
+     i = i + 1
+  end do
+  !$omp end do
+end subroutine reduction_inscan
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-task.f90 b/flang/test/Lower/OpenMP/Todo/reduction-task.f90
new file mode 100644
index 0000000000000000000000000000000000000000..6707f65e1a4cc3112d648f4c8dce02c3ea054463
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/reduction-task.f90
@@ -0,0 +1,12 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Reduction modifiers are not supported
+subroutine reduction_task()
+  integer :: i
+  i = 0
+
+  !$omp parallel reduction(task, +:i)
+  i = i + 1
+  !$omp end parallel 
+end subroutine reduction_task
diff --git a/flang/test/Lower/OpenMP/Todo/target-inreduction.f90 b/flang/test/Lower/OpenMP/Todo/target-inreduction.f90
new file mode 100644
index 0000000000000000000000000000000000000000..e5a9cffac5a1170cfd63fd61c3b1c75d6d5548ad
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/target-inreduction.f90
@@ -0,0 +1,15 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+
+!===============================================================================
+! `mergeable` clause
+!===============================================================================
+
+! CHECK: not yet implemented: Unhandled clause IN_REDUCTION in TARGET construct
+subroutine omp_target_inreduction()
+  integer i
+  i = 0
+  !$omp target in_reduction(+:i)
+  i = i + 1
+  !$omp end target
+end subroutine omp_target_inreduction
diff --git a/flang/test/Lower/OpenMP/Todo/task-inreduction.f90 b/flang/test/Lower/OpenMP/Todo/task-inreduction.f90
new file mode 100644
index 0000000000000000000000000000000000000000..aeed680a6dba7cb20084645cee97e1bd6363f31c
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/task-inreduction.f90
@@ -0,0 +1,15 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+!===============================================================================
+! `mergeable` clause
+!===============================================================================
+
+! CHECK: not yet implemented: Unhandled clause IN_REDUCTION in TASK construct
+subroutine omp_task_in_reduction()
+  integer i
+  i = 0
+  !$omp task in_reduction(+:i)
+  i = i + 1
+  !$omp end task
+end subroutine omp_task_in_reduction
diff --git a/flang/test/Lower/OpenMP/Todo/task_mergeable.f90 b/flang/test/Lower/OpenMP/Todo/task_mergeable.f90
index 13145d92ccf902e78582acfc0528dd96f9a70248..ddc27487abfe9cf3ea5798ba7657c12528f38c77 100644
--- a/flang/test/Lower/OpenMP/Todo/task_mergeable.f90
+++ b/flang/test/Lower/OpenMP/Todo/task_mergeable.f90
@@ -5,7 +5,7 @@
 ! `mergeable` clause
 !===============================================================================
 
-! CHECK: not yet implemented: OpenMP Block construct clause
+! CHECK: not yet implemented: Unhandled clause MERGEABLE in TASK construct
 subroutine omp_task_mergeable()
   !$omp task mergeable
   call foo()
diff --git a/flang/test/Lower/OpenMP/Todo/taskgroup-task-reduction.f90 b/flang/test/Lower/OpenMP/Todo/taskgroup-task-reduction.f90
new file mode 100644
index 0000000000000000000000000000000000000000..1cb471d784d766e73cc2a0c3a0008923be49c2ae
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/taskgroup-task-reduction.f90
@@ -0,0 +1,10 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Unhandled clause TASK_REDUCTION in TASKGROUP construct
+subroutine omp_taskgroup_task_reduction
+  integer :: res
+  !$omp taskgroup task_reduction(+:res)
+  res = res + 1
+  !$omp end taskgroup
+end subroutine omp_taskgroup_task_reduction
diff --git a/flang/test/Lower/OpenMP/Todo/taskloop.f90 b/flang/test/Lower/OpenMP/Todo/taskloop.f90
new file mode 100644
index 0000000000000000000000000000000000000000..aca050584cbbe35c0d598d1c37fa3a5ebadb57b4
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/taskloop.f90
@@ -0,0 +1,13 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Taskloop construct
+subroutine omp_taskloop
+  integer :: res, i
+  !$omp taskloop
+  do i = 1, 10
+     res = res + 1
+  end do
+  !$omp end taskloop
+end subroutine omp_taskloop
+
diff --git a/flang/test/Lower/OpenMP/Todo/taskwait-depend.f90 b/flang/test/Lower/OpenMP/Todo/taskwait-depend.f90
new file mode 100644
index 0000000000000000000000000000000000000000..d1f953be8802fae4006b97989d82e6704c11857d
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/taskwait-depend.f90
@@ -0,0 +1,10 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Unhandled clause DEPEND in TASKWAIT construct
+subroutine omp_tw_depend
+  integer :: res
+  !$omp taskwait depend(out: res)
+  res = res + 1
+end subroutine omp_tw_depend
+
diff --git a/flang/test/Lower/OpenMP/Todo/taskwait-nowait.f90 b/flang/test/Lower/OpenMP/Todo/taskwait-nowait.f90
new file mode 100644
index 0000000000000000000000000000000000000000..21e8609b08ba37d902630444683abe4852531730
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/taskwait-nowait.f90
@@ -0,0 +1,8 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s -fopenmp-version=51 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s -fopenmp-version=51 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Unhandled clause NOWAIT in TASKWAIT construct
+subroutine omp_tw_nowait
+  !$omp taskwait nowait
+end subroutine omp_tw_nowait
+
diff --git a/flang/test/Lower/OpenMP/copyin.f90 b/flang/test/Lower/OpenMP/copyin.f90
index b1c6b9420f4c4671c779ba8d30bc10dd44a66f70..f3d147c10668f49de622a822f9d2dd639db6f273 100644
--- a/flang/test/Lower/OpenMP/copyin.f90
+++ b/flang/test/Lower/OpenMP/copyin.f90
@@ -166,7 +166,6 @@ end
 ! CHECK:                 fir.call @_QPsub4(%[[VAL_9]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -338,7 +337,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_40]] to %[[VAL_31]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index 6cdff407a9790173a906d57c04284d74ed3bbd96..654c13ada9e39f1906d0ec1cf05cf2091aaf675f 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -350,7 +350,6 @@ subroutine skipped_default_clause_checks()
 !CHECK-NEXT: omp.loop_nest (%[[ARG:.*]]) {{.*}} {
 !CHECK: omp.yield
 !CHECK: }
-!CHECK: omp.terminator
 !CHECK: }
 !CHECK: omp.terminator
 !CHECK: }
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index 1b1b47c40b545c0095e268aaf66026e6d6945ac7..c004813a911f737c355a215842c09ee44ac632ef 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -390,7 +390,6 @@ subroutine skipped_default_clause_checks()
 !CHECK-NEXT: omp.loop_nest (%[[ARG:.*]]) {{.*}} {
 !CHECK: omp.yield
 !CHECK: }
-!CHECK: omp.terminator
 !CHECK: }
 !CHECK: omp.terminator
 !CHECK: }
diff --git a/flang/test/Lower/OpenMP/hlfir-wsloop.f90 b/flang/test/Lower/OpenMP/hlfir-wsloop.f90
index fc5b5999f2708d25435339dd72b2f06728d4ce81..f7b0ba681efebf3c831e329025465922daa32839 100644
--- a/flang/test/Lower/OpenMP/hlfir-wsloop.f90
+++ b/flang/test/Lower/OpenMP/hlfir-wsloop.f90
@@ -22,7 +22,6 @@ subroutine simple_loop
     print*, i
   end do
   ! CHECK:             omp.yield
-  ! CHECK:           omp.terminator
   !$OMP END DO
   ! CHECK:         omp.terminator
   !$OMP END PARALLEL
diff --git a/flang/test/Lower/OpenMP/lastprivate-commonblock.f90 b/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
index dcba34b2da8ef55315e4b3ce0674cb2ac923edf9..faa3d3e053f345e02fc7eecf028d1337c0baa8cf 100644
--- a/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
@@ -32,7 +32,6 @@
 !CHECK:          }
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 subroutine lastprivate_common
   common /c/ x, y
diff --git a/flang/test/Lower/OpenMP/lastprivate-iv.f90 b/flang/test/Lower/OpenMP/lastprivate-iv.f90
index 609192471eae1ddbc5abd35f3c4a1634bb4f6b98..63a81e818bc8ba2aff55d459b45cd2055755fc2b 100644
--- a/flang/test/Lower/OpenMP/lastprivate-iv.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-iv.f90
@@ -28,7 +28,6 @@
 !CHECK:          }
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 subroutine lastprivate_iv_inc()
   integer :: i
@@ -66,7 +65,6 @@ end subroutine
 !CHECK:          }
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 subroutine lastprivate_iv_dec()
   integer :: i
diff --git a/flang/test/Lower/OpenMP/location.f90 b/flang/test/Lower/OpenMP/location.f90
index 5d340937a81ce01fa710617dd684b6416489ae24..2dab22a1c1f90d5f3a3739db2dec8cbee4041a36 100644
--- a/flang/test/Lower/OpenMP/location.f90
+++ b/flang/test/Lower/OpenMP/location.f90
@@ -35,7 +35,6 @@ subroutine sub_loop()
     print *, i
 !CHECK:   omp.yield loc(#[[LOOP_LOC:.*]])
 !CHECK: } loc(#[[LOOP_LOC]])
-!CHECK:   omp.terminator loc(#[[LOOP_LOC]])
 !CHECK: } loc(#[[LOOP_LOC]])
   end do
   !$omp end do
@@ -64,8 +63,8 @@ end subroutine
 !CHECK: #[[PAR_LOC]] = loc("{{.*}}location.f90":9:9)
 !CHECK: #[[TAR_LOC]] = loc("{{.*}}location.f90":21:9)
 !CHECK: #[[LOOP_LOC]] = loc("{{.*}}location.f90":33:9)
-!CHECK: #[[BAR_LOC]] = loc("{{.*}}location.f90":47:9)
-!CHECK: #[[TW_LOC]] = loc("{{.*}}location.f90":49:9)
-!CHECK: #[[TY_LOC]] = loc("{{.*}}location.f90":51:9)
-!CHECK: #[[IF_LOC]] = loc("{{.*}}location.f90":58:14)
-!CHECK: #[[TASK_LOC]] = loc("{{.*}}location.f90":58:9)
+!CHECK: #[[BAR_LOC]] = loc("{{.*}}location.f90":46:9)
+!CHECK: #[[TW_LOC]] = loc("{{.*}}location.f90":48:9)
+!CHECK: #[[TY_LOC]] = loc("{{.*}}location.f90":50:9)
+!CHECK: #[[IF_LOC]] = loc("{{.*}}location.f90":57:14)
+!CHECK: #[[TASK_LOC]] = loc("{{.*}}location.f90":57:9)
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index bc5baf4e1cf60404e0a613c1b98a875a98279b91..86309a24f91a03601c8827f4fabb6766c87c5903 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -41,7 +41,6 @@
 !CHECK: }
 !CHECK: omp.yield
 !CHECK: }
-!CHECK: omp.terminator
 !CHECK: }
 
 subroutine lastprivate_character(arg1)
@@ -80,7 +79,6 @@ end subroutine
 !CHECK: }
 !CHECK: omp.yield
 !CHECK: }
-!CHECK: omp.terminator
 !CHECK: }
 
 subroutine lastprivate_int(arg1)
@@ -124,7 +122,6 @@ end subroutine
 !CHECK: }
 !CHECK: omp.yield
 !CHECK: }
-!CHECK: omp.terminator
 !CHECK: }
 
 subroutine mult_lastprivate_int(arg1, arg2)
@@ -169,7 +166,6 @@ end subroutine
 !CHECK: }
 !CHECK: omp.yield
 !CHECK: }
-!CHECK: omp.terminator
 !CHECK: }
 
 subroutine mult_lastprivate_int2(arg1, arg2)
@@ -217,7 +213,6 @@ end subroutine
 !CHECK-NEXT: }
 !CHECK-NEXT: omp.yield
 !CHECK-NEXT: }
-!CHECK-NEXT: omp.terminator
 !CHECK-NEXT: }
 
 subroutine firstpriv_lastpriv_int(arg1, arg2)
@@ -265,7 +260,6 @@ end subroutine
 !CHECK-NEXT: }
 !CHECK-NEXT: omp.yield
 !CHECK-NEXT: }
-!CHECK-NEXT: omp.terminator
 !CHECK-NEXT: }
 
 subroutine firstpriv_lastpriv_int2(arg1)
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
index 073146b8b2e87aceb94f80243fb08e2994323377..5e76e8ff1663bf74adc2c735ccdad4ed9c055343 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -91,7 +91,6 @@
 ! CHECK:               fir.store %[[VAL_12]]#1 to %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
 ! CHECK:               omp.yield
 ! CHECK:             }
-! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.terminator
 ! CHECK:         }
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause.f90 b/flang/test/Lower/OpenMP/parallel-private-clause.f90
index edb8f9193b827c1e619f058e582aa916a531bbe9..8dc843fc9d96aef34eff083efa1cf3ff375e489a 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause.f90
@@ -272,7 +272,6 @@ subroutine simple_loop_1
     print*, i
   end do
   ! FIRDialect:      omp.yield
-  ! FIRDialect:      omp.terminator
   !$OMP END DO
   ! FIRDialect:  omp.terminator
   !$OMP END PARALLEL
@@ -306,7 +305,6 @@ subroutine simple_loop_2
     print*, i
   end do
   ! FIRDialect:     omp.yield
-  ! FIRDialect:     omp.terminator
   ! FIRDialect:     {{%.*}} = fir.load %[[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:     fir.if {{%.*}} {
   ! FIRDialect:     [[LD:%.*]] = fir.load %[[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
@@ -346,7 +344,6 @@ subroutine simple_loop_3
     print*, i
   end do
   ! FIRDialect:      omp.yield
-  ! FIRDialect:      omp.terminator
   ! FIRDialect:      {{%.*}} = fir.load [[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:      fir.if {{%.*}} {
   ! FIRDialect:      [[LD:%.*]] = fir.load [[R_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index 7a2db3299784c714772c753da0166f0e46900c9e..dabd495d733b558fdb667d84d3ca551b937a9856 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -115,7 +115,6 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_29]] : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
index 1273b250117da4848c32a5d6a669857008bfee5f..01d8dc33f40e6f2375f927157d4d406aae82dfa0 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-pointer-array.f90
@@ -127,7 +127,6 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_36]] to %[[VAL_39]] : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/parallel-reduction3.f90 b/flang/test/Lower/OpenMP/parallel-reduction3.f90
index 591f41cb946602ef80983b0512c90974aa50b76b..70b4f0f12820be71445fe49da46a12a0965a71a6 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction3.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction3.f90
@@ -102,7 +102,6 @@
 ! CHECK:                 hlfir.destroy %[[VAL_28]] : !hlfir.expr<?xi32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
index de3f42be10482921b008e2a05e94563aade3844c..4e7c2c15df74321aa407712f857af9c658de6cf6 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
@@ -29,7 +29,6 @@ subroutine omp_do_firstprivate(a)
   ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
   ! CHECK-NEXT: omp.yield
   ! CHECK-NEXT: }
-  ! CHECK-NEXT: omp.terminator
   ! CHECK-NEXT: }
     do i=1, a
       call foo(i, a)
@@ -70,7 +69,6 @@ subroutine omp_do_firstprivate2(a, n)
   ! CHECK: fir.store %[[ARG2]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
   ! CHECK: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
   ! CHECK: omp.yield
-  ! CHECK: omp.terminator
     do i= a, n
       call foo(i, a)
     end do
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
index 254aeff21d06edfd2f913e5cb3f2880c94e56e2b..dbde5291c01c8b0223626aad6f8de56cbc261f4d 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
@@ -39,7 +39,6 @@ subroutine omp_do_lastprivate(a)
 
   ! CHECK-NEXT: omp.yield
   ! CHECK-NEXT: }
-  ! CHECK-NEXT: omp.terminator
   ! CHECK-NEXT: }
     do i=1, a
       call foo(i, a)
@@ -90,7 +89,6 @@ subroutine omp_do_lastprivate2(a, n)
   ! CHECK: }
 
   ! CHECK: omp.yield
-  ! CHECK: omp.terminator
     do i= a, n
       call foo(i, a)
     end do
@@ -148,7 +146,6 @@ subroutine omp_do_lastprivate_collapse2(a)
 
   ! CHECK-NEXT: omp.yield
   ! CHECK-NEXT: }
-  ! CHECK-NEXT: omp.terminator
   ! CHECK-NEXT: }
     do i=1, a
       do j=1, a
@@ -224,7 +221,6 @@ subroutine omp_do_lastprivate_collapse3(a)
 
   ! CHECK-NEXT: omp.yield
   ! CHECK-NEXT: }
-  ! CHECK-NEXT: omp.terminator
   ! CHECK-NEXT: }
     do i=1, a
       do j=1, a
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop.f90 b/flang/test/Lower/OpenMP/parallel-wsloop.f90
index 4a9c66857ffaaf12d7e4d3a3807e09cc9720c00b..cba5209f85989b5803f69e253bca297f309487c8 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop.f90
@@ -21,7 +21,6 @@ subroutine simple_parallel_do
   end do
   ! CHECK:      omp.yield
   ! CHECK:      omp.terminator
-  ! CHECK:      omp.terminator
   !$OMP END PARALLEL DO
 end subroutine
 
@@ -51,7 +50,6 @@ subroutine parallel_do_with_parallel_clauses(cond, nt)
   end do
   ! CHECK:      omp.yield
   ! CHECK:      omp.terminator
-  ! CHECK:      omp.terminator
   !$OMP END PARALLEL DO
 end subroutine
 
@@ -77,7 +75,6 @@ subroutine parallel_do_with_clauses(nt)
   end do
   ! CHECK:      omp.yield
   ! CHECK:      omp.terminator
-  ! CHECK:      omp.terminator
   !$OMP END PARALLEL DO
 end subroutine
 
@@ -120,7 +117,6 @@ subroutine parallel_do_with_privatisation_clauses(cond,nt)
   end do
   ! CHECK:      omp.yield
   ! CHECK:    omp.terminator
-  ! CHECK:    omp.terminator
   !$OMP END PARALLEL DO
 end subroutine
 
@@ -165,7 +161,6 @@ end subroutine parallel_private_do
 ! CHECK:                 fir.call @_QPfoo(%[[I_PRIV_DECL]]#1, %[[COND_DECL]]#1, %[[NT_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<!fir.logical<4>>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -212,7 +207,6 @@ end subroutine omp_parallel_multiple_firstprivate_do
 ! CHECK:                 fir.call @_QPbar(%[[I_PRIV_DECL]]#1, %[[A_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -264,7 +258,6 @@ end subroutine parallel_do_private
 ! CHECK:                 fir.call @_QPfoo(%[[I_PRIV_DECL]]#1, %[[COND_PRIV_DECL]]#1, %[[NT_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<!fir.logical<4>>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -317,7 +310,6 @@ end subroutine omp_parallel_do_multiple_firstprivate
 ! CHECK:                 fir.call @_QPbar(%[[I_PRIV_DECL]]#1, %[[A_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/private-derived-type.f90 b/flang/test/Lower/OpenMP/private-derived-type.f90
index 230484f20c11d87d3ca1f1c40af2408d890a3aea..af9a5b72e717592b2efe2d3156fb581d2be22e0d 100644
--- a/flang/test/Lower/OpenMP/private-derived-type.f90
+++ b/flang/test/Lower/OpenMP/private-derived-type.f90
@@ -37,7 +37,6 @@ end subroutine s4
 !                    Check we do call FortranAInitialize on the derived type
 ! CHECK:             %[[VAL_30:.*]] = fir.call @_FortranAInitialize(%[[VAL_28]], %[[VAL_29]], %[[VAL_27]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
 ! CHECK:             omp.wsloop {
-! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           %[[VAL_39:.*]] = fir.embox %[[VAL_9]]#1 : (!fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
 ! CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (!fir.box<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<none>
diff --git a/flang/test/Lower/OpenMP/simd.f90 b/flang/test/Lower/OpenMP/simd.f90
index 3b2aeceb4c3f9faa958ebd2d5c2431287a5cc2e1..f574a1265e06c4a847999ca1f69125fcd6edd46d 100644
--- a/flang/test/Lower/OpenMP/simd.f90
+++ b/flang/test/Lower/OpenMP/simd.f90
@@ -267,7 +267,6 @@ subroutine lastprivate_with_simd
 !CHECK: }
 !CHECK: omp.yield
 !CHECK: }
-!CHECK: omp.terminator
 !CHECK: }
   !$omp simd lastprivate(sum)
   do i = 1, 100
diff --git a/flang/test/Lower/OpenMP/stop-stmt-in-region.f90 b/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
index f8f2604caacca1fe6e1a2bb6efcbfe7a88f10152..d119c2120c7c586be8efde87109a8a1c31f9c1b8 100644
--- a/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
+++ b/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
@@ -106,7 +106,6 @@ end
 ! CHECK:           ^bb3:
 ! CHECK:             omp.yield
 ! CHECK:           }
-! CHECK:           omp.terminator
 ! CHECK:         }
 ! CHECK:         cf.br ^bb1
 ! CHECK:       ^bb1:
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index ab33b6b380831591fdb7fc057fbba3089e66ed5c..63a43e750979d580793c58ad77668e4776b3fa0b 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -601,7 +601,6 @@ subroutine omp_target_parallel_do
          end do
          !CHECK: omp.yield
          !CHECK: }
-         !CHECK: omp.terminator
          !CHECK: }
       !CHECK: omp.terminator
       !CHECK: }
diff --git a/flang/test/Lower/OpenMP/unstructured.f90 b/flang/test/Lower/OpenMP/unstructured.f90
index e7860b1d6ca8c498a42037ae304a8097964adbc2..a0955c8440c1c85176cf43f084501586c899002a 100644
--- a/flang/test/Lower/OpenMP/unstructured.f90
+++ b/flang/test/Lower/OpenMP/unstructured.f90
@@ -79,7 +79,6 @@ end
 ! CHECK:         @_FortranAioOutputInteger32(%{{.*}}, %[[LOAD_1]])
 ! CHECK:         omp.yield
 ! CHECK:       }
-! CHECK:       omp.terminator
 ! CHECK:     }
 
 ! CHECK:     %[[ALLOCA_1:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
@@ -101,7 +100,6 @@ end
 ! CHECK:       ^bb6:  // 2 preds: ^bb2, ^bb4
 ! CHECK:         omp.yield
 ! CHECK:       }
-! CHECK:       omp.terminator
 ! CHECK:     }
 ! CHECK:     br ^bb1
 ! CHECK:   ^bb4:  // pred: ^bb1
@@ -144,7 +142,6 @@ end
 ! CHECK:             }
 ! CHECK-NEXT:        omp.yield
 ! CHECK-NEXT:      }
-! CHECK-NEXT:      omp.terminator
 ! CHECK-NEXT:    }
 ! CHECK:         omp.terminator
 ! CHECK-NEXT:  }
@@ -179,7 +176,6 @@ end
 ! CHECK:      ^[[BB6]]:
 ! CHECK:        omp.yield
 ! CHECK:      }
-! CHECK:      omp.terminator
 ! CHECK:    }
 ! CHECK:    omp.terminator
 ! CHECK:  }
@@ -222,7 +218,6 @@ end
 ! CHECK:      ^[[BB6]]:
 ! CHECK:        omp.yield
 ! CHECK:      }
-! CHECK:      omp.terminator
 ! CHECK:    }
 ! CHECK:    br ^[[BB1_OUTER]]
 ! CHECK:  ^[[BB3_OUTER]]:
@@ -269,7 +264,6 @@ end
 ! CHECK-NEXT:       ^[[BB6]]:
 ! CHECK:         omp.yield
 ! CHECK:       }
-! CHECK:       omp.terminator
 ! CHECK:     }
 ! CHECK:     omp.terminator
 ! CHECK:   }
@@ -310,7 +304,6 @@ end
 ! CHECK-NEXT:      ^[[BB6]]:
 ! CHECK:        omp.yield
 ! CHECK:      }
-! CHECK:      omp.terminator
 ! CHECK:    }
 ! CHECK:    omp.terminator
 ! CHECK:  }
diff --git a/flang/test/Lower/OpenMP/wsloop-chunks.f90 b/flang/test/Lower/OpenMP/wsloop-chunks.f90
index cacb5b0d4b4a1eaa3295efd687accc6e7cfc5b6c..0fb7d6f1b64faae9fdf2da2198362a786959a78a 100644
--- a/flang/test/Lower/OpenMP/wsloop-chunks.f90
+++ b/flang/test/Lower/OpenMP/wsloop-chunks.f90
@@ -27,7 +27,6 @@ do i=1, 9
 ! CHECK:             {{.*}} = fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
 ! CHECK:             omp.yield
 ! CHECK:           }
-! CHECK:           omp.terminator
 ! CHECK:         }
 
 end do
@@ -50,7 +49,6 @@ do i=1, 9
 ! CHECK:             {{.*}} = fir.call @_FortranAioOutputInteger32({{.*}}, %[[VAL_25]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
 ! CHECK:             omp.yield
 ! CHECK:           }
-! CHECK:           omp.terminator
 ! CHECK:         }
   
 end do
@@ -77,7 +75,6 @@ end do
 ! CHECK:             {{.*}} = fir.call @_FortranAioOutputInteger32({{.*}}, %[[VAL_40]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
 ! CHECK:             omp.yield
 ! CHECK:           }
-! CHECK:           omp.terminator
 ! CHECK:         }
 ! CHECK:         return
 ! CHECK:       }
diff --git a/flang/test/Lower/OpenMP/wsloop-collapse.f90 b/flang/test/Lower/OpenMP/wsloop-collapse.f90
index 67351ca275efbdc370c6260cbcd0b6c17276e631..61ee76d589107177a57b10f9e7b71f5dadca33e4 100644
--- a/flang/test/Lower/OpenMP/wsloop-collapse.f90
+++ b/flang/test/Lower/OpenMP/wsloop-collapse.f90
@@ -84,7 +84,6 @@ program wsloop_collapse
 !CHECK:               hlfir.assign %[[VAL_42]] to %[[VAL_19]]#0 : i32, !fir.ref<i32>
 !CHECK:               omp.yield
 !CHECK-NEXT:        }
-!CHECK-NEXT:        omp.terminator
            x = x + i + j + k
         end do
      end do
diff --git a/flang/test/Lower/OpenMP/wsloop-monotonic.f90 b/flang/test/Lower/OpenMP/wsloop-monotonic.f90
index ec1b216a8695fa8870eb678abced085751e1663b..9659fff2d42e7da114adeafe5b79785d721497d4 100644
--- a/flang/test/Lower/OpenMP/wsloop-monotonic.f90
+++ b/flang/test/Lower/OpenMP/wsloop-monotonic.f90
@@ -27,7 +27,6 @@ program wsloop_dynamic
 !CHECK:         fir.call @_FortranAioEndIoStatement(%[[RTBEGIN]]) {{.*}}: (!fir.ref<i8>) -> i32
   end do
 !CHECK:         omp.yield
-!CHECK:       omp.terminator
 !CHECK:     omp.terminator
 !CHECK:   }
 
diff --git a/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90 b/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
index e011a8de1e69ed06a954503fd6caf1f96e3dc6d3..b1bea525ff4896b18e2e0bfe437f11bad4ee9c8c 100644
--- a/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
+++ b/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
@@ -30,7 +30,6 @@ program wsloop_dynamic
   end do
 !CHECK:         omp.yield
 !CHECK:       }
-!CHECK:       omp.terminator
 !CHECK:     }
 !CHECK:     omp.terminator
 !CHECK:   }
diff --git a/flang/test/Lower/OpenMP/wsloop-ordered.f90 b/flang/test/Lower/OpenMP/wsloop-ordered.f90
index f4fa81c52315c8827f5545352313f131d18da79e..5fa53f7b284471d81b5a7ab7c01fcf344c364fd4 100644
--- a/flang/test/Lower/OpenMP/wsloop-ordered.f90
+++ b/flang/test/Lower/OpenMP/wsloop-ordered.f90
@@ -10,7 +10,6 @@ subroutine wsloop_ordered_no_para()
 ! CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
 ! CHECK:        omp.yield
 ! CHECK:      }
-! CHECK:      omp.terminator
 ! CHECK:    }
 
   !$omp do ordered
@@ -32,7 +31,6 @@ subroutine wsloop_ordered_with_para()
 ! CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
 ! CHECK:        omp.yield
 ! CHECK:      }
-! CHECK:      omp.terminator
 ! CHECK:    }
 
   !$omp do ordered(1)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
index 44fb49239eab423ade67d865e0f672957a042d35..c38a79191bc4ee0ae104313a41d8f0a56cebdefc 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
@@ -100,7 +100,6 @@
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -144,7 +143,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -187,7 +185,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -230,7 +227,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -290,7 +286,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_31]] to %[[VAL_22]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -357,7 +352,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_34]] to %[[VAL_22]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -433,7 +427,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_42]] to %[[VAL_27]]#0 : f64, !fir.ref<f64>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
index d1617bc839cc87da3b784eaf416e67058c9e8fa3..c5278e0ef881524d5e1989778de2c713379986b8 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
@@ -68,7 +68,6 @@
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -112,7 +111,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -155,7 +153,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -198,7 +195,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -258,7 +254,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_31]] to %[[VAL_22]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -325,7 +320,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_34]] to %[[VAL_22]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -401,7 +395,6 @@ end subroutine
 ! CHECK:                 hlfir.assign %[[VAL_42]] to %[[VAL_27]]#0 : f64, !fir.ref<f64>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index 6b901bae539ff9b52e6aeb9f6e5304cfc13145b6..a49eba69ff38cce19d2650262106938e5ef53434 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -252,7 +252,6 @@ end program
 ! CHECK:                 hlfir.destroy %[[VAL_68]] : !hlfir.expr<?xi32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
@@ -295,7 +294,6 @@ end program
 ! CHECK:                 hlfir.destroy %[[VAL_100]] : !hlfir.expr<?xi32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index 66db62a36bc175fa379107ab58dc37d726f853b2..ba7aea0d96c5bb5491c558b2a1e0325f83e06811 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -90,7 +90,6 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_17]] to %[[VAL_16]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index d881ff8c1a026a20342922f46b0537dbfeb5414d..9785f77c0e09185a3ba5b014ab6925c130c22837 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -103,7 +103,6 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_18]] to %[[VAL_21]] : f64, !fir.ref<f64>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index afdd486c7be2a4aab51a15f4719f085a47f6c3a7..ea5df5a8369723a1383aa32f768bdb6b903f5a09 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -99,7 +99,6 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_22]] to %[[VAL_25]] : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index ee77332e9b412a12e5eed902e5b2e724c3eb03a4..9815cfa9c315083ecd5d26ceb65fe7f623c3f64b 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -107,7 +107,6 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_30]] to %[[VAL_33]] : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
index 0696236e8f0736d86b84905e57762dfb34fdbcc2..829229807698a9b16682ddeaef4b68516dd98fcd 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
@@ -49,7 +49,6 @@
 ! CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_20]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
index 713e50f72e9dc954d8e11f2f862ae710a4500dc6..6c060f2e5292a864d44655b4346e004ce7966d9d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
@@ -41,7 +41,6 @@
 ! CHECK:                 %[[VAL_20:.*]] = arith.andi %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_20]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
index 5b0758ac3fcc1d1364e3e065fe622c1bd3acf5c1..284ada404bd60f997c875178d68ce6be43511a1a 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
@@ -43,7 +43,6 @@
 !CHECK: hlfir.assign %[[RES]] to %[[PRV_DECL]]#0 : i32, !fir.ref<i32>
 !CHECK: omp.yield
 !CHECK: omp.terminator
-!CHECK: omp.terminator
 
 subroutine reduction_ieor(y)
   integer :: x, y(:)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
index 958cc6f87e4f8e4e3ddce7d3871a5822edcf03bf..e67253a413ce2ddf0821b340ee33c666a74eb6e0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
@@ -32,7 +32,6 @@
 !CHECK: hlfir.assign %[[RES]] to %[[PRV_DECL]]#0 : i32, !fir.ref<i32>
 !CHECK: omp.yield
 !CHECK: omp.terminator
-!CHECK: omp.terminator
 
 subroutine reduction_ieor(y)
   integer :: x, y(:)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
index 8604a274a659fd93609b35b30fa24176266e6774..315121cc7beb7475c44ff457b7e4d1edd0f3cc04 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
@@ -47,7 +47,6 @@
 ! CHECK:                 %[[VAL_20:.*]] = arith.ori %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_20]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
index a80838047bac3ffcb99c870f61d38c70aedf025f..3da250da9703de6b69b2556ecf1e1f5cb7f010ad 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
@@ -41,7 +41,6 @@
 ! CHECK:                 %[[VAL_20:.*]] = arith.ori %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_20]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 65072f3ccce351f5974f8335c257b13b9c268dd2..30908b6bdd4ce9e00e0225404b5d65a0bc255ecb 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -59,7 +59,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -109,7 +108,6 @@ end subroutine simple_reduction
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -190,7 +188,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 818e1a652064eb34bf6536b2a14c28aaaa3d9e1b..367683de02080a97058e5c8a40708d5dc0c2ec75 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -51,7 +51,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -101,7 +100,6 @@ end subroutine simple_reduction
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -182,7 +180,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index 71d4a34da2d34bea84f340b5aed9b63c908eac0e..9137dd8ff4454e5ca819d0b950c7ab99a366cf20 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -59,7 +59,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -108,7 +107,6 @@ end subroutine
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -189,7 +187,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index 926cf380b0f228b06edb063a02110288ec3a86c1..d1ef676c37407c6fc37e3de4d838a13499481aa3 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -51,7 +51,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -100,7 +99,6 @@ end subroutine
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -181,7 +179,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index 3b7f5771b87c5bf92f771841015c5030b63dd7fa..d1491a0f5561dbad4b06f98da29d5291f4fe5d76 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -59,7 +59,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -109,7 +108,6 @@ end subroutine
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -192,7 +190,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 ! CHECK:         }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index 1a1bffad19d2731a64530b1766a6c3b66ccd6f30..b4df699c49ffaef45711c152620bfcabacc8c9d0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -51,7 +51,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -101,7 +100,6 @@ end subroutine
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -184,7 +182,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 ! CHECK:         }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index 22239ea69e9808164779189c59d48e4814a8e466..8f4a6c22c1d7470332ddb2edc7ee95f9c7d73dd0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -58,7 +58,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -107,7 +106,6 @@ end subroutine
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -188,7 +186,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index 6bdf47bc89d6472d2acb181e07daac33e1252255..9d367797ec216a85d01e719c6f27800467b44c45 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -51,7 +51,6 @@
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -100,7 +99,6 @@ end subroutine
 ! CHECK:                 %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_17]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -181,7 +179,6 @@ end subroutine
 ! CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i1) -> !fir.logical<4>
 ! CHECK:                 hlfir.assign %[[VAL_56]] to %[[VAL_29]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
index 56a43abca42a76fb48728d9e4cf0b9a1937af1d3..9abff8ccfa3b6272842eb6974802865db5957026 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
@@ -63,7 +63,6 @@
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 ! CHECK-LABEL:   func.func @_QPreduction_max_real(
@@ -94,7 +93,6 @@
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : f32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
@@ -120,7 +118,6 @@
 ! CHECK:                   hlfir.assign %[[VAL_47]] to %[[VAL_37]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 }
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
index 775554fd3dcca1c0302a399f0b5cc4382166d3f3..7237d3f903b743cf7660745cc15a6bc62eefffbe 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
@@ -53,7 +53,6 @@
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 ! CHECK-LABEL:   func.func @_QPreduction_max_real(
@@ -84,7 +83,6 @@
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : f32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
@@ -110,7 +108,6 @@
 ! CHECK:                   hlfir.assign %[[VAL_47]] to %[[VAL_37]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 }
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
index d16de4a867a24cfd6922b05f70eb751fbd8e2520..a4bfbaa09d2fa9ff7cb885452b10429b034aeb8a 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
@@ -63,7 +63,6 @@
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 ! CHECK-LABEL:   func.func @_QPreduction_min_real(
@@ -95,7 +94,6 @@
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.parallel {
@@ -122,7 +120,6 @@
 ! CHECK:                   hlfir.assign %[[VAL_47]] to %[[VAL_37]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 }
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
index 04957c7287eae44277ceb1efe46f172c24280265..ce9e53a17523c842908e470c1fe8f6ae69acb78e 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
@@ -53,7 +53,6 @@
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 ! CHECK-LABEL:   func.func @_QPreduction_min_real(
@@ -85,7 +84,6 @@
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.parallel {
@@ -112,7 +110,6 @@
 ! CHECK:                   hlfir.assign %[[VAL_47]] to %[[VAL_37]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 }
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
index 834fb03c1d14d42991c4c42597717b6467ad646d..d83ebb77af3ebd3bfa9dcdcd0ed749cda5272339 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
@@ -47,6 +47,5 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_12]] to %[[VAL_11]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
index 04c6787ab01e0fb31c6ca59142dda65f43a9c45e..18554fbb72aee1e538faac32f0b239e78742c6f1 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
@@ -102,7 +102,6 @@
 ! CHECK:                 %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_14]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -141,7 +140,6 @@ end subroutine
 ! CHECK:                 %[[VAL_16:.*]] = arith.mulf %[[VAL_13]], %[[VAL_15]] fastmath<contract> : f32
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -179,7 +177,6 @@ end subroutine
 ! CHECK:                 %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_14]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -218,7 +215,6 @@ end subroutine
 ! CHECK:                 %[[VAL_16:.*]] = arith.mulf %[[VAL_14]], %[[VAL_15]] fastmath<contract> : f32
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -274,7 +270,6 @@ end subroutine
 ! CHECK:                 %[[VAL_31:.*]] = arith.muli %[[VAL_29]], %[[VAL_30]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_31]] to %[[VAL_22]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -337,7 +332,6 @@ end subroutine
 ! CHECK:                 %[[VAL_34:.*]] = arith.mulf %[[VAL_31]], %[[VAL_33]] fastmath<contract> : f32
 ! CHECK:                 hlfir.assign %[[VAL_34]] to %[[VAL_22]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -409,7 +403,6 @@ end subroutine
 ! CHECK:                 %[[VAL_42:.*]] = arith.mulf %[[VAL_39]], %[[VAL_41]] fastmath<contract> : f64
 ! CHECK:                 hlfir.assign %[[VAL_42]] to %[[VAL_27]]#0 : f64, !fir.ref<f64>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
index d7cafb22852226c265f284faa1dc9d7cb27ce61f..f5c12ccf61f7658f0ea47db2838b4ca5e77a4e41 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
@@ -69,7 +69,6 @@
 ! CHECK:                 %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_14]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -108,7 +107,6 @@ end subroutine
 ! CHECK:                 %[[VAL_16:.*]] = arith.mulf %[[VAL_13]], %[[VAL_15]] fastmath<contract> : f32
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -146,7 +144,6 @@ end subroutine
 ! CHECK:                 %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_14]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -185,7 +182,6 @@ end subroutine
 ! CHECK:                 %[[VAL_16:.*]] = arith.mulf %[[VAL_14]], %[[VAL_15]] fastmath<contract> : f32
 ! CHECK:                 hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -241,7 +237,6 @@ end subroutine
 ! CHECK:                 %[[VAL_31:.*]] = arith.muli %[[VAL_29]], %[[VAL_30]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_31]] to %[[VAL_22]]#0 : i32, !fir.ref<i32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -304,7 +299,6 @@ end subroutine
 ! CHECK:                 %[[VAL_34:.*]] = arith.mulf %[[VAL_31]], %[[VAL_33]] fastmath<contract> : f32
 ! CHECK:                 hlfir.assign %[[VAL_34]] to %[[VAL_22]]#0 : f32, !fir.ref<f32>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
@@ -376,7 +370,6 @@ end subroutine
 ! CHECK:                 %[[VAL_42:.*]] = arith.mulf %[[VAL_39]], %[[VAL_41]] fastmath<contract> : f64
 ! CHECK:                 hlfir.assign %[[VAL_42]] to %[[VAL_27]]#0 : f64, !fir.ref<f64>
 ! CHECK:                 omp.yield
-! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           return
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90
index 1ca8cdc622ea92c0d1a25fedbd53ad8eee703f9e..659ba0600567034ace9be45dae32d0554a5b0bc2 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90
@@ -61,7 +61,6 @@
 !CHECK:          hlfir.assign %[[RES_Z]] to %[[PRV_Z_DECL]]#0 : i32, !fir.ref<i32>
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 !CHECK:      return
 subroutine multiple_reduction(v)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index 746229b8699f396aff5d0f58da768ffb216c501c..5b6ab095b45b65e441682e8d32c17ef8c63a56a9 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -154,7 +154,6 @@ endprogram
 ! CHECK:                 hlfir.assign %[[VAL_44]] to %[[VAL_48]] : f64, !fir.ref<f64>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
index 8d4aa8cd8303890ce0c980f9a07c57ad46089f50..2c126bb8962c2b36a0198f4d579887f0e2141643 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-pointer.f90
@@ -106,7 +106,6 @@ end program
 ! CHECK:                 hlfir.assign %[[VAL_30]] to %[[VAL_32]] : i32, !fir.ptr<i32>
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-schedule.f90 b/flang/test/Lower/OpenMP/wsloop-schedule.f90
index de0a2dae6b5874d1b95c885468c69d178a4d95aa..ae854a2de0c9df8f110435c50e9de1bf24e423bb 100644
--- a/flang/test/Lower/OpenMP/wsloop-schedule.f90
+++ b/flang/test/Lower/OpenMP/wsloop-schedule.f90
@@ -27,7 +27,6 @@ program wsloop_dynamic
   end do
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 !CHECK:      omp.terminator
 !CHECK:    }
diff --git a/flang/test/Lower/OpenMP/wsloop-unstructured.f90 b/flang/test/Lower/OpenMP/wsloop-unstructured.f90
index bd6a0bade8c7ee5c9a5c376385e99a40ed5326b8..8c89f863ab87728c3c338a7923ea0baabca8c364 100644
--- a/flang/test/Lower/OpenMP/wsloop-unstructured.f90
+++ b/flang/test/Lower/OpenMP/wsloop-unstructured.f90
@@ -54,7 +54,6 @@ end subroutine sub
 ! CHECK:               ^bb7:
 ! CHECK:                 omp.yield
 ! CHECK:               }
-! CHECK:               omp.terminator
 ! CHECK:             }
 ! CHECK:             omp.terminator
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/wsloop-variable.f90 b/flang/test/Lower/OpenMP/wsloop-variable.f90
index 7bfb9274f389a37c627fb2e9965cc6884331d945..8d235c10fa1d60289f124ad6609696ab8c176f87 100644
--- a/flang/test/Lower/OpenMP/wsloop-variable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-variable.f90
@@ -35,7 +35,6 @@ program wsloop_variable
 !CHECK:          hlfir.assign %[[TMP11]] to %{{.*}} : f32, !fir.ref<f32>
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 
   !$omp do collapse(2)
@@ -58,7 +57,6 @@ program wsloop_variable
 !CHECK:          hlfir.assign %[[TMP16]] to %{{.*}} : f32, !fir.ref<f32>
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 
   !$omp do
@@ -79,7 +77,6 @@ program wsloop_variable
 !CHECK:          hlfir.assign %[[TMP21]] to %{{.*}} : f32, !fir.ref<f32>
 !CHECK:          omp.yield
 !CHECK:        }
-!CHECK:        omp.terminator
 !CHECK:      }
 
   !$omp do
@@ -162,7 +159,6 @@ subroutine wsloop_variable_sub
 !CHECK:               fir.store %[[VAL_48:.*]]#1 to %[[VAL_17]]#1 : !fir.ref<i64>
 !CHECK:               omp.yield
 !CHECK:             }
-!CHECK:             omp.terminator
 !CHECK:           }
 
   !$omp do
@@ -193,7 +189,6 @@ subroutine wsloop_variable_sub
 !CHECK:               }
 !CHECK:               omp.yield
 !CHECK:             }
-!CHECK:             omp.terminator
 !CHECK:           }
   j1 = 5
   !$omp do
diff --git a/flang/test/Lower/OpenMP/wsloop.f90 b/flang/test/Lower/OpenMP/wsloop.f90
index 6566b3cbd6d4d970baff23ecb1947f3ddb55a1d7..4378233a622ed1ab06fec2ea712b0eef526a3800 100644
--- a/flang/test/Lower/OpenMP/wsloop.f90
+++ b/flang/test/Lower/OpenMP/wsloop.f90
@@ -22,7 +22,6 @@ subroutine simple_loop
     print*, i
   end do
   ! CHECK:          omp.yield
-  ! CHECK:        omp.terminator
   !$OMP END DO
   ! CHECK:      omp.terminator
   !$OMP END PARALLEL
@@ -48,7 +47,6 @@ subroutine simple_loop_with_step
     print*, i
   end do
   ! CHECK:          omp.yield
-  ! CHECK:        omp.terminator
   !$OMP END DO
   ! CHECK:      omp.terminator
   !$OMP END PARALLEL
@@ -74,7 +72,6 @@ subroutine loop_with_schedule_nowait
     print*, i
   end do
   ! CHECK:          omp.yield
-  ! CHECK:        omp.terminator
   !$OMP END DO NOWAIT
   ! CHECK:      omp.terminator
   !$OMP END PARALLEL
diff --git a/flang/test/Lower/record-command-line.f90 b/flang/test/Lower/record-command-line.f90
new file mode 100644
index 0000000000000000000000000000000000000000..a7a25bb9e8725c5d9b94b5a87213a46e8d28f481
--- /dev/null
+++ b/flang/test/Lower/record-command-line.f90
@@ -0,0 +1,9 @@
+! The actual command line is recorded by the frontend and passed on to FC1 as
+! the argument to -record-command-line, so in this test, we just match against
+! some string with spaces that mimics what a hypothetical command line.
+
+! RUN: %flang_fc1 -record-command-line "exec -o infile" %s -emit-fir -o - | FileCheck %s
+
+! CHECK: module attributes {
+! CHECK-SAME: llvm.commandline = "exec -o infile"
+
diff --git a/flang/test/Parser/OpenMP/map-modifiers.f90 b/flang/test/Parser/OpenMP/map-modifiers.f90
new file mode 100644
index 0000000000000000000000000000000000000000..4c6dcde5a20c5dd9193f3ebe4bcedbf75096e34b
--- /dev/null
+++ b/flang/test/Parser/OpenMP/map-modifiers.f90
@@ -0,0 +1,136 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=52 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=52 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(x)
+  integer :: x
+  !$omp target map(ompx_hold, always, present, close, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(OMPX_HOLD, ALWAYS, PRESENT, CLOSE, TO: x)
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | TypeModifier = OmpxHold
+!PARSE-TREE: | | TypeModifier = Always
+!PARSE-TREE: | | TypeModifier = Present
+!PARSE-TREE: | | TypeModifier = Close
+!PARSE-TREE: | | Type = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
+subroutine f01(x)
+  integer :: x
+  !$omp target map(ompx_hold, always, present, close: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f01 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(OMPX_HOLD, ALWAYS, PRESENT, CLOSE: x)
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | TypeModifier = OmpxHold
+!PARSE-TREE: | | TypeModifier = Always
+!PARSE-TREE: | | TypeModifier = Present
+!PARSE-TREE: | | TypeModifier = Close
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
+subroutine f02(x)
+  integer :: x
+  !$omp target map(from: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(FROM: x)
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | Type = From
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
+subroutine f03(x)
+  integer :: x
+  !$omp target map(x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(x)
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
+subroutine f10(x)
+  integer :: x
+  !$omp target map(ompx_hold always, present, close, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f10 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(OMPX_HOLD, ALWAYS, PRESENT, CLOSE, TO: x)
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | TypeModifier = OmpxHold
+!PARSE-TREE: | | TypeModifier = Always
+!PARSE-TREE: | | TypeModifier = Present
+!PARSE-TREE: | | TypeModifier = Close
+!PARSE-TREE: | | Type = To
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
+subroutine f11(x)
+  integer :: x
+  !$omp target map(ompx_hold, always, present, close: x)
+  x = x + 1
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f11 (x)
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP TARGET  MAP(OMPX_HOLD, ALWAYS, PRESENT, CLOSE: x)
+!UNPARSE:   x=x+1_4
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginBlockDirective
+!PARSE-TREE: | OmpBlockDirective -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | TypeModifier = OmpxHold
+!PARSE-TREE: | | TypeModifier = Always
+!PARSE-TREE: | | TypeModifier = Present
+!PARSE-TREE: | | TypeModifier = Close
+!PARSE-TREE: | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+
diff --git a/flang/test/Semantics/OpenMP/map-modifiers.f90 b/flang/test/Semantics/OpenMP/map-modifiers.f90
new file mode 100644
index 0000000000000000000000000000000000000000..355df6e083aa5b5dc855771b000f8086d2a0b622
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/map-modifiers.f90
@@ -0,0 +1,18 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -Werror
+
+subroutine f10(x)
+  integer :: x
+!PORTABILITY: the specification of modifiers without comma separators for the 'MAP' clause has been deprecated
+  !$omp target map(always, present close, to: x)
+  x = x + 1
+  !$omp end target
+end
+
+subroutine f11(x)
+  integer :: x
+!PORTABILITY: the specification of modifiers without comma separators for the 'MAP' clause has been deprecated
+  !$omp target map(always, present, close to: x)
+  x = x + 1
+  !$omp end target
+end
+
diff --git a/flang/test/Transforms/debug-module-2.fir b/flang/test/Transforms/debug-module-2.fir
index 6acdc1df23d27c27b7e9ae1236d265193c571002..c8d618ce34b26609f516731eccc1f3311f0017a4 100644
--- a/flang/test/Transforms/debug-module-2.fir
+++ b/flang/test/Transforms/debug-module-2.fir
@@ -31,5 +31,5 @@ module {
 // CHECK-DAG: #[[GLR:.*]] = #llvm.di_global_variable<{{.*}}name = "glr", linkageName = "_QMhelperEglr"{{.*}}>
 // CHECK-DAG: #[[GLIE:.*]] = #llvm.di_global_variable_expression<var = #[[GLI]]>
 // CHECK-DAG: #[[GLRE:.*]] = #llvm.di_global_variable_expression<var = #[[GLR]]>
-// CHECK-DAG: llvm.mlir.global{{.*}}@_QMhelperEgli() {{{.*}}dbg_expr = #[[GLIE]]}
-// CHECK-DAG: llvm.mlir.global{{.*}}@_QMhelperEglr() {{{.*}}dbg_expr = #[[GLRE]]}
+// CHECK-DAG: llvm.mlir.global{{.*}}@_QMhelperEgli() {{{.*}}dbg_exprs = [#[[GLIE]]]}
+// CHECK-DAG: llvm.mlir.global{{.*}}@_QMhelperEglr() {{{.*}}dbg_exprs = [#[[GLRE]]]}
diff --git a/flang/tools/bbc/CMakeLists.txt b/flang/tools/bbc/CMakeLists.txt
index 18fa7ac1d6cdc9afd2b250127f5527da8878fdbf..9abdcf17b6ebfd7de9b4df55daa633009213e6ff 100644
--- a/flang/tools/bbc/CMakeLists.txt
+++ b/flang/tools/bbc/CMakeLists.txt
@@ -25,6 +25,7 @@ target_link_libraries(bbc PRIVATE
   FIRBuilder
   HLFIRDialect
   HLFIRTransforms
+  flangFrontend
   flangPasses
   FlangOpenMPTransforms
   ${dialect_libs}
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 3a05f5f9844875df3f9f4693d7e29496b09eb4d8..fe5e36f704c76cf9ce3dea335308399001d868e3 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -19,6 +19,7 @@
 #include "flang/Common/OpenMP-features.h"
 #include "flang/Common/Version.h"
 #include "flang/Common/default-kinds.h"
+#include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/TargetOptions.h"
 #include "flang/Lower/Bridge.h"
 #include "flang/Lower/PFTBuilder.h"
@@ -382,12 +383,13 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setIntegerWrapAround(integerWrapAround);
   loweringOptions.setNSWOnLoopVarInc(setNSW);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
-  constexpr const char *tuneCPU = "";
+  Fortran::frontend::TargetOptions targetOpts;
+  Fortran::frontend::CodeGenOptions cgOpts;
   auto burnside = Fortran::lower::LoweringBridge::create(
       ctx, semanticsContext, defKinds, semanticsContext.intrinsics(),
       semanticsContext.targetCharacteristics(), parsing.allCooked(),
       targetTriple, kindMap, loweringOptions, envDefaults,
-      semanticsContext.languageFeatures(), targetMachine, tuneCPU);
+      semanticsContext.languageFeatures(), targetMachine, targetOpts, cgOpts);
   mlir::ModuleOp mlirModule = burnside.getModule();
   if (enableOpenMP) {
     if (enableOpenMPGPU && !enableOpenMPDevice) {
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 8643c9bb48ad41bd0cdb4af4bd75b301134bb298..737ac87f4c7a2133117c65109cee684e68237060 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -152,6 +152,14 @@ function(_get_common_compile_options output_var flags)
     endif()
     list(APPEND compile_options "-Wconversion")
     list(APPEND compile_options "-Wno-sign-conversion")
+    # Silence this warning because _Complex is a part of C99.
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+      list(APPEND compile_options "-fext-numeric-literals")
+    else()
+      list(APPEND compile_options "-Wno-c99-extensions")
+      list(APPEND compile_options "-Wno-gnu-imaginary-constant")
+    endif()
+    list(APPEND compile_options "-Wno-pedantic")
     list(APPEND compile_options "-Wimplicit-fallthrough")
     list(APPEND compile_options "-Wwrite-strings")
     list(APPEND compile_options "-Wextra-semi")
@@ -207,7 +215,7 @@ function(_get_common_test_compile_options output_var c_test flags)
       list(APPEND compile_options "-fno-exceptions")
       list(APPEND compile_options "-fno-unwind-tables")
       list(APPEND compile_options "-fno-asynchronous-unwind-tables")
-      if(NOT ${c_test})
+      if(NOT c_test)
         list(APPEND compile_options "-fno-rtti")
       endif()
     endif()
@@ -227,6 +235,16 @@ function(_get_common_test_compile_options output_var c_test flags)
     # list(APPEND compile_options "-Wimplicit-fallthrough")
     # list(APPEND compile_options "-Wwrite-strings")
     # list(APPEND compile_options "-Wextra-semi")
+    # Silence this warning because _Complex is a part of C99.
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+      if(NOT c_test)
+        list(APPEND compile_options "-fext-numeric-literals")
+      endif()
+    else()
+      list(APPEND compile_options "-Wno-c99-extensions")
+      list(APPEND compile_options "-Wno-gnu-imaginary-constant")
+    endif()
+    list(APPEND compile_options "-Wno-pedantic")
     # if(NOT CMAKE_COMPILER_IS_GNUCXX)
     #   list(APPEND compile_options "-Wnewline-eof")
     #   list(APPEND compile_options "-Wnonportable-system-include-path")
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 72a4d9b65ce9909f671e8c765c3bf27dc34afacc..32c0d199489317d96e1ee6cc904524a8568a2fd8 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -250,6 +250,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sys.mman.munlock
     libc.src.sys.mman.munlockall
     libc.src.sys.mman.munmap
+    libc.src.sys.mman.remap_file_pages
     libc.src.sys.mman.posix_madvise
     libc.src.sys.mman.shm_open
     libc.src.sys.mman.shm_unlink
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index b2e1aefccef66f0c1ec8bac5d3e2bde45d879841..5c09edf7cfb266e1a8c680db27acf265196ae295 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -249,6 +249,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sys.mman.munlock
     libc.src.sys.mman.munlockall
     libc.src.sys.mman.munmap
+    libc.src.sys.mman.remap_file_pages
     libc.src.sys.mman.posix_madvise
     libc.src.sys.mman.shm_open
     libc.src.sys.mman.shm_unlink
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 0daf3ccacbb270a9aefc3b9350d957ca5bda75e3..1cd817171de4be4ac51d42a5a3f85a9bff44228a 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -249,6 +249,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sys.mman.munlock
     libc.src.sys.mman.munlockall
     libc.src.sys.mman.munmap
+    libc.src.sys.mman.remap_file_pages
     libc.src.sys.mman.posix_madvise
     libc.src.sys.mman.shm_open
     libc.src.sys.mman.shm_unlink
@@ -611,6 +612,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.copysignf16
     libc.src.math.exp10f16
     libc.src.math.exp2f16
+    libc.src.math.exp2m1f16
     libc.src.math.expf16
     libc.src.math.expm1f16
     libc.src.math.f16add
diff --git a/libc/docs/complex.rst b/libc/docs/complex.rst
new file mode 100644
index 0000000000000000000000000000000000000000..09fbdd7c6179ef5a5799b55a576a28a1681683a3
--- /dev/null
+++ b/libc/docs/complex.rst
@@ -0,0 +1,65 @@
+.. include:: check.rst
+
+=========
+complex.h
+=========
+
+Macros
+======
+
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| <Func>    | <Func_f> (float) | <Func> (double) | <Func_l> (long double) | <Func_f16> (float16) | <Func_f128> (float128) | C23 Definition Section | C23 Error Handling Section |
++===========+==================+=================+========================+======================+========================+========================+============================+
+| CMPLX     |                  |                 |                        |                      |                        | 7.3.9.3                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+
+Functions
+=========
+
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| <Func>    | <Func_f> (float) | <Func> (double) | <Func_l> (long double) | <Func_f16> (float16) | <Func_f128> (float128) | C23 Definition Section | C23 Error Handling Section |
++===========+==================+=================+========================+======================+========================+========================+============================+
+| cacos     |                  |                 |                        |                      |                        | 7.3.5.1                | G.6.2.1                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| casin     |                  |                 |                        |                      |                        | 7.3.5.2                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| catan     |                  |                 |                        |                      |                        | 7.3.5.3                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| ccos      |                  |                 |                        |                      |                        | 7.3.5.4                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| csin      |                  |                 |                        |                      |                        | 7.3.5.5                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| ctan      |                  |                 |                        |                      |                        | 7.3.5.6                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| cacosh    |                  |                 |                        |                      |                        | 7.3.6.1                | G.6.3.1                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| casinh    |                  |                 |                        |                      |                        | 7.3.6.2                | G.6.3.2                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| catanh    |                  |                 |                        |                      |                        | 7.3.6.3                | G.6.3.3                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| ccosh     |                  |                 |                        |                      |                        | 7.3.6.4                | G.6.3.4                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| csinh     |                  |                 |                        |                      |                        | 7.3.6.5                | G.6.3.5                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| ctanh     |                  |                 |                        |                      |                        | 7.3.6.6                | G.6.3.6                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| cexp      |                  |                 |                        |                      |                        | 7.3.7.1                | G.6.4.1                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| clog      |                  |                 |                        |                      |                        | 7.3.7.2                | G.6.4.2                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| cabs      |                  |                 |                        |                      |                        | 7.3.8.1                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| cpow      |                  |                 |                        |                      |                        | 7.3.8.2                | G.6.5.1                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| csqrt     |                  |                 |                        |                      |                        | 7.3.8.3                | G.6.5.2                    |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| carg      |                  |                 |                        |                      |                        | 7.3.9.1                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| cimag     |                  |                 |                        |                      |                        | 7.3.9.2                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| conj      |                  |                 |                        |                      |                        | 7.3.9.4                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| cproj     |                  |                 |                        |                      |                        | 7.3.9.5                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| creal     |                  |                 |                        |                      |                        | 7.3.9.6                | N/A                        |
++-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/docs/index.rst b/libc/docs/index.rst
index d089a800ab90abe81a06191e57aab1ab1bfb1456..6f759aa215b62cc52a0b63449af49f4a410ead14 100644
--- a/libc/docs/index.rst
+++ b/libc/docs/index.rst
@@ -73,6 +73,7 @@ stages there is no ABI stability in any form.
    libc_search
    c23
    ctype
+   complex
    signal
    threads
    setjmp
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index 0e757e7978bfa53ded78e3e5c1f69db4cf186abd..806dcc64bb93962ab4b7222e92f356715ea4c5a5 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -296,7 +296,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | exp2      | |check|          | |check|         |                        | |check|              |                        | 7.12.6.4               | F.10.3.4                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| exp2m1    | |check|          |                 |                        |                      |                        | 7.12.6.5               | F.10.3.5                   |
+| exp2m1    | |check|          |                 |                        | |check|              |                        | 7.12.6.5               | F.10.3.5                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | expm1     | |check|          | |check|         |                        | |check|              |                        | 7.12.6.6               | F.10.3.6                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 1f3cb59f69e96e889992425461046527d7993cd8..5deb5258d532fe4fc9f5ca3d65a9aad8c22f56f0 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -202,6 +202,16 @@ add_header_macro(
     .llvm-libc-macros.assert_macros
 )
 
+add_header_macro(
+  complex
+  ../libc/newhdrgen/yaml/complex.yaml
+  complex.h.def
+  complex.h
+  DEPENDS
+    .llvm_libc_common_h
+    .llvm-libc-macros.complex_macros
+)
+
 add_header_macro(
   setjmp
   ../libc/newhdrgen/yaml/setjmp.yaml
diff --git a/libc/include/complex.h.def b/libc/include/complex.h.def
new file mode 100644
index 0000000000000000000000000000000000000000..65f5765573e840e3549027c458dc19e2d23b0648
--- /dev/null
+++ b/libc/include/complex.h.def
@@ -0,0 +1,17 @@
+//===-- C standard library header complex.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_COMPLEX_H
+#define LLVM_LIBC_COMPLEX_H
+
+#include "__llvm-libc-common.h"
+#include "llvm-libc-macros/complex-macros.h"
+
+%%public_api()
+
+#endif // LLVM_LIBC_COMPLEX_H
diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt
index 2ba437c8437f2832601a0b3cdb45955ce2b9c775..75194923a452fbf5b42a62cea201e58933b3d829 100644
--- a/libc/include/llvm-libc-macros/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/CMakeLists.txt
@@ -61,6 +61,12 @@ add_macro_header(
     fcntl-macros.h
 )
 
+add_macro_header(
+  complex_macros
+  HDR
+    complex-macros.h
+)
+
 add_macro_header(
   features_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/complex-macros.h b/libc/include/llvm-libc-macros/complex-macros.h
new file mode 100644
index 0000000000000000000000000000000000000000..427c68d289e0bde3090b15de4cf8cd9ccbb89bc4
--- /dev/null
+++ b/libc/include/llvm-libc-macros/complex-macros.h
@@ -0,0 +1,24 @@
+//===-- Definition of macros to be used with complex functions ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_MACROS_COMPLEX_MACROS_H
+#define __LLVM_LIBC_MACROS_COMPLEX_MACROS_H
+
+#ifndef __STDC_NO_COMPLEX__
+
+#define __STDC_VERSION_COMPLEX_H__ 202311L
+
+#define complex _Complex
+#define _Complex_I ((_Complex float)1.0fi)
+#define I _Complex_I
+
+// TODO: Add imaginary macros once GCC or Clang support _Imaginary builtin-type.
+
+#endif
+
+#endif // __LLVM_LIBC_MACROS_COMPLEX_MACROS_H
diff --git a/libc/newhdrgen/yaml/math.yaml b/libc/newhdrgen/yaml/math.yaml
index 04b6a073deace045781047ebf8cb8d1a3ea1a4eb..d8b810b542cb9f3d8a2a602a143f2c95718b8357 100644
--- a/libc/newhdrgen/yaml/math.yaml
+++ b/libc/newhdrgen/yaml/math.yaml
@@ -284,6 +284,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: exp2m1f16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: expf
     standards:
       - stdc
diff --git a/libc/newhdrgen/yaml/sys/mman.yaml b/libc/newhdrgen/yaml/sys/mman.yaml
index 272e7e2af16aaf70ebe66b2fd97c761492a058e1..7c4fb1ef09e79173b1e378944e03e4923c6c1151 100644
--- a/libc/newhdrgen/yaml/sys/mman.yaml
+++ b/libc/newhdrgen/yaml/sys/mman.yaml
@@ -98,6 +98,16 @@ functions:
       - type: void *
       - type: size_t
       - type: int
+  - name: remap_file_pages
+    standards:
+      - Linux
+    return_type: int
+    arguments:
+      - type: void *
+      - type: size_t
+      - type: int
+      - type: size_t
+      - type: int
   - name: shm_open
     standards:
       - POSIX
diff --git a/libc/spec/linux.td b/libc/spec/linux.td
index 395c2a6fe853a77d554411a73f03da193b68d8f3..4aaf18b0803f3da23bc78148a3168ca74db6bddf 100644
--- a/libc/spec/linux.td
+++ b/libc/spec/linux.td
@@ -103,6 +103,17 @@ def Linux : StandardSpec<"Linux"> {
             ArgSpec<UnsignedIntType>,
           ]
         >,
+         FunctionSpec<
+          "remap_file_pages",
+          RetValSpec<IntType>,
+          [
+            ArgSpec<VoidPtr>,
+            ArgSpec<SizeTType>,
+            ArgSpec<IntType>,
+            ArgSpec<SizeTType>,
+            ArgSpec<IntType>,
+          ]
+        >,
       ]  // Functions
   >;
 
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index dadd1ef11d7d3f9cb192b613a75af43203c03342..1b255690c2e453eb04b0ad5d46a48dbed4d2fbb9 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -682,6 +682,7 @@ def StdC : StandardSpec<"stdc"> {
           GuardedFunctionSpec<"exp2f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
 
           FunctionSpec<"exp2m1f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
+          GuardedFunctionSpec<"exp2m1f16", RetValSpec<Float16Type>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
 
           FunctionSpec<"expm1", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"expm1f", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
diff --git a/libc/src/__support/CPP/string_view.h b/libc/src/__support/CPP/string_view.h
index 88f5270c3170a36ff34aa86c54243010ed599f46..745c62c35f0a0a304a5b025db16d8916bf080061 100644
--- a/libc/src/__support/CPP/string_view.h
+++ b/libc/src/__support/CPP/string_view.h
@@ -31,8 +31,8 @@ private:
 
   LIBC_INLINE static int compareMemory(const char *Lhs, const char *Rhs,
                                        size_t Length) {
-    for (size_t I = 0; I < Length; ++I)
-      if (int Diff = (int)Lhs[I] - (int)Rhs[I])
+    for (size_t i = 0; i < Length; ++i)
+      if (int Diff = (int)Lhs[i] - (int)Rhs[i])
         return Diff;
     return 0;
   }
diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h
index d50b6612656dbb0e6a7013dc39fa343f074c8a5e..cef4e5d1f0b139ad91f5c53a1a8e9e7dd3c6c363 100644
--- a/libc/src/__support/CPP/type_traits.h
+++ b/libc/src/__support/CPP/type_traits.h
@@ -25,6 +25,7 @@
 #include "src/__support/CPP/type_traits/is_array.h"
 #include "src/__support/CPP/type_traits/is_base_of.h"
 #include "src/__support/CPP/type_traits/is_class.h"
+#include "src/__support/CPP/type_traits/is_complex.h"
 #include "src/__support/CPP/type_traits/is_const.h"
 #include "src/__support/CPP/type_traits/is_constant_evaluated.h"
 #include "src/__support/CPP/type_traits/is_convertible.h"
diff --git a/libc/src/__support/CPP/type_traits/is_complex.h b/libc/src/__support/CPP/type_traits/is_complex.h
new file mode 100644
index 0000000000000000000000000000000000000000..4f5ee9abdb33a54084b9ce3b834f7a03fbb401b8
--- /dev/null
+++ b/libc/src/__support/CPP/type_traits/is_complex.h
@@ -0,0 +1,40 @@
+//===-- is_complex type_traits ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COMPLEX_H
+#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COMPLEX_H
+
+#include "src/__support/CPP/type_traits/is_same.h"
+#include "src/__support/CPP/type_traits/remove_cv.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace cpp {
+
+// is_complex
+template <typename T> struct is_complex {
+private:
+  template <typename Head, typename... Args>
+  LIBC_INLINE_VAR static constexpr bool __is_unqualified_any_of() {
+    return (... || is_same_v<remove_cv_t<Head>, Args>);
+  }
+
+public:
+  LIBC_INLINE_VAR static constexpr bool value =
+      __is_unqualified_any_of<T, _Complex float, _Complex double,
+                              _Complex long double>();
+};
+template <typename T>
+LIBC_INLINE_VAR constexpr bool is_complex_v = is_complex<T>::value;
+template <typename T1, typename T2>
+LIBC_INLINE_VAR constexpr bool is_complex_type_same() {
+  return is_same_v<remove_cv_t<T1>, T2>;
+}
+
+} // namespace cpp
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_COMPLEX_H
diff --git a/libc/src/__support/CPP/utility/in_place.h b/libc/src/__support/CPP/utility/in_place.h
index b5411f247d5519ea89c58af3b9bf32c34acc322d..3967eb1c535e45682ada54cfddf2f4b786926dbd 100644
--- a/libc/src/__support/CPP/utility/in_place.h
+++ b/libc/src/__support/CPP/utility/in_place.h
@@ -27,11 +27,11 @@ template <class T> struct in_place_type_t {
 };
 template <class T> LIBC_INLINE_VAR constexpr in_place_type_t<T> in_place_type{};
 
-template <size_t I> struct in_place_index_t {
+template <size_t IDX> struct in_place_index_t {
   LIBC_INLINE explicit in_place_index_t() = default;
 };
-template <size_t I>
-LIBC_INLINE_VAR constexpr in_place_index_t<I> in_place_index{};
+template <size_t IDX>
+LIBC_INLINE_VAR constexpr in_place_index_t<IDX> in_place_index{};
 
 } // namespace cpp
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h
index a69bca6acc5123f68329d972bccd596d997b4100..2357b053b60b854927cd6803557b529c36c024e2 100644
--- a/libc/src/__support/FPUtil/BasicOperations.h
+++ b/libc/src/__support/FPUtil/BasicOperations.h
@@ -247,8 +247,9 @@ LIBC_INLINE T fdim(T x, T y) {
   return (x > y ? x - y : 0);
 }
 
+// Avoid reusing `issignaling` macro.
 template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
-LIBC_INLINE int issignaling(const T &x) {
+LIBC_INLINE int issignaling_impl(const T &x) {
   FPBits<T> sx(x);
   return sx.is_signaling_nan();
 }
diff --git a/libc/src/__support/FPUtil/NearestIntegerOperations.h b/libc/src/__support/FPUtil/NearestIntegerOperations.h
index e4e497b898e83a23d2e66f85549a4192cb06d782..93166614cc12a23e9b77d5b9f16ec6ec844648a2 100644
--- a/libc/src/__support/FPUtil/NearestIntegerOperations.h
+++ b/libc/src/__support/FPUtil/NearestIntegerOperations.h
@@ -346,13 +346,14 @@ fromfpx(T x, int rnd, unsigned int width) {
 
 namespace internal {
 
-template <typename F, typename I,
-          cpp::enable_if_t<cpp::is_floating_point_v<F> && cpp::is_integral_v<I>,
+template <typename FloatType, typename IntType,
+          cpp::enable_if_t<cpp::is_floating_point_v<FloatType> &&
+                               cpp::is_integral_v<IntType>,
                            int> = 0>
-LIBC_INLINE I rounded_float_to_signed_integer(F x) {
-  constexpr I INTEGER_MIN = (I(1) << (sizeof(I) * 8 - 1));
-  constexpr I INTEGER_MAX = -(INTEGER_MIN + 1);
-  FPBits<F> bits(x);
+LIBC_INLINE IntType rounded_float_to_signed_integer(FloatType x) {
+  constexpr IntType INTEGER_MIN = (IntType(1) << (sizeof(IntType) * 8 - 1));
+  constexpr IntType INTEGER_MAX = -(INTEGER_MIN + 1);
+  FPBits<FloatType> bits(x);
   auto set_domain_error_and_raise_invalid = []() {
     set_errno_if_required(EDOM);
     raise_except_if_required(FE_INVALID);
@@ -364,7 +365,7 @@ LIBC_INLINE I rounded_float_to_signed_integer(F x) {
   }
 
   int exponent = bits.get_exponent();
-  constexpr int EXPONENT_LIMIT = sizeof(I) * 8 - 1;
+  constexpr int EXPONENT_LIMIT = sizeof(IntType) * 8 - 1;
   if (exponent > EXPONENT_LIMIT) {
     set_domain_error_and_raise_invalid();
     return bits.is_neg() ? INTEGER_MIN : INTEGER_MAX;
@@ -374,29 +375,33 @@ LIBC_INLINE I rounded_float_to_signed_integer(F x) {
       return bits.is_neg() ? INTEGER_MIN : INTEGER_MAX;
     }
     // If the control reaches here, then it means that the rounded
-    // value is the most negative number for the signed integer type I.
+    // value is the most negative number for the signed integer type IntType.
   }
 
-  // For all other cases, if `x` can fit in the integer type `I`,
+  // For all other cases, if `x` can fit in the integer type `IntType`,
   // we just return `x`. static_cast will convert the floating
   // point value to the exact integer value.
-  return static_cast<I>(x);
+  return static_cast<IntType>(x);
 }
 
 } // namespace internal
 
-template <typename F, typename I,
-          cpp::enable_if_t<cpp::is_floating_point_v<F> && cpp::is_integral_v<I>,
+template <typename FloatType, typename IntType,
+          cpp::enable_if_t<cpp::is_floating_point_v<FloatType> &&
+                               cpp::is_integral_v<IntType>,
                            int> = 0>
-LIBC_INLINE I round_to_signed_integer(F x) {
-  return internal::rounded_float_to_signed_integer<F, I>(round(x));
+LIBC_INLINE IntType round_to_signed_integer(FloatType x) {
+  return internal::rounded_float_to_signed_integer<FloatType, IntType>(
+      round(x));
 }
 
-template <typename F, typename I,
-          cpp::enable_if_t<cpp::is_floating_point_v<F> && cpp::is_integral_v<I>,
+template <typename FloatType, typename IntType,
+          cpp::enable_if_t<cpp::is_floating_point_v<FloatType> &&
+                               cpp::is_integral_v<IntType>,
                            int> = 0>
-LIBC_INLINE I round_to_signed_integer_using_current_rounding_mode(F x) {
-  return internal::rounded_float_to_signed_integer<F, I>(
+LIBC_INLINE IntType
+round_to_signed_integer_using_current_rounding_mode(FloatType x) {
+  return internal::rounded_float_to_signed_integer<FloatType, IntType>(
       round_using_current_rounding_mode(x));
 }
 
diff --git a/libc/src/__support/str_to_integer.h b/libc/src/__support/str_to_integer.h
index 1f80229a9eff40ce443576b0bd1ff9c879c5dadd..c8d02434c89ce29f9c54f95dbbe4727b5e872fc2 100644
--- a/libc/src/__support/str_to_integer.h
+++ b/libc/src/__support/str_to_integer.h
@@ -11,6 +11,8 @@
 
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/type_traits.h"
+#include "src/__support/CPP/type_traits/make_unsigned.h"
+#include "src/__support/big_int.h"
 #include "src/__support/common.h"
 #include "src/__support/ctype_utils.h"
 #include "src/__support/macros/config.h"
@@ -77,9 +79,7 @@ template <class T>
 LIBC_INLINE StrToNumResult<T>
 strtointeger(const char *__restrict src, int base,
              const size_t src_len = cpp::numeric_limits<size_t>::max()) {
-  using ResultType = typename cpp::conditional_t<(cpp::is_same_v<T, UInt128> ||
-                                                  cpp::is_same_v<T, Int128>),
-                                                 UInt128, unsigned long long>;
+  using ResultType = make_integral_or_big_int_unsigned_t<T>;
 
   ResultType result = 0;
 
@@ -115,7 +115,8 @@ strtointeger(const char *__restrict src, int base,
                    : cpp::numeric_limits<T>::max();
   ResultType const abs_max =
       (is_positive ? cpp::numeric_limits<T>::max() : NEGATIVE_MAX);
-  ResultType const abs_max_div_by_base = abs_max / base;
+  ResultType const abs_max_div_by_base =
+      static_cast<ResultType>(abs_max / base);
 
   while (src_cur < src_len && isalnum(src[src_cur])) {
     int cur_digit = b36_char_to_int(src[src_cur]);
@@ -137,13 +138,13 @@ strtointeger(const char *__restrict src, int base,
       result = abs_max;
       error_val = ERANGE;
     } else {
-      result = result * base;
+      result = static_cast<ResultType>(result * base);
     }
     if (result > abs_max - cur_digit) {
       result = abs_max;
       error_val = ERANGE;
     } else {
-      result = result + cur_digit;
+      result = static_cast<ResultType>(result + cur_digit);
     }
   }
 
@@ -156,12 +157,7 @@ strtointeger(const char *__restrict src, int base,
       return {cpp::numeric_limits<T>::min(), str_len, error_val};
   }
 
-  return {
-      is_positive
-          ? static_cast<T>(result)
-          : static_cast<T>(
-                -static_cast<make_integral_or_big_int_unsigned_t<T>>(result)),
-      str_len, error_val};
+  return {static_cast<T>(is_positive ? result : -result), str_len, error_val};
 }
 
 } // namespace internal
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 97e2d0d64a147ba9dcf2d1abf9133b533b5e1d1d..53907e47323e9e2e55d22779986c59b4a6f60aff 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -121,6 +121,7 @@ add_math_entrypoint_object(exp2f)
 add_math_entrypoint_object(exp2f16)
 
 add_math_entrypoint_object(exp2m1f)
+add_math_entrypoint_object(exp2m1f16)
 
 add_math_entrypoint_object(exp10)
 add_math_entrypoint_object(exp10f)
diff --git a/libc/src/math/exp2m1f16.h b/libc/src/math/exp2m1f16.h
new file mode 100644
index 0000000000000000000000000000000000000000..39fcebd5a6930003dba9aac185d5fccca9bc4cb2
--- /dev/null
+++ b/libc/src/math/exp2m1f16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for exp2m1f16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_EXP2M1F16_H
+#define LLVM_LIBC_SRC_MATH_EXP2M1F16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 exp2m1f16(float16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_EXP2M1F16_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 627168ee1369971732a92f2a229855e1d50f1ba5..b58935abdf056d0709733b414ff9b439139996bf 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -1500,14 +1500,10 @@ add_entrypoint_object(
     .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
-    libc.src.__support.CPP.array
     libc.src.__support.FPUtil.cast
     libc.src.__support.FPUtil.except_value_utils
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.nearest_integer
-    libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
   COMPILE_OPTIONS
@@ -1535,6 +1531,30 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  exp2m1f16
+  SRCS
+    exp2m1f16.cpp
+  HDRS
+    ../exp2m1f16.h
+  DEPENDS
+    .expxf16
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.common
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.except_value_utils
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.FPUtil.rounding_mode
+    libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.cpu_features
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   exp10
   SRCS
@@ -5235,7 +5255,9 @@ add_header_library(
     expxf16.h
   DEPENDS
     libc.src.__support.CPP.array
+    libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.nearest_integer
     libc.src.__support.FPUtil.polyeval
+    libc.src.__support.macros.attributes
 )
diff --git a/libc/src/math/generic/cos.cpp b/libc/src/math/generic/cos.cpp
index 923ea96852d889e0573138e50c4a9390b5f9ef76..568b1254c6f02be1bc199fdd71acd50bc08b4258 100644
--- a/libc/src/math/generic/cos.cpp
+++ b/libc/src/math/generic/cos.cpp
@@ -93,7 +93,7 @@ LLVM_LIBC_FUNCTION(double, cos, (double x)) {
     }
     return ans;
   };
-  DoubleDouble sin_k = get_idx_dd(k + 128);
+  DoubleDouble msin_k = get_idx_dd(k + 128);
   DoubleDouble cos_k = get_idx_dd(k + 64);
 #else
   // Fast look up version, but needs 256-entry table.
diff --git a/libc/src/math/generic/exp2f16.cpp b/libc/src/math/generic/exp2f16.cpp
index 3c4310259b1df9163233697f5e4e25b468ac1235..858053fe2bccbaae134dd6b0b7036d463179c455 100644
--- a/libc/src/math/generic/exp2f16.cpp
+++ b/libc/src/math/generic/exp2f16.cpp
@@ -10,14 +10,10 @@
 #include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
-#include "src/__support/CPP/array.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/except_value_utils.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/FPUtil/nearest_integer.h"
 #include "src/__support/FPUtil/rounding_mode.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
@@ -89,39 +85,8 @@ LLVM_LIBC_FUNCTION(float16, exp2f16, (float16 x)) {
   if (auto r = EXP2F16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
 
-  // For -25 < x < 16, to compute 2^x, we perform the following range reduction:
-  // find hi, mid, lo, such that:
-  //   x = hi + mid + lo, in which
-  //     hi is an integer,
-  //     mid * 2^3 is an integer,
-  //     -2^(-4) <= lo < 2^(-4).
-  // In particular,
-  //   hi + mid = round(x * 2^3) * 2^(-3).
-  // Then,
-  //   2^x = 2^(hi + mid + lo) = 2^hi * 2^mid * 2^lo.
-  // We store 2^mid in the lookup table EXP2_MID_BITS, and compute 2^hi * 2^mid
-  // by adding hi to the exponent field of 2^mid.  2^lo is computed using a
-  // degree-3 minimax polynomial generated by Sollya.
-
-  float xf = x;
-  float kf = fputil::nearest_integer(xf * 0x1.0p+3f);
-  int x_hi_mid = static_cast<int>(kf);
-  int x_hi = x_hi_mid >> 3;
-  int x_mid = x_hi_mid & 0x7;
-  // lo = x - (hi + mid) = round(x * 2^3) * (-2^(-3)) + x
-  float lo = fputil::multiply_add(kf, -0x1.0p-3f, xf);
-
-  uint32_t exp2_hi_mid_bits =
-      EXP2_MID_BITS[x_mid] +
-      static_cast<uint32_t>(x_hi << fputil::FPBits<float>::FRACTION_LEN);
-  float exp2_hi_mid = fputil::FPBits<float>(exp2_hi_mid_bits).get_val();
-  // Degree-3 minimax polynomial generated by Sollya with the following
-  // commands:
-  //   > display = hexadecimal;
-  //   > P = fpminimax((2^x - 1)/x, 2, [|SG...|], [-2^-4, 2^-4]);
-  //   > 1 + x * P;
-  float exp2_lo = fputil::polyeval(lo, 0x1p+0f, 0x1.62e43p-1f, 0x1.ec0aa6p-3f,
-                                   0x1.c6b4a6p-5f);
+  // exp2(x) = exp2(hi + mid) * exp2(lo)
+  auto [exp2_hi_mid, exp2_lo] = exp2_range_reduction(x);
   return fputil::cast<float16>(exp2_hi_mid * exp2_lo);
 }
 
diff --git a/libc/src/math/generic/exp2m1f16.cpp b/libc/src/math/generic/exp2m1f16.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6a1cd2328a050ea5f6ae904910021bd8287531f5
--- /dev/null
+++ b/libc/src/math/generic/exp2m1f16.cpp
@@ -0,0 +1,162 @@
+//===-- Half-precision 2^x - 1 function -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/exp2m1f16.h"
+#include "expxf16.h"
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
+#include "src/__support/macros/properties/cpu_features.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+static constexpr fputil::ExceptValues<float16, 6> EXP2M1F16_EXCEPTS_LO = {{
+    // (input, RZ output, RU offset, RD offset, RN offset)
+    // x = 0x1.cf4p-13, exp2m1f16(x) = 0x1.41p-13 (RZ)
+    {0x0b3dU, 0x0904U, 1U, 0U, 1U},
+    // x = 0x1.4fcp-12, exp2m1f16(x) = 0x1.d14p-13 (RZ)
+    {0x0d3fU, 0x0b45U, 1U, 0U, 1U},
+    // x = 0x1.63p-11, exp2m1f16(x) = 0x1.ec4p-12 (RZ)
+    {0x118cU, 0x0fb1U, 1U, 0U, 0U},
+    // x = 0x1.6fp-7, exp2m1f16(x) = 0x1.fe8p-8 (RZ)
+    {0x21bcU, 0x1ffaU, 1U, 0U, 1U},
+    // x = -0x1.c6p-10, exp2m1f16(x) = -0x1.3a8p-10 (RZ)
+    {0x9718U, 0x94eaU, 0U, 1U, 0U},
+    // x = -0x1.cfcp-10, exp2m1f16(x) = -0x1.414p-10 (RZ)
+    {0x973fU, 0x9505U, 0U, 1U, 0U},
+}};
+
+#ifdef LIBC_TARGET_CPU_HAS_FMA
+static constexpr size_t N_EXP2M1F16_EXCEPTS_HI = 6;
+#else
+static constexpr size_t N_EXP2M1F16_EXCEPTS_HI = 7;
+#endif
+
+static constexpr fputil::ExceptValues<float16, N_EXP2M1F16_EXCEPTS_HI>
+    EXP2M1F16_EXCEPTS_HI = {{
+        // (input, RZ output, RU offset, RD offset, RN offset)
+        // x = 0x1.e58p-3, exp2m1f16(x) = 0x1.6dcp-3 (RZ)
+        {0x3396U, 0x31b7U, 1U, 0U, 0U},
+#ifndef LIBC_TARGET_CPU_HAS_FMA
+        // x = 0x1.2e8p-2, exp2m1f16(x) = 0x1.d14p-3 (RZ)
+        {0x34baU, 0x3345U, 1U, 0U, 0U},
+#endif
+        // x = 0x1.ad8p-2, exp2m1f16(x) = 0x1.598p-2 (RZ)
+        {0x36b6U, 0x3566U, 1U, 0U, 0U},
+#ifdef LIBC_TARGET_CPU_HAS_FMA
+        // x = 0x1.edcp-2, exp2m1f16(x) = 0x1.964p-2 (RZ)
+        {0x37b7U, 0x3659U, 1U, 0U, 1U},
+#endif
+        // x = -0x1.804p-3, exp2m1f16(x) = -0x1.f34p-4 (RZ)
+        {0xb201U, 0xafcdU, 0U, 1U, 1U},
+        // x = -0x1.f3p-3, exp2m1f16(x) = -0x1.3e4p-3 (RZ)
+        {0xb3ccU, 0xb0f9U, 0U, 1U, 0U},
+        // x = -0x1.294p-1, exp2m1f16(x) = -0x1.53p-2 (RZ)
+        {0xb8a5U, 0xb54cU, 0U, 1U, 1U},
+#ifndef LIBC_TARGET_CPU_HAS_FMA
+        // x = -0x1.a34p-1, exp2m1f16(x) = -0x1.bb4p-2 (RZ)
+        {0xba8dU, 0xb6edU, 0U, 1U, 1U},
+#endif
+    }};
+
+LLVM_LIBC_FUNCTION(float16, exp2m1f16, (float16 x)) {
+  using FPBits = fputil::FPBits<float16>;
+  FPBits x_bits(x);
+
+  uint16_t x_u = x_bits.uintval();
+  uint16_t x_abs = x_u & 0x7fffU;
+
+  // When |x| <= 2^(-3), or |x| >= 11, or x is NaN.
+  if (LIBC_UNLIKELY(x_abs <= 0x3000U || x_abs >= 0x4980U)) {
+    // exp2m1(NaN) = NaN
+    if (x_bits.is_nan()) {
+      if (x_bits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+
+      return x;
+    }
+
+    // When x >= 16.
+    if (x_u >= 0x4c00 && x_bits.is_pos()) {
+      // exp2m1(+inf) = +inf
+      if (x_bits.is_inf())
+        return FPBits::inf().get_val();
+
+      switch (fputil::quick_get_round()) {
+      case FE_TONEAREST:
+      case FE_UPWARD:
+        fputil::set_errno_if_required(ERANGE);
+        fputil::raise_except_if_required(FE_OVERFLOW | FE_INEXACT);
+        return FPBits::inf().get_val();
+      default:
+        return FPBits::max_normal().get_val();
+      }
+    }
+
+    // When x < -11.
+    if (x_u > 0xc980U) {
+      // exp2m1(-inf) = -1
+      if (x_bits.is_inf())
+        return FPBits::one(Sign::NEG).get_val();
+
+      // When -12 < x < -11, round(2^x - 1, HP, RN) = -0x1.ffcp-1.
+      if (x_u < 0xca00U)
+        return fputil::round_result_slightly_down(
+            fputil::cast<float16>(-0x1.ffcp-1));
+
+      // When x <= -12, round(2^x - 1, HP, RN) = -1.
+      switch (fputil::quick_get_round()) {
+      case FE_TONEAREST:
+      case FE_DOWNWARD:
+        return FPBits::one(Sign::NEG).get_val();
+      default:
+        return fputil::cast<float16>(-0x1.ffcp-1);
+      }
+    }
+
+    // When |x| <= 2^(-3).
+    if (x_abs <= 0x3000U) {
+      if (auto r = EXP2M1F16_EXCEPTS_LO.lookup(x_u);
+          LIBC_UNLIKELY(r.has_value()))
+        return r.value();
+
+      float xf = x;
+      // Degree-5 minimax polynomial generated by Sollya with the following
+      // commands:
+      //   > display = hexadecimal;
+      //   > P = fpminimax((2^x - 1)/x, 4, [|SG...|], [-2^-3, 2^-3]);
+      //   > x * P;
+      return fputil::cast<float16>(
+          xf * fputil::polyeval(xf, 0x1.62e43p-1f, 0x1.ebfbdep-3f,
+                                0x1.c6af88p-5f, 0x1.3b45d6p-7f,
+                                0x1.641e7cp-10f));
+    }
+  }
+
+  if (auto r = EXP2M1F16_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
+    return r.value();
+
+  // exp2(x) = exp2(hi + mid) * exp2(lo)
+  auto [exp2_hi_mid, exp2_lo] = exp2_range_reduction(x);
+  // exp2m1(x) = exp2(hi + mid) * exp2(lo) - 1
+  return fputil::cast<float16>(
+      fputil::multiply_add(exp2_hi_mid, exp2_lo, -1.0f));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/expxf16.h b/libc/src/math/generic/expxf16.h
index a0db6cee438e92dc5ec8dfcb087366b2b5b8eec5..35294130a15007be410c05934e693bb091a3321c 100644
--- a/libc/src/math/generic/expxf16.h
+++ b/libc/src/math/generic/expxf16.h
@@ -10,6 +10,7 @@
 #define LLVM_LIBC_SRC_MATH_GENERIC_EXPXF16_H
 
 #include "src/__support/CPP/array.h"
+#include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/FPUtil/nearest_integer.h"
@@ -89,6 +90,43 @@ constexpr cpp::array<uint32_t, 8> EXP2_MID_BITS = {
     0x3fb5'04f3U, 0x3fc5'672aU, 0x3fd7'44fdU, 0x3fea'c0c7U,
 };
 
+LIBC_INLINE ExpRangeReduction exp2_range_reduction(float16 x) {
+  // For -25 < x < 16, to compute 2^x, we perform the following range reduction:
+  // find hi, mid, lo, such that:
+  //   x = hi + mid + lo, in which
+  //     hi is an integer,
+  //     mid * 2^3 is an integer,
+  //     -2^(-4) <= lo < 2^(-4).
+  // In particular,
+  //   hi + mid = round(x * 2^3) * 2^(-3).
+  // Then,
+  //   2^x = 2^(hi + mid + lo) = 2^hi * 2^mid * 2^lo.
+  // We store 2^mid in the lookup table EXP2_MID_BITS, and compute 2^hi * 2^mid
+  // by adding hi to the exponent field of 2^mid.  2^lo is computed using a
+  // degree-3 minimax polynomial generated by Sollya.
+
+  float xf = x;
+  float kf = fputil::nearest_integer(xf * 0x1.0p+3f);
+  int x_hi_mid = static_cast<int>(kf);
+  int x_hi = x_hi_mid >> 3;
+  int x_mid = x_hi_mid & 0x7;
+  // lo = x - (hi + mid) = round(x * 2^3) * (-2^(-3)) + x
+  float lo = fputil::multiply_add(kf, -0x1.0p-3f, xf);
+
+  uint32_t exp2_hi_mid_bits =
+      EXP2_MID_BITS[x_mid] +
+      static_cast<uint32_t>(x_hi << fputil::FPBits<float>::FRACTION_LEN);
+  float exp2_hi_mid = fputil::FPBits<float>(exp2_hi_mid_bits).get_val();
+  // Degree-3 minimax polynomial generated by Sollya with the following
+  // commands:
+  //   > display = hexadecimal;
+  //   > P = fpminimax((2^x - 1)/x, 2, [|SG...|], [-2^-4, 2^-4]);
+  //   > 1 + x * P;
+  float exp2_lo = fputil::polyeval(lo, 0x1p+0f, 0x1.62e43p-1f, 0x1.ec0aa6p-3f,
+                                   0x1.c6b4a6p-5f);
+  return {exp2_hi_mid, exp2_lo};
+}
+
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_SRC_MATH_GENERIC_EXPXF16_H
diff --git a/libc/src/math/generic/issignaling.cpp b/libc/src/math/generic/issignaling.cpp
index 0687a32817b29551c960c32b068f00952238c5a3..657be5ec5015663ad1f9d57ce13f02ae9bc6c91a 100644
--- a/libc/src/math/generic/issignaling.cpp
+++ b/libc/src/math/generic/issignaling.cpp
@@ -14,7 +14,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, issignaling, (double x)) {
-  return fputil::issignaling(x);
+  return fputil::issignaling_impl(x);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingf.cpp b/libc/src/math/generic/issignalingf.cpp
index 410bf7b4d427821db60467e56a4d86fc8b39ccf8..8a8003a2609dc7c7101ac9a649d946dba7779bb0 100644
--- a/libc/src/math/generic/issignalingf.cpp
+++ b/libc/src/math/generic/issignalingf.cpp
@@ -14,7 +14,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, issignalingf, (float x)) {
-  return fputil::issignaling(x);
+  return fputil::issignaling_impl(x);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingf128.cpp b/libc/src/math/generic/issignalingf128.cpp
index 29612ccf677f67c968928bcd0d9c2c3f6e203569..adf0ebac85bd044d9698e6ab6cfa5f65945eb86a 100644
--- a/libc/src/math/generic/issignalingf128.cpp
+++ b/libc/src/math/generic/issignalingf128.cpp
@@ -14,7 +14,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, issignalingf128, (float128 x)) {
-  return fputil::issignaling(x);
+  return fputil::issignaling_impl(x);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingf16.cpp b/libc/src/math/generic/issignalingf16.cpp
index 976b8e2b689b3cf2bda14566ef235de98d7c9f28..68a771e7f898b5c1058bb379b5bd7b25bfbbe6a4 100644
--- a/libc/src/math/generic/issignalingf16.cpp
+++ b/libc/src/math/generic/issignalingf16.cpp
@@ -14,7 +14,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, issignalingf16, (float16 x)) {
-  return fputil::issignaling(x);
+  return fputil::issignaling_impl(x);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingl.cpp b/libc/src/math/generic/issignalingl.cpp
index a3959bcae23575e2f732387abfc5a176615d6d86..b993c5559461e08b914acaad77961c34175cc189 100644
--- a/libc/src/math/generic/issignalingl.cpp
+++ b/libc/src/math/generic/issignalingl.cpp
@@ -14,7 +14,7 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, issignalingl, (long double x)) {
-  return fputil::issignaling(x);
+  return fputil::issignaling_impl(x);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/range_reduction_double_common.h b/libc/src/math/generic/range_reduction_double_common.h
index e23bbff144bee871c3fb11a02f8fa0e86f3e15c5..bcab82f6c9c3a849843d167a0bd7de8ec49626b9 100644
--- a/libc/src/math/generic/range_reduction_double_common.h
+++ b/libc/src/math/generic/range_reduction_double_common.h
@@ -278,6 +278,7 @@ private:
   DoubleDouble y_mid;
 };
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static Float128 range_reduction_small_f128(double x) {
   constexpr Float128 PI_OVER_128_F128 = {
       Sign::POS, -133, 0xc90f'daa2'2168'c234'c4c6'628b'80dc'1cd1_u128};
@@ -300,7 +301,6 @@ static Float128 range_reduction_small_f128(double x) {
   return fputil::quick_mul(y, PI_OVER_128_F128);
 }
 
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr Float128 SIN_K_PI_OVER_128_F128[65] = {
     {Sign::POS, 0, 0},
     {Sign::POS, -133, 0xc90a'afbd'1b33'efc9'c539'edcb'fda0'cf2c_u128},
diff --git a/libc/src/sys/mman/CMakeLists.txt b/libc/src/sys/mman/CMakeLists.txt
index 9c74202a09f035e9e816630950362c9b93a62089..4ea43e14be0297adc423fc0ed71363fb69509579 100644
--- a/libc/src/sys/mman/CMakeLists.txt
+++ b/libc/src/sys/mman/CMakeLists.txt
@@ -86,6 +86,13 @@ add_entrypoint_object(
     .${LIBC_TARGET_OS}.msync
 )
 
+add_entrypoint_object(
+  remap_file_pages
+  ALIAS
+  DEPENDS
+  .${LIBC_TARGET_OS}.remap_file_pages
+)
+
 add_entrypoint_object(
   shm_open
   ALIAS
diff --git a/libc/src/sys/mman/linux/CMakeLists.txt b/libc/src/sys/mman/linux/CMakeLists.txt
index 00f4f0e64ec06bce190b87935f98a68b20d087ae..11188254cfbd4596a2ef15a1d927abaa25d3a7aa 100644
--- a/libc/src/sys/mman/linux/CMakeLists.txt
+++ b/libc/src/sys/mman/linux/CMakeLists.txt
@@ -153,6 +153,19 @@ add_entrypoint_object(
     libc.src.errno.errno
 )
 
+add_entrypoint_object(
+  remap_file_pages
+  SRCS
+    remap_file_pages.cpp
+  HDRS
+    ../remap_file_pages.h
+  DEPENDS
+    libc.include.sys_mman
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
+
 add_header_library(
   shm_common
   HDRS
diff --git a/libc/src/sys/mman/linux/remap_file_pages.cpp b/libc/src/sys/mman/linux/remap_file_pages.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f616e1915ecc5e88293a0c95943c252acbf0c087
--- /dev/null
+++ b/libc/src/sys/mman/linux/remap_file_pages.cpp
@@ -0,0 +1,40 @@
+//===------- Linux implementation of the remap_file_pages function --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/mman/remap_file_pages.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/errno/libc_errno.h"
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, remap_file_pages,
+                   (void *addr, size_t size, int prot, size_t pgoff,
+                    int flags)) {
+#ifdef SYS_remap_file_pages
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_remap_file_pages,
+                                              reinterpret_cast<long>(addr),
+                                              size, prot, pgoff, flags);
+#else
+#error "remap_file_pages syscall is not available."
+#endif
+
+  // A negative return value indicates an error with the magnitude of the
+  // value being the error code.
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+
+  return 0;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/sys/mman/remap_file_pages.h b/libc/src/sys/mman/remap_file_pages.h
new file mode 100644
index 0000000000000000000000000000000000000000..518502960d5cf1eb0f82dfb00f57320b068ddf45
--- /dev/null
+++ b/libc/src/sys/mman/remap_file_pages.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for remap_file_pages function -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_MMAN_REMAP_FILE_PAGES_H
+#define LLVM_LIBC_SRC_SYS_MMAN_REMAP_FILE_PAGES_H
+
+#include "src/__support/macros/config.h"
+#include <stddef.h>
+
+namespace LIBC_NAMESPACE_DECL {
+
+int remap_file_pages(void *addr, size_t size, int prot, size_t pgoff,
+                     int flags);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_SYS_MMAN_REMAP_FILE_PAGES_H
diff --git a/libc/test/UnitTest/FPMatcher.h b/libc/test/UnitTest/FPMatcher.h
index 3cf0b04952ec07778e04138e009732445b0a378a..5220b1245bf3a51d3a9c52fc026f8e0ecf8418d5 100644
--- a/libc/test/UnitTest/FPMatcher.h
+++ b/libc/test/UnitTest/FPMatcher.h
@@ -60,10 +60,95 @@ public:
   }
 };
 
+template <typename T, TestCond Condition> class CFPMatcher : public Matcher<T> {
+  static_assert(
+      cpp::is_complex_v<T>,
+      "CFPMatcher can only be used with complex floating point values.");
+  static_assert(Condition == TestCond::EQ || Condition == TestCond::NE,
+                "Unsupported CFPMatcher test condition.");
+
+  T expected;
+  T actual;
+
+public:
+  CFPMatcher(T expectedValue) : expected(expectedValue) {}
+
+  template <typename CFT> bool matchComplex() {
+    CFT *actualCmplxPtr = reinterpret_cast<CFT *>(&actual);
+    CFT *expectedCmplxPtr = reinterpret_cast<CFT *>(&expected);
+    CFT actualReal = actualCmplxPtr[0];
+    CFT actualImag = actualCmplxPtr[1];
+    CFT expectedReal = expectedCmplxPtr[0];
+    CFT expectedImag = expectedCmplxPtr[1];
+    fputil::FPBits<CFT> actualRealBits(actualReal),
+        expectedRealBits(expectedReal);
+    fputil::FPBits<CFT> actualImagBits(actualImag),
+        expectedImagBits(expectedImag);
+    if (Condition == TestCond::EQ)
+      return ((actualRealBits.is_nan() && expectedRealBits.is_nan()) ||
+              (actualRealBits.uintval() == expectedRealBits.uintval())) &&
+             ((actualImagBits.is_nan() && expectedImagBits.is_nan()) ||
+              (actualImagBits.uintval() == expectedImagBits.uintval()));
+
+    // If condition == TestCond::NE.
+    if (actualRealBits.is_nan() && expectedRealBits.is_nan())
+      return !expectedRealBits.is_nan() && !expectedImagBits.is_nan();
+    if (actualRealBits.is_nan())
+      return !expectedRealBits.is_nan();
+    if (actualImagBits.is_nan())
+      return !expectedImagBits.is_nan();
+    return (expectedRealBits.is_nan() ||
+            actualRealBits.uintval() != expectedRealBits.uintval()) &&
+           (expectedImagBits.is_nan() ||
+            actualImagBits.uintval() != expectedImagBits.uintval());
+  }
+
+  template <typename CFT> void explainErrorComplex() {
+    CFT *actualCmplxPtr = reinterpret_cast<CFT *>(&actual);
+    CFT *expectedCmplxPtr = reinterpret_cast<CFT *>(&expected);
+    CFT actualReal = actualCmplxPtr[0];
+    CFT actualImag = actualCmplxPtr[1];
+    CFT expectedReal = expectedCmplxPtr[0];
+    CFT expectedImag = expectedCmplxPtr[1];
+    tlog << "Expected complex floating point value: "
+         << str(fputil::FPBits<CFT>(expectedReal)) + " + " +
+                str(fputil::FPBits<CFT>(expectedImag)) + "i"
+         << '\n';
+    tlog << "Actual complex floating point value: "
+         << str(fputil::FPBits<CFT>(actualReal)) + " + " +
+                str(fputil::FPBits<CFT>(actualImag)) + "i"
+         << '\n';
+  }
+
+  bool match(T actualValue) {
+    actual = actualValue;
+    if (cpp::is_complex_type_same<T, _Complex float>())
+      return matchComplex<float>();
+    else if (cpp::is_complex_type_same<T, _Complex double>())
+      return matchComplex<double>();
+    else if (cpp::is_complex_type_same<T, _Complex long double>())
+      return matchComplex<long double>();
+  }
+
+  void explainError() override {
+    if (cpp::is_complex_type_same<T, _Complex float>())
+      return explainErrorComplex<float>();
+    else if (cpp::is_complex_type_same<T, _Complex double>())
+      return explainErrorComplex<double>();
+    else if (cpp::is_complex_type_same<T, _Complex long double>())
+      return explainErrorComplex<long double>();
+  }
+};
+
 template <TestCond C, typename T> FPMatcher<T, C> getMatcher(T expectedValue) {
   return FPMatcher<T, C>(expectedValue);
 }
 
+template <TestCond C, typename T>
+CFPMatcher<T, C> getMatcherComplex(T expectedValue) {
+  return CFPMatcher<T, C>(expectedValue);
+}
+
 template <typename T> struct FPTest : public Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using StorageType = typename FPBits::StorageType;
@@ -125,6 +210,10 @@ template <typename T> struct FPTest : public Test {
   EXPECT_THAT(actual, LIBC_NAMESPACE::testing::getMatcher<                     \
                           LIBC_NAMESPACE::testing::TestCond::EQ>(expected))
 
+#define EXPECT_CFP_EQ(expected, actual)                                        \
+  EXPECT_THAT(actual, LIBC_NAMESPACE::testing::getMatcherComplex<              \
+                          LIBC_NAMESPACE::testing::TestCond::EQ>(expected))
+
 #define TEST_FP_EQ(expected, actual)                                           \
   LIBC_NAMESPACE::testing::getMatcher<LIBC_NAMESPACE::testing::TestCond::EQ>(  \
       expected)                                                                \
diff --git a/libc/test/include/CMakeLists.txt b/libc/test/include/CMakeLists.txt
index 921851606287909b56f27a033aed29f8e7062fcf..0d882eb49a7386e87945ac5879583d3edb874709 100644
--- a/libc/test/include/CMakeLists.txt
+++ b/libc/test/include/CMakeLists.txt
@@ -11,6 +11,16 @@ add_libc_test(
     libc.include.llvm-libc-macros.assert_macros
 )
 
+add_libc_test(
+  complex_test
+  SUITE
+    libc_include_tests
+  SRCS
+    complex_test.cpp
+  DEPENDS
+    libc.include.llvm-libc-macros.complex_macros
+)
+
 add_libc_test(
   sys_queue_test
   SUITE
diff --git a/libc/test/include/complex_test.cpp b/libc/test/include/complex_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f6bfe997d98dd1e49a99b6281b375d4d38b8341b
--- /dev/null
+++ b/libc/test/include/complex_test.cpp
@@ -0,0 +1,19 @@
+//===-- Unittests for complex ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDSList-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/llvm-libc-macros/complex-macros.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcComplexTest, VersionMacro) {
+  EXPECT_EQ(__STDC_VERSION_COMPLEX_H__, 202311L);
+}
+
+TEST(LlvmLibcComplexTest, IMacro) { EXPECT_CFP_EQ(1.0fi, I); }
+
+TEST(LlvmLibcComplexTest, _Complex_IMacro) { EXPECT_CFP_EQ(1.0fi, _Complex_I); }
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 0c4118c3694548bf34cf3d96695121d8fc976107..07a9405081f97d1e65e1ed6c761fe9b6ac523978 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -1003,6 +1003,17 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  exp2m1f16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    exp2m1f16_test.cpp
+  DEPENDS
+    libc.src.math.exp2m1f16
+)
+
 add_fp_unittest(
   exp10_test
   NEED_MPFR
diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h
index 995aba7b233ec8a147210877f704e1f796cba8b7..77b465a3a0e631fe81ac1680a57d32650d333229 100644
--- a/libc/test/src/math/RoundToIntegerTest.h
+++ b/libc/test/src/math/RoundToIntegerTest.h
@@ -26,21 +26,21 @@ using LIBC_NAMESPACE::Sign;
 static constexpr int ROUNDING_MODES[4] = {FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO,
                                           FE_TONEAREST};
 
-template <typename F, typename I, bool TestModes = false>
+template <typename FloatType, typename IntType, bool TestModes = false>
 class RoundToIntegerTestTemplate
     : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 public:
-  typedef I (*RoundToIntegerFunc)(F);
+  typedef IntType (*RoundToIntegerFunc)(FloatType);
 
 private:
-  using FPBits = LIBC_NAMESPACE::fputil::FPBits<F>;
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<FloatType>;
   using StorageType = typename FPBits::StorageType;
 
-  const F zero = FPBits::zero().get_val();
-  const F neg_zero = FPBits::zero(Sign::NEG).get_val();
-  const F inf = FPBits::inf().get_val();
-  const F neg_inf = FPBits::inf(Sign::NEG).get_val();
-  const F nan = FPBits::quiet_nan().get_val();
+  const FloatType zero = FPBits::zero().get_val();
+  const FloatType neg_zero = FPBits::zero(Sign::NEG).get_val();
+  const FloatType inf = FPBits::inf().get_val();
+  const FloatType neg_inf = FPBits::inf(Sign::NEG).get_val();
+  const FloatType nan = FPBits::quiet_nan().get_val();
 
   static constexpr StorageType MAX_NORMAL = FPBits::max_normal().uintval();
   static constexpr StorageType MIN_NORMAL = FPBits::min_normal().uintval();
@@ -49,11 +49,12 @@ private:
   static constexpr StorageType MIN_SUBNORMAL =
       FPBits::min_subnormal().uintval();
 
-  static constexpr I INTEGER_MIN = I(1) << (sizeof(I) * 8 - 1);
-  static constexpr I INTEGER_MAX = -(INTEGER_MIN + 1);
+  static constexpr IntType INTEGER_MIN = IntType(1)
+                                         << (sizeof(IntType) * 8 - 1);
+  static constexpr IntType INTEGER_MAX = -(INTEGER_MIN + 1);
 
-  void test_one_input(RoundToIntegerFunc func, F input, I expected,
-                      bool expectError) {
+  void test_one_input(RoundToIntegerFunc func, FloatType input,
+                      IntType expected, bool expectError) {
     LIBC_NAMESPACE::libc_errno = 0;
     LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
 
@@ -120,35 +121,35 @@ public:
   }
 
   void do_round_numbers_test(RoundToIntegerFunc func) {
-    test_one_input(func, zero, I(0), false);
-    test_one_input(func, neg_zero, I(0), false);
-    test_one_input(func, F(1.0), I(1), false);
-    test_one_input(func, F(-1.0), I(-1), false);
-    test_one_input(func, F(10.0), I(10), false);
-    test_one_input(func, F(-10.0), I(-10), false);
-    test_one_input(func, F(1234.0), I(1234), false);
-    test_one_input(func, F(-1234.0), I(-1234), false);
+    test_one_input(func, zero, IntType(0), false);
+    test_one_input(func, neg_zero, IntType(0), false);
+    test_one_input(func, FloatType(1.0), IntType(1), false);
+    test_one_input(func, FloatType(-1.0), IntType(-1), false);
+    test_one_input(func, FloatType(10.0), IntType(10), false);
+    test_one_input(func, FloatType(-10.0), IntType(-10), false);
+    test_one_input(func, FloatType(1234.0), IntType(1234), false);
+    test_one_input(func, FloatType(-1234.0), IntType(-1234), false);
 
     // The rest of this function compares with an equivalent MPFR function
     // which rounds floating point numbers to long values. There is no MPFR
     // function to round to long long or wider integer values. So, we will
-    // the remaining tests only if the width of I less than equal to that of
-    // long.
-    if (sizeof(I) > sizeof(long))
+    // the remaining tests only if the width of IntType less than equal to that
+    // of long.
+    if (sizeof(IntType) > sizeof(long))
       return;
 
-    constexpr int EXPONENT_LIMIT = sizeof(I) * 8 - 1;
+    constexpr int EXPONENT_LIMIT = sizeof(IntType) * 8 - 1;
     constexpr int BIASED_EXPONENT_LIMIT = EXPONENT_LIMIT + FPBits::EXP_BIAS;
     if (BIASED_EXPONENT_LIMIT > FPBits::MAX_BIASED_EXPONENT)
       return;
     // We start with 1.0 so that the implicit bit for x86 long doubles
     // is set.
-    FPBits bits(F(1.0));
+    FPBits bits(FloatType(1.0));
     bits.set_biased_exponent(BIASED_EXPONENT_LIMIT);
     bits.set_sign(Sign::NEG);
     bits.set_mantissa(0);
 
-    F x = bits.get_val();
+    FloatType x = bits.get_val();
     long mpfr_result;
     bool erangeflag = mpfr::round_to_long(x, mpfr_result);
     ASSERT_FALSE(erangeflag);
@@ -167,10 +168,11 @@ public:
   }
 
   void do_fractions_test(RoundToIntegerFunc func, int mode) {
-    constexpr F FRACTIONS[] = {
-        F(0.5), F(-0.5), F(0.115), F(-0.115), F(0.715), F(-0.715),
+    constexpr FloatType FRACTIONS[] = {
+        FloatType(0.5),    FloatType(-0.5),  FloatType(0.115),
+        FloatType(-0.115), FloatType(0.715), FloatType(-0.715),
     };
-    for (F x : FRACTIONS) {
+    for (FloatType x : FRACTIONS) {
       long mpfr_long_result;
       bool erangeflag;
       if (TestModes)
@@ -179,7 +181,7 @@ public:
       else
         erangeflag = mpfr::round_to_long(x, mpfr_long_result);
       ASSERT_FALSE(erangeflag);
-      I mpfr_result = mpfr_long_result;
+      IntType mpfr_result = mpfr_long_result;
       test_one_input(func, x, mpfr_result, false);
     }
   }
@@ -201,23 +203,23 @@ public:
     // This function compares with an equivalent MPFR function which rounds
     // floating point numbers to long values. There is no MPFR function to
     // round to long long or wider integer values. So, we will peform the
-    // comparisons in this function only if the width of I less than equal to
-    // that of long.
-    if (sizeof(I) > sizeof(long))
+    // comparisons in this function only if the width of IntType less than equal
+    // to that of long.
+    if (sizeof(IntType) > sizeof(long))
       return;
 
-    constexpr int EXPONENT_LIMIT = sizeof(I) * 8 - 1;
+    constexpr int EXPONENT_LIMIT = sizeof(IntType) * 8 - 1;
     constexpr int BIASED_EXPONENT_LIMIT = EXPONENT_LIMIT + FPBits::EXP_BIAS;
     if (BIASED_EXPONENT_LIMIT > FPBits::MAX_BIASED_EXPONENT)
       return;
     // We start with 1.0 so that the implicit bit for x86 long doubles
     // is set.
-    FPBits bits(F(1.0));
+    FPBits bits(FloatType(1.0));
     bits.set_biased_exponent(BIASED_EXPONENT_LIMIT);
     bits.set_sign(Sign::NEG);
     bits.set_mantissa(FPBits::FRACTION_MASK);
 
-    F x = bits.get_val();
+    FloatType x = bits.get_val();
     if (TestModes) {
       for (int m : ROUNDING_MODES) {
         LIBC_NAMESPACE::fputil::set_round(m);
@@ -241,29 +243,29 @@ public:
         static_cast<StorageType>((MAX_SUBNORMAL - MIN_SUBNORMAL) / COUNT),
         StorageType(1));
     for (StorageType i = MIN_SUBNORMAL; i <= MAX_SUBNORMAL; i += STEP) {
-      F x = FPBits(i).get_val();
-      if (x == F(0.0))
+      FloatType x = FPBits(i).get_val();
+      if (x == FloatType(0.0))
         continue;
       // All subnormal numbers should round to zero.
       if (TestModes) {
         if (x > 0) {
           LIBC_NAMESPACE::fputil::set_round(FE_UPWARD);
-          test_one_input(func, x, I(1), false);
+          test_one_input(func, x, IntType(1), false);
           LIBC_NAMESPACE::fputil::set_round(FE_DOWNWARD);
-          test_one_input(func, x, I(0), false);
+          test_one_input(func, x, IntType(0), false);
           LIBC_NAMESPACE::fputil::set_round(FE_TOWARDZERO);
-          test_one_input(func, x, I(0), false);
+          test_one_input(func, x, IntType(0), false);
           LIBC_NAMESPACE::fputil::set_round(FE_TONEAREST);
-          test_one_input(func, x, I(0), false);
+          test_one_input(func, x, IntType(0), false);
         } else {
           LIBC_NAMESPACE::fputil::set_round(FE_UPWARD);
-          test_one_input(func, x, I(0), false);
+          test_one_input(func, x, IntType(0), false);
           LIBC_NAMESPACE::fputil::set_round(FE_DOWNWARD);
-          test_one_input(func, x, I(-1), false);
+          test_one_input(func, x, IntType(-1), false);
           LIBC_NAMESPACE::fputil::set_round(FE_TOWARDZERO);
-          test_one_input(func, x, I(0), false);
+          test_one_input(func, x, IntType(0), false);
           LIBC_NAMESPACE::fputil::set_round(FE_TONEAREST);
-          test_one_input(func, x, I(0), false);
+          test_one_input(func, x, IntType(0), false);
         }
       } else {
         test_one_input(func, x, 0L, false);
@@ -275,9 +277,9 @@ public:
     // This function compares with an equivalent MPFR function which rounds
     // floating point numbers to long values. There is no MPFR function to
     // round to long long or wider integer values. So, we will peform the
-    // comparisons in this function only if the width of I less than equal to
-    // that of long.
-    if (sizeof(I) > sizeof(long))
+    // comparisons in this function only if the width of IntType less than equal
+    // to that of long.
+    if (sizeof(IntType) > sizeof(long))
       return;
 
     constexpr int COUNT = 1'000'001;
@@ -286,7 +288,7 @@ public:
         StorageType(1));
     for (StorageType i = MIN_NORMAL; i <= MAX_NORMAL; i += STEP) {
       FPBits xbits(i);
-      F x = xbits.get_val();
+      FloatType x = xbits.get_val();
       // In normal range on x86 platforms, the long double implicit 1 bit can be
       // zero making the numbers NaN. We will skip them.
       if (xbits.is_nan())
@@ -297,7 +299,7 @@ public:
           long mpfr_long_result;
           bool erangeflag = mpfr::round_to_long(x, to_mpfr_rounding_mode(m),
                                                 mpfr_long_result);
-          I mpfr_result = mpfr_long_result;
+          IntType mpfr_result = mpfr_long_result;
           LIBC_NAMESPACE::fputil::set_round(m);
           if (erangeflag)
             test_one_input(func, x, x > 0 ? INTEGER_MAX : INTEGER_MIN, true);
@@ -307,7 +309,7 @@ public:
       } else {
         long mpfr_long_result;
         bool erangeflag = mpfr::round_to_long(x, mpfr_long_result);
-        I mpfr_result = mpfr_long_result;
+        IntType mpfr_result = mpfr_long_result;
         if (erangeflag)
           test_one_input(func, x, x > 0 ? INTEGER_MAX : INTEGER_MIN, true);
         else
@@ -317,9 +319,10 @@ public:
   }
 };
 
-#define LIST_ROUND_TO_INTEGER_TESTS_HELPER(F, I, func, TestModes)              \
+#define LIST_ROUND_TO_INTEGER_TESTS_HELPER(FloatType, IntType, func,           \
+                                           TestModes)                          \
   using LlvmLibcRoundToIntegerTest =                                           \
-      RoundToIntegerTestTemplate<F, I, TestModes>;                             \
+      RoundToIntegerTestTemplate<FloatType, IntType, TestModes>;               \
   TEST_F(LlvmLibcRoundToIntegerTest, InfinityAndNaN) {                         \
     testInfinityAndNaN(&func);                                                 \
   }                                                                            \
@@ -335,10 +338,10 @@ public:
   }                                                                            \
   TEST_F(LlvmLibcRoundToIntegerTest, NormalRange) { testNormalRange(&func); }
 
-#define LIST_ROUND_TO_INTEGER_TESTS(F, I, func)                                \
-  LIST_ROUND_TO_INTEGER_TESTS_HELPER(F, I, func, false)
+#define LIST_ROUND_TO_INTEGER_TESTS(FloatType, IntType, func)                  \
+  LIST_ROUND_TO_INTEGER_TESTS_HELPER(FloatType, IntType, func, false)
 
-#define LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(F, I, func)                     \
-  LIST_ROUND_TO_INTEGER_TESTS_HELPER(F, I, func, true)
+#define LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(FloatType, IntType, func)       \
+  LIST_ROUND_TO_INTEGER_TESTS_HELPER(FloatType, IntType, func, true)
 
 #endif // LLVM_LIBC_TEST_SRC_MATH_ROUNDTOINTEGERTEST_H
diff --git a/libc/test/src/math/exp2m1f16_test.cpp b/libc/test/src/math/exp2m1f16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bf299118429d7eb3681460db1063f762fcdaf70d
--- /dev/null
+++ b/libc/test/src/math/exp2m1f16_test.cpp
@@ -0,0 +1,40 @@
+//===-- Exhaustive test for exp2m1f16 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/exp2m1f16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+using LlvmLibcExp2m1f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+
+// Range: [0, Inf];
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7c00U;
+
+// Range: [-Inf, 0];
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xfc00U;
+
+TEST_F(LlvmLibcExp2m1f16Test, PositiveRange) {
+  for (uint16_t v = POS_START; v <= POS_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp2m1, x,
+                                   LIBC_NAMESPACE::exp2m1f16(x), 0.5);
+  }
+}
+
+TEST_F(LlvmLibcExp2m1f16Test, NegativeRange) {
+  for (uint16_t v = NEG_START; v <= NEG_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Exp2m1, x,
+                                   LIBC_NAMESPACE::exp2m1f16(x), 0.5);
+  }
+}
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index a70b65f23f4b71e2498e053781bbf9abbf6b6db4..c4787229c3ec15288c0c98c4f5b253731c95e159 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -1175,6 +1175,18 @@ add_fp_unittest(
     libc.src.math.exp2m1f
 )
 
+add_fp_unittest(
+  exp2m1f16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    exp2m1f16_test.cpp
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.errno.errno
+    libc.src.math.exp2m1f16
+)
+
 add_fp_unittest(
   exp10_test
   SUITE
diff --git a/libc/test/src/math/smoke/exp2m1f16_test.cpp b/libc/test/src/math/smoke/exp2m1f16_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8be86973f3213458152feba9b8925a5a6561e44b
--- /dev/null
+++ b/libc/test/src/math/smoke/exp2m1f16_test.cpp
@@ -0,0 +1,87 @@
+//===-- Unittests for exp2m1f16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/fenv_macros.h"
+#include "src/errno/libc_errno.h"
+#include "src/math/exp2m1f16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using LlvmLibcExp2m1f16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+TEST_F(LlvmLibcExp2m1f16Test, SpecialNumbers) {
+  LIBC_NAMESPACE::libc_errno = 0;
+
+  EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::exp2m1f16(aNaN));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION(aNaN, LIBC_NAMESPACE::exp2m1f16(sNaN),
+                              FE_INVALID);
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ_ALL_ROUNDING(inf, LIBC_NAMESPACE::exp2m1f16(inf));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ_ALL_ROUNDING(-1.0, LIBC_NAMESPACE::exp2m1f16(neg_inf));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::exp2m1f16(zero));
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, LIBC_NAMESPACE::exp2m1f16(neg_zero));
+  EXPECT_MATH_ERRNO(0);
+}
+
+TEST_F(LlvmLibcExp2m1f16Test, Overflow) {
+  LIBC_NAMESPACE::libc_errno = 0;
+
+  EXPECT_FP_EQ_WITH_EXCEPTION(inf, LIBC_NAMESPACE::exp2m1f16(max_normal),
+                              FE_OVERFLOW | FE_INEXACT);
+  EXPECT_MATH_ERRNO(ERANGE);
+
+  float16 x = 16.0;
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_NEAREST(
+      inf, LIBC_NAMESPACE::exp2m1f16(x), FE_OVERFLOW | FE_INEXACT);
+  EXPECT_MATH_ERRNO(ERANGE);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_UPWARD(inf, LIBC_NAMESPACE::exp2m1f16(x),
+                                              FE_OVERFLOW | FE_INEXACT);
+  EXPECT_MATH_ERRNO(ERANGE);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_DOWNWARD(
+      max_normal, LIBC_NAMESPACE::exp2m1f16(x), FE_INEXACT);
+  EXPECT_MATH_ERRNO(0);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_TOWARD_ZERO(
+      max_normal, LIBC_NAMESPACE::exp2m1f16(x), FE_INEXACT);
+  EXPECT_MATH_ERRNO(0);
+}
+
+TEST_F(LlvmLibcExp2m1f16Test, ResultNearNegOne) {
+  LIBC_NAMESPACE::libc_errno = 0;
+
+  EXPECT_FP_EQ_WITH_EXCEPTION(-1.0, LIBC_NAMESPACE::exp2m1f16(neg_max_normal),
+                              FE_INEXACT);
+
+  EXPECT_FP_EQ_ALL_ROUNDING(-0x1.ffcp-1, LIBC_NAMESPACE::exp2m1f16(-11.0));
+
+  float16 x = -12.0;
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_NEAREST(
+      -1.0, LIBC_NAMESPACE::exp2m1f16(x), FE_INEXACT);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_UPWARD(
+      -0x1.ffcp-1, LIBC_NAMESPACE::exp2m1f16(x), FE_INEXACT);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_DOWNWARD(
+      -1.0, LIBC_NAMESPACE::exp2m1f16(x), FE_INEXACT);
+
+  EXPECT_FP_EQ_WITH_EXCEPTION_ROUNDING_TOWARD_ZERO(
+      -0x1.ffcp-1, LIBC_NAMESPACE::exp2m1f16(x), FE_INEXACT);
+}
diff --git a/libc/test/src/sys/mman/linux/CMakeLists.txt b/libc/test/src/sys/mman/linux/CMakeLists.txt
index b63c76f4306fac5da40d31945758c803ef95016a..a432d88ffb90c906545176b236cb1b211ce61871 100644
--- a/libc/test/src/sys/mman/linux/CMakeLists.txt
+++ b/libc/test/src/sys/mman/linux/CMakeLists.txt
@@ -128,6 +128,23 @@ add_libc_unittest(
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
+add_libc_unittest(
+  remap_file_pages_test
+  SUITE
+    libc_sys_mman_unittests
+  SRCS
+    remap_file_pages_test.cpp
+  DEPENDS
+    libc.include.sys_mman
+    libc.include.sys_stat
+    libc.src.unistd.sysconf
+    libc.test.UnitTest.ErrnoSetterMatcher
+    libc.src.sys.mman.remap_file_pages
+    libc.src.errno.errno
+    libc.src.sys.mman.mmap
+    libc.src.sys.mman.munmap
+)
+
 add_libc_unittest(
   shm_test
   SUITE
diff --git a/libc/test/src/sys/mman/linux/remap_file_pages_test.cpp b/libc/test/src/sys/mman/linux/remap_file_pages_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..267f7598ff70e6ae4eda002146e418cb61406a0e
--- /dev/null
+++ b/libc/test/src/sys/mman/linux/remap_file_pages_test.cpp
@@ -0,0 +1,93 @@
+//===-- Unittests for remap_file_pages ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/errno/libc_errno.h"
+#include "src/fcntl/open.h"
+#include "src/sys/mman/mmap.h"
+#include "src/sys/mman/munmap.h"
+#include "src/sys/mman/remap_file_pages.h"
+#include "src/unistd/close.h"
+#include "src/unistd/sysconf.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+#include <sys/mman.h>
+#include <sys/stat.h> // For S_IRWXU
+
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
+using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
+
+TEST(LlvmLibcRemapFilePagesTest, NoError) {
+  size_t page_size = sysconf(_SC_PAGE_SIZE);
+  ASSERT_GT(page_size, size_t(0));
+
+  // Create a file-backed mapping
+  constexpr const char *file_name = "remap_file_pages.test.noerror";
+  auto test_file = libc_make_test_file_path(file_name);
+  int fd = LIBC_NAMESPACE::open(test_file, O_RDWR | O_CREAT, S_IRWXU);
+  ASSERT_GT(fd, 0);
+
+  // First, allocate some memory using mmap
+  size_t alloc_size = 2 * page_size;
+  LIBC_NAMESPACE::libc_errno = 0;
+  void *addr = LIBC_NAMESPACE::mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE,
+                                    MAP_SHARED, fd, 0);
+  ASSERT_ERRNO_SUCCESS();
+  EXPECT_NE(addr, MAP_FAILED);
+
+  // Now try to remap the pages
+  EXPECT_THAT(LIBC_NAMESPACE::remap_file_pages(addr, page_size, 0, 1, 0),
+              Succeeds());
+
+  // Reset error number for the new function
+  LIBC_NAMESPACE::libc_errno = 0;
+
+  // Clean up
+  EXPECT_THAT(LIBC_NAMESPACE::munmap(addr, alloc_size), Succeeds());
+  EXPECT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+}
+
+TEST(LlvmLibcRemapFilePagesTest, ErrorInvalidFlags) {
+  size_t page_size = sysconf(_SC_PAGE_SIZE);
+  ASSERT_GT(page_size, size_t(0));
+
+  // Create a file-backed mapping
+  constexpr const char *file_name = "remap_file_pages.test.error";
+  auto test_file = libc_make_test_file_path(file_name);
+  int fd = LIBC_NAMESPACE::open(test_file, O_RDWR | O_CREAT, S_IRWXU);
+  ASSERT_GT(fd, 0);
+
+  // First, allocate some memory using mmap
+  size_t alloc_size = 2 * page_size;
+  LIBC_NAMESPACE::libc_errno = 0;
+  void *addr = LIBC_NAMESPACE::mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE,
+                                    MAP_SHARED, fd, 0);
+  ASSERT_ERRNO_SUCCESS();
+  EXPECT_NE(addr, MAP_FAILED);
+
+  // Try to remap pages with an invalid flag MAP_PRIVATE
+  EXPECT_THAT(LIBC_NAMESPACE::remap_file_pages(addr, page_size, PROT_READ, 0,
+                                               MAP_PRIVATE),
+              Fails(EINVAL));
+
+  // Clean up
+  EXPECT_THAT(LIBC_NAMESPACE::munmap(addr, page_size), Succeeds());
+  EXPECT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+}
+
+TEST(LlvmLibcRemapFilePagesTest, ErrorInvalidAddress) {
+  size_t page_size = sysconf(_SC_PAGESIZE);
+  ASSERT_GT(page_size, size_t(0));
+
+  // Use an address that we haven't mapped
+  void *invalid_addr = reinterpret_cast<void *>(0x12345000);
+
+  EXPECT_THAT(LIBC_NAMESPACE::remap_file_pages(invalid_addr, page_size,
+                                               PROT_READ, 0, 0),
+              Fails(EINVAL));
+}
diff --git a/libcxx/.clang-format b/libcxx/.clang-format
index 84a2afaf28f659207a55a5f2752dd04c7a2b120e..a6154c7c4a2bc07b504eb9a15db96e39185ccc7b 100644
--- a/libcxx/.clang-format
+++ b/libcxx/.clang-format
@@ -30,7 +30,6 @@ AttributeMacros: [
                   '_LIBCPP_DEPRECATED_IN_CXX20',
                   '_LIBCPP_DEPRECATED_IN_CXX23',
                   '_LIBCPP_DEPRECATED',
-                  '_LIBCPP_DISABLE_EXTENTSION_WARNING',
                   '_LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION',
                   '_LIBCPP_EXPORTED_FROM_ABI',
                   '_LIBCPP_EXTERN_TEMPLATE_TYPE_VIS',
@@ -38,7 +37,6 @@ AttributeMacros: [
                   '_LIBCPP_HIDDEN',
                   '_LIBCPP_HIDE_FROM_ABI_AFTER_V1',
                   '_LIBCPP_HIDE_FROM_ABI',
-                  '_LIBCPP_INTERNAL_LINKAGE',
                   '_LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS',
                   '_LIBCPP_NO_SANITIZE',
                   '_LIBCPP_NO_UNIQUE_ADDRESS',
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index c2a597f49e317f2af4bf92b35c868ca0e03c36c2..3431ea7dab386b92ecbcfbe89acb221c41902d10 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -747,6 +747,7 @@ set(files
   __type_traits/common_type.h
   __type_traits/conditional.h
   __type_traits/conjunction.h
+  __type_traits/container_traits.h
   __type_traits/copy_cv.h
   __type_traits/copy_cvref.h
   __type_traits/datasizeof.h
diff --git a/libcxx/include/__atomic/aliases.h b/libcxx/include/__atomic/aliases.h
index e27e09af6b77d90a7d314d5a531e17fbb36b6e8a..afc64eaaa69e7bff59b26886031cd3f783d34313 100644
--- a/libcxx/include/__atomic/aliases.h
+++ b/libcxx/include/__atomic/aliases.h
@@ -37,7 +37,7 @@ using atomic_long   = atomic<long>;
 using atomic_ulong  = atomic<unsigned long>;
 using atomic_llong  = atomic<long long>;
 using atomic_ullong = atomic<unsigned long long>;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 using atomic_char8_t = atomic<char8_t>;
 #endif
 using atomic_char16_t = atomic<char16_t>;
diff --git a/libcxx/include/__atomic/atomic_lock_free.h b/libcxx/include/__atomic/atomic_lock_free.h
index 0715439db45039131e833d0f4dfc400e8a459b6b..3ae9b8856e81020ee55a2d438037176b70644151 100644
--- a/libcxx/include/__atomic/atomic_lock_free.h
+++ b/libcxx/include/__atomic/atomic_lock_free.h
@@ -18,7 +18,7 @@
 #if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE)
 #  define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE
 #  define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 #    define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE
 #  endif
 #  define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE
@@ -32,7 +32,7 @@
 #elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE)
 #  define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE
 #  define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 #    define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE
 #  endif
 #  define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE
diff --git a/libcxx/include/__bit/byteswap.h b/libcxx/include/__bit/byteswap.h
index 6225ecf2f92dfb4fff1146e73a7272ccf2af7e1d..d761e6a6fdb46ed35b8c8f69071e14d6bc7827a2 100644
--- a/libcxx/include/__bit/byteswap.h
+++ b/libcxx/include/__bit/byteswap.h
@@ -32,7 +32,7 @@ template <integral _Tp>
     return __builtin_bswap32(__val);
   } else if constexpr (sizeof(_Tp) == 8) {
     return __builtin_bswap64(__val);
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   } else if constexpr (sizeof(_Tp) == 16) {
 #    if __has_builtin(__builtin_bswap128)
     return __builtin_bswap128(__val);
@@ -40,7 +40,7 @@ template <integral _Tp>
     return static_cast<_Tp>(byteswap(static_cast<uint64_t>(__val))) << 64 |
            static_cast<_Tp>(byteswap(static_cast<uint64_t>(__val >> 64)));
 #    endif // __has_builtin(__builtin_bswap128)
-#  endif   // _LIBCPP_HAS_NO_INT128
+#  endif   // _LIBCPP_HAS_INT128
   } else {
     static_assert(sizeof(_Tp) == 0, "byteswap is unimplemented for integral types of this size");
   }
diff --git a/libcxx/include/__bit/countl.h b/libcxx/include/__bit/countl.h
index bb09e8e5b58564ecb541a1c4af750daafa7a628c..d4df1d049b294b982cf65b0aec36fd8f87de55f2 100644
--- a/libcxx/include/__bit/countl.h
+++ b/libcxx/include/__bit/countl.h
@@ -39,7 +39,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
   return __builtin_clzll(__x);
 }
 
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(__uint128_t __x) _NOEXCEPT {
 #  if __has_builtin(__builtin_clzg)
   return __builtin_clzg(__x);
@@ -57,7 +57,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_clz(__uint128_t __x)
                             : __builtin_clzll(static_cast<unsigned long long>(__x >> 64));
 #  endif
 }
-#endif // _LIBCPP_HAS_NO_INT128
+#endif // _LIBCPP_HAS_INT128
 
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countl_zero(_Tp __t) _NOEXCEPT {
diff --git a/libcxx/include/__charconv/tables.h b/libcxx/include/__charconv/tables.h
index 6b93536b8c1bac893919787248b061b817e04efa..9568bf841cd0296370842f59d957ccf47d6a2c1c 100644
--- a/libcxx/include/__charconv/tables.h
+++ b/libcxx/include/__charconv/tables.h
@@ -95,7 +95,7 @@ inline constexpr uint64_t __pow10_64[20] = {
     UINT64_C(1000000000000000000),
     UINT64_C(10000000000000000000)};
 
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 inline constexpr int __pow10_128_offset      = 0;
 inline constexpr __uint128_t __pow10_128[40] = {
     UINT64_C(0),
diff --git a/libcxx/include/__charconv/to_chars_base_10.h b/libcxx/include/__charconv/to_chars_base_10.h
index c49f4f6797aa4373953046297bd014de7ed1cc25..06e4e692337df50123431aa0f895dacfd46fb0a4 100644
--- a/libcxx/include/__charconv/to_chars_base_10.h
+++ b/libcxx/include/__charconv/to_chars_base_10.h
@@ -124,7 +124,7 @@ __base_10_u64(char* __buffer, uint64_t __value) noexcept {
   return __itoa::__append10(__buffer, __value);
 }
 
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 /// \returns 10^\a exp
 ///
 /// \pre \a exp [19, 39]
diff --git a/libcxx/include/__charconv/to_chars_integral.h b/libcxx/include/__charconv/to_chars_integral.h
index ccb5856df17993fae9e88714dfa8ba1f8f143eaf..fd92be4b4ce91e2ffdce96d8951416c25295bc93 100644
--- a/libcxx/include/__charconv/to_chars_integral.h
+++ b/libcxx/include/__charconv/to_chars_integral.h
@@ -71,7 +71,7 @@ __to_chars_itoa(char* __first, char* __last, _Tp __value, false_type) {
     return {__last, errc::value_too_large};
 }
 
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 template <>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI to_chars_result
 __to_chars_itoa(char* __first, char* __last, __uint128_t __value, false_type) {
diff --git a/libcxx/include/__charconv/traits.h b/libcxx/include/__charconv/traits.h
index c91c6da324797872c1217ae58a7e6c58b8bca8fc..2cb37c8cfb023829b357ed5009dcc2a6c73bf620 100644
--- a/libcxx/include/__charconv/traits.h
+++ b/libcxx/include/__charconv/traits.h
@@ -88,7 +88,7 @@ struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) == sizeof(uin
   }
 };
 
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 template <typename _Tp>
 struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) == sizeof(__uint128_t)> > {
   using type = __uint128_t;
diff --git a/libcxx/include/__chrono/exception.h b/libcxx/include/__chrono/exception.h
index cc408d78a36da6f5e8b2c13aa9ef017ae570d855..64484fb551b77daa0e5b8f46753df556a3a29657 100644
--- a/libcxx/include/__chrono/exception.h
+++ b/libcxx/include/__chrono/exception.h
@@ -73,7 +73,7 @@ private:
 template <class _Duration>
 [[noreturn]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI void __throw_nonexistent_local_time(
     [[maybe_unused]] const local_time<_Duration>& __time, [[maybe_unused]] const local_info& __info) {
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   throw nonexistent_local_time(__time, __info);
 #    else
   _LIBCPP_VERBOSE_ABORT("nonexistent_local_time was thrown in -fno-exceptions mode");
@@ -117,7 +117,7 @@ private:
 template <class _Duration>
 [[noreturn]] _LIBCPP_AVAILABILITY_TZDB _LIBCPP_HIDE_FROM_ABI void __throw_ambiguous_local_time(
     [[maybe_unused]] const local_time<_Duration>& __time, [[maybe_unused]] const local_info& __info) {
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   throw ambiguous_local_time(__time, __info);
 #    else
   _LIBCPP_VERBOSE_ABORT("ambiguous_local_time was thrown in -fno-exceptions mode");
diff --git a/libcxx/include/__chrono/file_clock.h b/libcxx/include/__chrono/file_clock.h
index 4dd3f88ce5ba4bf499316539bec89ea75b5a75a9..b4b7e9dc14e70c4725bcdf3f8c0cc53d380b1e01 100644
--- a/libcxx/include/__chrono/file_clock.h
+++ b/libcxx/include/__chrono/file_clock.h
@@ -47,7 +47,7 @@ _LIBCPP_END_NAMESPACE_STD
 #ifndef _LIBCPP_CXX03_LANG
 _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
 struct _FilesystemClock {
-#  if !defined(_LIBCPP_HAS_NO_INT128)
+#  if _LIBCPP_HAS_INT128
   typedef __int128_t rep;
   typedef nano period;
 #  else
diff --git a/libcxx/include/__condition_variable/condition_variable.h b/libcxx/include/__condition_variable/condition_variable.h
index de35aaca1070eb03354cbd8aef014425adeba331..61412541b4b51a8ba3e3cd10dae9d063b233eede 100644
--- a/libcxx/include/__condition_variable/condition_variable.h
+++ b/libcxx/include/__condition_variable/condition_variable.h
@@ -45,7 +45,7 @@ class _LIBCPP_EXPORTED_FROM_ABI condition_variable {
 public:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR condition_variable() _NOEXCEPT = default;
 
-#  ifdef _LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
+#  if _LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
   ~condition_variable() = default;
 #  else
   ~condition_variable();
@@ -83,7 +83,7 @@ public:
 private:
   void
   __do_timed_wait(unique_lock<mutex>& __lk, chrono::time_point<chrono::system_clock, chrono::nanoseconds>) _NOEXCEPT;
-#  if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+#  if _LIBCPP_HAS_COND_CLOCKWAIT
   _LIBCPP_HIDE_FROM_ABI void
   __do_timed_wait(unique_lock<mutex>& __lk, chrono::time_point<chrono::steady_clock, chrono::nanoseconds>) _NOEXCEPT;
 #  endif
@@ -180,7 +180,7 @@ cv_status condition_variable::wait_for(unique_lock<mutex>& __lk, const chrono::d
   using __ns_rep                   = nanoseconds::rep;
   steady_clock::time_point __c_now = steady_clock::now();
 
-#  if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+#  if _LIBCPP_HAS_COND_CLOCKWAIT
   using __clock_tp_ns     = time_point<steady_clock, nanoseconds>;
   __ns_rep __now_count_ns = std::__safe_nanosecond_cast(__c_now.time_since_epoch()).count();
 #  else
@@ -205,7 +205,7 @@ condition_variable::wait_for(unique_lock<mutex>& __lk, const chrono::duration<_R
   return wait_until(__lk, chrono::steady_clock::now() + __d, std::move(__pred));
 }
 
-#  if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+#  if _LIBCPP_HAS_COND_CLOCKWAIT
 inline void condition_variable::__do_timed_wait(
     unique_lock<mutex>& __lk, chrono::time_point<chrono::steady_clock, chrono::nanoseconds> __tp) _NOEXCEPT {
   using namespace chrono;
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 0d71264611ff66d87b01188453536e837a2c59df..fcba56f7e3d5b153b2ae4555c28f1525f570245f 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -225,9 +225,14 @@ _LIBCPP_HARDENING_MODE_DEBUG
 #      define _LIBCPP_MSVCRT // Using Microsoft's C Runtime library
 #    endif
 #    if (defined(_M_AMD64) || defined(__x86_64__)) || (defined(_M_ARM) || defined(__arm__))
-#      define _LIBCPP_HAS_BITSCAN64
+#      define _LIBCPP_HAS_BITSCAN64 1
+#    else
+#      define _LIBCPP_HAS_BITSCAN64 0
 #    endif
-#    define _LIBCPP_HAS_OPEN_WITH_WCHAR
+#    define _LIBCPP_HAS_OPEN_WITH_WCHAR 1
+#  else
+#    define _LIBCPP_HAS_OPEN_WITH_WCHAR 0
+#    define _LIBCPP_HAS_BITSCAN64 0
 #  endif // defined(_WIN32)
 
 #  if defined(_AIX) && !defined(__64BIT__)
@@ -318,23 +323,33 @@ typedef __char32_t char32_t;
 
 // Objective-C++ features (opt-in)
 #  if __has_feature(objc_arc)
-#    define _LIBCPP_HAS_OBJC_ARC
+#    define _LIBCPP_HAS_OBJC_ARC 1
+#  else
+#    define _LIBCPP_HAS_OBJC_ARC 0
 #  endif
 
 #  if __has_feature(objc_arc_weak)
-#    define _LIBCPP_HAS_OBJC_ARC_WEAK
+#    define _LIBCPP_HAS_OBJC_ARC_WEAK 1
+#  else
+#    define _LIBCPP_HAS_OBJC_ARC_WEAK 0
 #  endif
 
 #  if __has_extension(blocks)
-#    define _LIBCPP_HAS_EXTENSION_BLOCKS
+#    define _LIBCPP_HAS_EXTENSION_BLOCKS 1
+#  else
+#    define _LIBCPP_HAS_EXTENSION_BLOCKS 0
 #  endif
 
-#  if defined(_LIBCPP_HAS_EXTENSION_BLOCKS) && defined(__APPLE__)
-#    define _LIBCPP_HAS_BLOCKS_RUNTIME
+#  if _LIBCPP_HAS_EXTENSION_BLOCKS && defined(__APPLE__)
+#    define _LIBCPP_HAS_BLOCKS_RUNTIME 1
+#  else
+#    define _LIBCPP_HAS_BLOCKS_RUNTIME 0
 #  endif
 
-#  if !__has_feature(address_sanitizer)
-#    define _LIBCPP_HAS_NO_ASAN
+#  if __has_feature(address_sanitizer)
+#    define _LIBCPP_HAS_ASAN 1
+#  else
+#    define _LIBCPP_HAS_ASAN 0
 #  endif
 
 #  define _LIBCPP_ALWAYS_INLINE __attribute__((__always_inline__))
@@ -457,7 +472,7 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_HARDENING_SIG n // "none"
 #  endif
 
-#  ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if !_LIBCPP_HAS_EXCEPTIONS
 #    define _LIBCPP_EXCEPTIONS_SIG n
 #  else
 #    define _LIBCPP_EXCEPTIONS_SIG e
@@ -597,7 +612,9 @@ typedef __char32_t char32_t;
 #  endif
 
 #  if !defined(__SIZEOF_INT128__) || defined(_MSC_VER)
-#    define _LIBCPP_HAS_NO_INT128
+#    define _LIBCPP_HAS_INT128 0
+#  else
+#    define _LIBCPP_HAS_INT128 1
 #  endif
 
 #  ifdef _LIBCPP_CXX03_LANG
@@ -629,33 +646,39 @@ typedef __char32_t char32_t;
 // If we are getting operator new from the MSVC CRT, then allocation overloads
 // for align_val_t were added in 19.12, aka VS 2017 version 15.3.
 #  if defined(_LIBCPP_MSVCRT) && defined(_MSC_VER) && _MSC_VER < 1912
-#    define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#    define _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION 0
 #  elif defined(_LIBCPP_ABI_VCRUNTIME) && !defined(__cpp_aligned_new)
 // We're deferring to Microsoft's STL to provide aligned new et al. We don't
 // have it unless the language feature test macro is defined.
-#    define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#    define _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION 0
 #  elif defined(__MVS__)
-#    define _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#    define _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION 0
+#  else
+#    define _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION 1
 #  endif
 
-#  if defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) || (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606)
-#    define _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+#  if !_LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION || (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606)
+#    define _LIBCPP_HAS_ALIGNED_ALLOCATION 0
+#  else
+#    define _LIBCPP_HAS_ALIGNED_ALLOCATION 1
 #  endif
 
 // It is not yet possible to use aligned_alloc() on all Apple platforms since
 // 10.15 was the first version to ship an implementation of aligned_alloc().
 #  if defined(__APPLE__)
 #    if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) &&                                                     \
-         __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500)
-#      define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
-#    endif
-#    if (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) &&                                                    \
+         __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500) ||                                                    \
+        (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) &&                                                    \
          __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 130000)
-#      define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
+#      define _LIBCPP_HAS_C11_ALIGNED_ALLOC 0
+#    else
+#      define _LIBCPP_HAS_C11_ALIGNED_ALLOC 1
 #    endif
 #  elif defined(__ANDROID__) && __ANDROID_API__ < 28
 // Android only provides aligned_alloc when targeting API 28 or higher.
-#    define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
+#    define _LIBCPP_HAS_C11_ALIGNED_ALLOC 0
+#  else
+#    define _LIBCPP_HAS_C11_ALIGNED_ALLOC 1
 #  endif
 
 #  if defined(__APPLE__) || defined(__FreeBSD__)
@@ -667,7 +690,9 @@ typedef __char32_t char32_t;
 #  endif
 
 #  if _LIBCPP_STD_VER <= 17 || !defined(__cpp_char8_t)
-#    define _LIBCPP_HAS_NO_CHAR8_T
+#    define _LIBCPP_HAS_CHAR8_T 0
+#  else
+#    define _LIBCPP_HAS_CHAR8_T 1
 #  endif
 
 // Deprecation macros.
@@ -726,7 +751,7 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_DEPRECATED_IN_CXX26
 #  endif
 
-#  if !defined(_LIBCPP_HAS_NO_CHAR8_T)
+#  if _LIBCPP_HAS_CHAR8_T
 #    define _LIBCPP_DEPRECATED_WITH_CHAR8_T _LIBCPP_DEPRECATED
 #  else
 #    define _LIBCPP_DEPRECATED_WITH_CHAR8_T
@@ -821,12 +846,18 @@ typedef __char32_t char32_t;
 
 #  if defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
 #    if defined(__ANDROID__) && __ANDROID_API__ >= 30
-#      define _LIBCPP_HAS_COND_CLOCKWAIT
+#      define _LIBCPP_HAS_COND_CLOCKWAIT 1
 #    elif defined(_LIBCPP_GLIBC_PREREQ)
 #      if _LIBCPP_GLIBC_PREREQ(2, 30)
-#        define _LIBCPP_HAS_COND_CLOCKWAIT
+#        define _LIBCPP_HAS_COND_CLOCKWAIT 1
+#      else
+#        define _LIBCPP_HAS_COND_CLOCKWAIT 0
 #      endif
+#    else
+#      define _LIBCPP_HAS_COND_CLOCKWAIT 0
 #    endif
+#  else
+#    define _LIBCPP_HAS_COND_CLOCKWAIT 0
 #  endif
 
 #  if defined(_LIBCPP_HAS_NO_THREADS) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
@@ -863,7 +894,9 @@ typedef __char32_t char32_t;
       (defined(_LIBCPP_HAS_THREAD_API_C11) && defined(__Fuchsia__)) ||                                                 \
        defined(_LIBCPP_HAS_THREAD_API_WIN32)
 // clang-format on
-#    define _LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION
+#    define _LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION 1
+#  else
+#    define _LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION 0
 #  endif
 
 // Destroying a condvar is a nop on Windows.
@@ -875,7 +908,9 @@ typedef __char32_t char32_t;
 // TODO(EricWF): This is potentially true for some pthread implementations
 // as well.
 #  if (defined(_LIBCPP_HAS_THREAD_API_C11) && defined(__Fuchsia__)) || defined(_LIBCPP_HAS_THREAD_API_WIN32)
-#    define _LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
+#    define _LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION 1
+#  else
+#    define _LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION 0
 #  endif
 
 #  if defined(__BIONIC__) || defined(__NuttX__) || defined(__Fuchsia__) || defined(__wasi__) ||                        \
@@ -891,8 +926,9 @@ typedef __char32_t char32_t;
 
 #  if !defined(_LIBCPP_HAS_C_ATOMIC_IMP) && !defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) &&                                    \
       !defined(_LIBCPP_HAS_EXTERNAL_ATOMIC_IMP)
-#    define _LIBCPP_HAS_NO_ATOMIC_HEADER
+#    define _LIBCPP_HAS_ATOMIC_HEADER 0
 #  else
+#    define _LIBCPP_HAS_ATOMIC_HEADER 1
 #    ifndef _LIBCPP_ATOMIC_FLAG_TYPE
 #      define _LIBCPP_ATOMIC_FLAG_TYPE bool
 #    endif
@@ -904,19 +940,18 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS
 #  endif
 
-#  if defined(_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS)
-#    if defined(__clang__) && __has_attribute(acquire_capability)
 // Work around the attribute handling in clang.  When both __declspec and
 // __attribute__ are present, the processing goes awry preventing the definition
 // of the types. In MinGW mode, __declspec evaluates to __attribute__, and thus
 // combining the two does work.
-#      if !defined(_MSC_VER)
-#        define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS
-#      endif
-#    endif
+#  if defined(_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS) && defined(__clang__) &&                                       \
+      __has_attribute(acquire_capability) && !defined(_MSC_VER)
+#    define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS 1
+#  else
+#    define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS 0
 #  endif
 
-#  ifdef _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS
+#  if _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS
 #    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x) __attribute__((x))
 #  else
 #    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x)
@@ -997,7 +1032,7 @@ typedef __char32_t char32_t;
 // functions is gradually being added to existing C libraries. The conditions
 // below check for known C library versions and conditions under which these
 // functions are declared by the C library.
-#  define _LIBCPP_HAS_NO_C8RTOMB_MBRTOC8
+//
 // GNU libc 2.36 and newer declare c8rtomb() and mbrtoc8() in C++ modes if
 // __cpp_char8_t is defined or if C2X extensions are enabled. Determining
 // the latter depends on internal GNU libc details that are not appropriate
@@ -1005,8 +1040,12 @@ typedef __char32_t char32_t;
 // defined are ignored.
 #  if defined(_LIBCPP_GLIBC_PREREQ)
 #    if _LIBCPP_GLIBC_PREREQ(2, 36) && defined(__cpp_char8_t)
-#      undef _LIBCPP_HAS_NO_C8RTOMB_MBRTOC8
+#      define _LIBCPP_HAS_C8RTOMB_MBRTOC8 1
+#    else
+#      define _LIBCPP_HAS_C8RTOMB_MBRTOC8 0
 #    endif
+#  else
+#    define _LIBCPP_HAS_C8RTOMB_MBRTOC8 0
 #  endif
 
 // There are a handful of public standard library types that are intended to
@@ -1188,7 +1227,9 @@ typedef __char32_t char32_t;
 
 // Clang-18 has support for deducing this, but it does not set the FTM.
 #  if defined(__cpp_explicit_this_parameter) || (defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1800)
-#    define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
+#    define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER 1
+#  else
+#    define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER 0
 #  endif
 
 #endif // __cplusplus
diff --git a/libcxx/include/__configuration/availability.h b/libcxx/include/__configuration/availability.h
index b10f29590a2c98898d71f5c4a7e86e0112b24702..f42ff460db45448565d484bf714db5bd1c05ded9 100644
--- a/libcxx/include/__configuration/availability.h
+++ b/libcxx/include/__configuration/availability.h
@@ -409,10 +409,10 @@
 #define _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19
 #define _LIBCPP_AVAILABILITY_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE
 
-// Define availability attributes that depend on _LIBCPP_HAS_NO_EXCEPTIONS.
+// Define availability attributes that depend on _LIBCPP_HAS_EXCEPTIONS.
 // Those are defined in terms of the availability attributes above, and
 // should not be vendor-specific.
-#if defined(_LIBCPP_HAS_NO_EXCEPTIONS)
+#if !_LIBCPP_HAS_EXCEPTIONS
 #  define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST
 #  define _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS
 #  define _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
@@ -423,8 +423,8 @@
 #endif
 
 // Define availability attributes that depend on both
-// _LIBCPP_HAS_NO_EXCEPTIONS and _LIBCPP_HAS_NO_RTTI.
-#if defined(_LIBCPP_HAS_NO_EXCEPTIONS) || defined(_LIBCPP_HAS_NO_RTTI)
+// _LIBCPP_HAS_EXCEPTIONS and _LIBCPP_HAS_RTTI.
+#if !_LIBCPP_HAS_EXCEPTIONS || !_LIBCPP_HAS_RTTI
 #  undef _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
 #  undef _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION
 #  define _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION 0
diff --git a/libcxx/include/__configuration/language.h b/libcxx/include/__configuration/language.h
index fa62a7b6f5c2a1f2cad2e8f1098ac9456700d0a3..9c224dfa76e4074da2d1d476e3e1038eba3ecd70 100644
--- a/libcxx/include/__configuration/language.h
+++ b/libcxx/include/__configuration/language.h
@@ -35,12 +35,16 @@
 #endif // __cplusplus
 // NOLINTEND(libcpp-cpp-version-check)
 
-#if !defined(__cpp_rtti) || __cpp_rtti < 199711L
-#  define _LIBCPP_HAS_NO_RTTI
+#if defined(__cpp_rtti) && __cpp_rtti >= 199711L
+#  define _LIBCPP_HAS_RTTI 1
+#else
+#  define _LIBCPP_HAS_RTTI 0
 #endif
 
-#if !defined(__cpp_exceptions) || __cpp_exceptions < 199711L
-#  define _LIBCPP_HAS_NO_EXCEPTIONS
+#if defined(__cpp_exceptions) && __cpp_exceptions >= 199711L
+#  define _LIBCPP_HAS_EXCEPTIONS 1
+#else
+#  define _LIBCPP_HAS_EXCEPTIONS 0
 #endif
 
 #endif // _LIBCPP___CONFIGURATION_LANGUAGE_H
diff --git a/libcxx/include/__debug_utils/sanitizers.h b/libcxx/include/__debug_utils/sanitizers.h
index d8547e32493303accd703d90aafbf2d1c8cce0dd..73d192711eabb66224c9c4e9691015e2e0b567ab 100644
--- a/libcxx/include/__debug_utils/sanitizers.h
+++ b/libcxx/include/__debug_utils/sanitizers.h
@@ -17,7 +17,7 @@
 #  pragma GCC system_header
 #endif
 
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
 
 extern "C" {
 _LIBCPP_EXPORTED_FROM_ABI void
@@ -28,12 +28,12 @@ _LIBCPP_EXPORTED_FROM_ABI int
 __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, const void*, const void*);
 }
 
-#endif // _LIBCPP_HAS_NO_ASAN
+#endif // _LIBCPP_HAS_ASAN
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 // ASan choices
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
 #  define _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS 1
 #endif
 
@@ -57,7 +57,7 @@ _LIBCPP_HIDE_FROM_ABI void __annotate_double_ended_contiguous_container(
     const void* __last_old_contained,
     const void* __first_new_contained,
     const void* __last_new_contained) {
-#ifdef _LIBCPP_HAS_NO_ASAN
+#if !_LIBCPP_HAS_ASAN
   (void)__first_storage;
   (void)__last_storage;
   (void)__first_old_contained;
@@ -86,7 +86,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __annotate_contiguous_c
     const void* __last_storage,
     const void* __old_last_contained,
     const void* __new_last_contained) {
-#ifdef _LIBCPP_HAS_NO_ASAN
+#if !_LIBCPP_HAS_ASAN
   (void)__first_storage;
   (void)__last_storage;
   (void)__old_last_contained;
diff --git a/libcxx/include/__exception/exception_ptr.h b/libcxx/include/__exception/exception_ptr.h
index 9e5351f534a1c18b6764bc954ec96833fac16f93..2cb250dfd42da7a7830b96388b3c1555f6f6748a 100644
--- a/libcxx/include/__exception/exception_ptr.h
+++ b/libcxx/include/__exception/exception_ptr.h
@@ -92,7 +92,7 @@ public:
 
 template <class _Ep>
 _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION && __cplusplus >= 201103L
   using _Ep2 = __decay_t<_Ep>;
 
diff --git a/libcxx/include/__exception/nested_exception.h b/libcxx/include/__exception/nested_exception.h
index 8e817e1c069788ac7949b1004cf0d70264ce0daf..cc20b038c871ba85f289e3c27f93d1c3483ae553 100644
--- a/libcxx/include/__exception/nested_exception.h
+++ b/libcxx/include/__exception/nested_exception.h
@@ -49,7 +49,7 @@ struct __nested : public _Tp, public nested_exception {
   _LIBCPP_HIDE_FROM_ABI explicit __nested(const _Tp& __t) : _Tp(__t) {}
 };
 
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
 template <class _Tp, class _Up, bool>
 struct __throw_with_nested;
 
@@ -68,7 +68,7 @@ struct __throw_with_nested<_Tp, _Up, false> {
 
 template <class _Tp>
 [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI void throw_with_nested(_Tp&& __t) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   using _Up = __decay_t<_Tp>;
   static_assert(is_copy_constructible<_Up>::value, "type thrown must be CopyConstructible");
   __throw_with_nested<_Tp,
diff --git a/libcxx/include/__expected/expected.h b/libcxx/include/__expected/expected.h
index e04e17d1ebe698ee342000c4ad6c91daa2ed38b9..3d3f11967ee746627f269c31685fc2214248288c 100644
--- a/libcxx/include/__expected/expected.h
+++ b/libcxx/include/__expected/expected.h
@@ -73,7 +73,7 @@ struct __expected_construct_unexpected_from_invoke_tag {};
 
 template <class _Err, class _Arg>
 _LIBCPP_HIDE_FROM_ABI void __throw_bad_expected_access(_Arg&& __arg) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_expected_access<_Err>(std::forward<_Arg>(__arg));
 #  else
   (void)__arg;
diff --git a/libcxx/include/__filesystem/filesystem_error.h b/libcxx/include/__filesystem/filesystem_error.h
index f43568c2004d21dfb6f4c92b02afd9958335bd64..73592bba31da0a89cd35edd532986efeb03d6259 100644
--- a/libcxx/include/__filesystem/filesystem_error.h
+++ b/libcxx/include/__filesystem/filesystem_error.h
@@ -67,7 +67,7 @@ private:
   shared_ptr<_Storage> __storage_;
 };
 
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
 template <class... _Args>
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY void
 __throw_filesystem_error(_Args&&... __args) {
diff --git a/libcxx/include/__filesystem/path.h b/libcxx/include/__filesystem/path.h
index f0d39680430ec54f5cd3198024a4372a4fc49178..2eb60810af02b5a7c35ff2642afac622a8063fe3 100644
--- a/libcxx/include/__filesystem/path.h
+++ b/libcxx/include/__filesystem/path.h
@@ -60,7 +60,7 @@ struct __can_convert_char<wchar_t> {
   static const bool value = true;
   using __char_type       = wchar_t;
 };
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 template <>
 struct __can_convert_char<char8_t> {
   static const bool value = true;
@@ -87,7 +87,7 @@ _LIBCPP_HIDE_FROM_ABI bool __is_separator(_ECharT __e) {
 #  endif
 }
 
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 typedef u8string __u8_string;
 #  else
 typedef string __u8_string;
@@ -366,7 +366,7 @@ struct _PathExport<char16_t> {
   }
 };
 
-#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
 template <>
 struct _PathExport<char8_t> {
   typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
@@ -376,7 +376,7 @@ struct _PathExport<char8_t> {
     _Narrower()(back_inserter(__dest), __src.data(), __src.data() + __src.size());
   }
 };
-#    endif /* !_LIBCPP_HAS_NO_CHAR8_T */
+#    endif // _LIBCPP_HAS_CHAR8_T
 #  endif   /* _LIBCPP_WIN32API */
 
 class _LIBCPP_EXPORTED_FROM_ABI path {
@@ -730,7 +730,7 @@ public:
 #  else    /* _LIBCPP_WIN32API */
 
   _LIBCPP_HIDE_FROM_ABI std::string string() const { return __pn_; }
-#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
   _LIBCPP_HIDE_FROM_ABI std::u8string u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); }
 #    else
   _LIBCPP_HIDE_FROM_ABI std::string u8string() const { return __pn_; }
@@ -756,7 +756,7 @@ public:
 
   // generic format observers
   _LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return __pn_; }
-#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
   _LIBCPP_HIDE_FROM_ABI std::u8string generic_u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); }
 #    else
   _LIBCPP_HIDE_FROM_ABI std::string generic_u8string() const { return __pn_; }
diff --git a/libcxx/include/__filesystem/u8path.h b/libcxx/include/__filesystem/u8path.h
index dae5823128f028a0fa3828a93b11d80d4413fa56..e13980298d9e99611d87783e99368aef18c5d3b1 100644
--- a/libcxx/include/__filesystem/u8path.h
+++ b/libcxx/include/__filesystem/u8path.h
@@ -34,7 +34,7 @@ _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
 template <class _InputIt, __enable_if_t<__is_pathable<_InputIt>::value, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _InputIt __l) {
   static_assert(
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
       is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
 #  endif
           is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
@@ -56,7 +56,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f,
 template <class _InputIt, __enable_if_t<__is_pathable<_InputIt>::value, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _NullSentinel) {
   static_assert(
-#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
       is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
 #    endif
           is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
@@ -77,7 +77,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f,
 template <class _Source, __enable_if_t<__is_pathable<_Source>::value, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(const _Source& __s) {
   static_assert(
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
       is_same<typename __is_pathable<_Source>::__char_type, char8_t>::value ||
 #  endif
           is_same<typename __is_pathable<_Source>::__char_type, char>::value,
diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h
index d1ce055874413eb6244bdea3e5e098e7b494e2a3..3f39d2aec81f7567cef0b3ca50331c1aba926bf8 100644
--- a/libcxx/include/__format/format_arg.h
+++ b/libcxx/include/__format/format_arg.h
@@ -113,7 +113,7 @@ _LIBCPP_HIDE_FROM_ABI decltype(auto) __visit_format_arg(_Visitor&& __vis, basic_
   case __format::__arg_t::__long_long:
     return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__long_long_);
   case __format::__arg_t::__i128:
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
     return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__i128_);
 #  else
     __libcpp_unreachable();
@@ -123,7 +123,7 @@ _LIBCPP_HIDE_FROM_ABI decltype(auto) __visit_format_arg(_Visitor&& __vis, basic_
   case __format::__arg_t::__unsigned_long_long:
     return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__unsigned_long_long_);
   case __format::__arg_t::__u128:
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
     return std::invoke(std::forward<_Visitor>(__vis), __arg.__value_.__u128_);
 #  else
     __libcpp_unreachable();
@@ -148,7 +148,7 @@ _LIBCPP_HIDE_FROM_ABI decltype(auto) __visit_format_arg(_Visitor&& __vis, basic_
   __libcpp_unreachable();
 }
 
-#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
 
 template <class _Rp, class _Visitor, class _Context>
 _LIBCPP_HIDE_FROM_ABI _Rp __visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
@@ -164,7 +164,7 @@ _LIBCPP_HIDE_FROM_ABI _Rp __visit_format_arg(_Visitor&& __vis, basic_format_arg<
   case __format::__arg_t::__long_long:
     return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__long_long_);
   case __format::__arg_t::__i128:
-#    ifndef _LIBCPP_HAS_NO_INT128
+#    if _LIBCPP_HAS_INT128
     return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__i128_);
 #    else
     __libcpp_unreachable();
@@ -174,7 +174,7 @@ _LIBCPP_HIDE_FROM_ABI _Rp __visit_format_arg(_Visitor&& __vis, basic_format_arg<
   case __format::__arg_t::__unsigned_long_long:
     return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__unsigned_long_long_);
   case __format::__arg_t::__u128:
-#    ifndef _LIBCPP_HAS_NO_INT128
+#    if _LIBCPP_HAS_INT128
     return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), __arg.__value_.__u128_);
 #    else
     __libcpp_unreachable();
@@ -199,7 +199,7 @@ _LIBCPP_HIDE_FROM_ABI _Rp __visit_format_arg(_Visitor&& __vis, basic_format_arg<
   __libcpp_unreachable();
 }
 
-#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
 
 /// Contains the values used in basic_format_arg.
 ///
@@ -237,7 +237,7 @@ public:
     unsigned __unsigned_;
     long long __long_long_;
     unsigned long long __unsigned_long_long_;
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
     __int128_t __i128_;
     __uint128_t __u128_;
 #  endif
@@ -261,7 +261,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(long long __value) noexcept : __long_long_(__value) {}
   _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(unsigned long long __value) noexcept
       : __unsigned_long_long_(__value) {}
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(__int128_t __value) noexcept : __i128_(__value) {}
   _LIBCPP_HIDE_FROM_ABI __basic_format_arg_value(__uint128_t __value) noexcept : __u128_(__value) {}
 #  endif
@@ -284,14 +284,14 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI explicit operator bool() const noexcept { return __type_ != __format::__arg_t::__none; }
 
-#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
 
   // This function is user facing, so it must wrap the non-standard types of
   // the "variant" in a handle to stay conforming. See __arg_t for more details.
   template <class _Visitor>
   _LIBCPP_HIDE_FROM_ABI decltype(auto) visit(this basic_format_arg __arg, _Visitor&& __vis) {
     switch (__arg.__type_) {
-#    ifndef _LIBCPP_HAS_NO_INT128
+#    if _LIBCPP_HAS_INT128
     case __format::__arg_t::__i128: {
       typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__i128_};
       return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
@@ -312,7 +312,7 @@ public:
   template <class _Rp, class _Visitor>
   _LIBCPP_HIDE_FROM_ABI _Rp visit(this basic_format_arg __arg, _Visitor&& __vis) {
     switch (__arg.__type_) {
-#    ifndef _LIBCPP_HAS_NO_INT128
+#    if _LIBCPP_HAS_INT128
     case __format::__arg_t::__i128: {
       typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__i128_};
       return std::invoke_r<_Rp>(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
@@ -328,7 +328,7 @@ public:
     }
   }
 
-#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
 
 private:
   using char_type = typename _Context::char_type;
@@ -370,13 +370,13 @@ private:
 // This function is user facing, so it must wrap the non-standard types of
 // the "variant" in a handle to stay conforming. See __arg_t for more details.
 template <class _Visitor, class _Context>
-#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
 _LIBCPP_DEPRECATED_IN_CXX26
 #  endif
     _LIBCPP_HIDE_FROM_ABI decltype(auto)
     visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
   switch (__arg.__type_) {
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   case __format::__arg_t::__i128: {
     typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__i128_};
     return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
@@ -386,7 +386,7 @@ _LIBCPP_DEPRECATED_IN_CXX26
     typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__u128_};
     return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
   }
-#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
   default:
     return std::__visit_format_arg(std::forward<_Visitor>(__vis), __arg);
   }
diff --git a/libcxx/include/__format/format_arg_store.h b/libcxx/include/__format/format_arg_store.h
index 9a4a41df6d4e684c41775af752a3befc6822e1ea..847b39d428515e4d877548bdcbceaf36e487dc44 100644
--- a/libcxx/include/__format/format_arg_store.h
+++ b/libcxx/include/__format/format_arg_store.h
@@ -64,7 +64,7 @@ consteval __arg_t __determine_arg_t() {
     return __arg_t::__int;
   else if constexpr (sizeof(_Tp) <= sizeof(long long))
     return __arg_t::__long_long;
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   else if constexpr (sizeof(_Tp) == sizeof(__int128_t))
     return __arg_t::__i128;
 #  endif
@@ -79,7 +79,7 @@ consteval __arg_t __determine_arg_t() {
     return __arg_t::__unsigned;
   else if constexpr (sizeof(_Tp) <= sizeof(unsigned long long))
     return __arg_t::__unsigned_long_long;
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   else if constexpr (sizeof(_Tp) == sizeof(__uint128_t))
     return __arg_t::__u128;
 #  endif
diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h
index a9be17b855837d7fcf64c26d4418d867aa6a7d05..019a71011a71405d633e9eec341ede28226c4da2 100644
--- a/libcxx/include/__format/format_context.h
+++ b/libcxx/include/__format/format_context.h
@@ -181,13 +181,13 @@ public:
                   __format::__determine_arg_t<basic_format_context, decltype(__arg)>(),
                   __basic_format_arg_value<basic_format_context>(__arg)};
           };
-#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
           return static_cast<_Context*>(__c)->arg(__id).visit(std::move(__visitor));
 #  else
           _LIBCPP_SUPPRESS_DEPRECATED_PUSH
           return std::visit_format_arg(std::move(__visitor), static_cast<_Context*>(__c)->arg(__id));
           _LIBCPP_SUPPRESS_DEPRECATED_POP
-#  endif // _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
         }) {
   }
 
diff --git a/libcxx/include/__format/format_error.h b/libcxx/include/__format/format_error.h
index 1df7dbff2b7df2be94dfaf85facb95e3afb0ae9e..b92e6d1de00e26628809fd07a4573a224582f8b9 100644
--- a/libcxx/include/__format/format_error.h
+++ b/libcxx/include/__format/format_error.h
@@ -36,7 +36,7 @@ public:
 _LIBCPP_DIAGNOSTIC_POP
 
 [[noreturn]] inline _LIBCPP_HIDE_FROM_ABI void __throw_format_error(const char* __s) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw format_error(__s);
 #  else
   _LIBCPP_VERBOSE_ABORT("format_error was thrown in -fno-exceptions mode with message \"%s\"", __s);
diff --git a/libcxx/include/__format/format_functions.h b/libcxx/include/__format/format_functions.h
index 1518ab5768d24316c9a78b533f0b91fdd1b41797..b3cda8bf84491898f99a9efa898a2a67b279746b 100644
--- a/libcxx/include/__format/format_functions.h
+++ b/libcxx/include/__format/format_functions.h
@@ -206,7 +206,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_visit_format_arg(
   case __arg_t::__long_long:
     return __format::__compile_time_validate_argument<_CharT, long long>(__parse_ctx, __ctx);
   case __arg_t::__i128:
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
     return __format::__compile_time_validate_argument<_CharT, __int128_t>(__parse_ctx, __ctx);
 #  else
     std::__throw_format_error("Invalid argument");
@@ -217,7 +217,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __compile_time_visit_format_arg(
   case __arg_t::__unsigned_long_long:
     return __format::__compile_time_validate_argument<_CharT, unsigned long long>(__parse_ctx, __ctx);
   case __arg_t::__u128:
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
     return __format::__compile_time_validate_argument<_CharT, __uint128_t>(__parse_ctx, __ctx);
 #  else
     std::__throw_format_error("Invalid argument");
diff --git a/libcxx/include/__format/formatter_integer.h b/libcxx/include/__format/formatter_integer.h
index b7f46014c572313e556a5011d6016868d27958c3..3f51b10d75aacaa677298c3520c7f48a1c892da0 100644
--- a/libcxx/include/__format/formatter_integer.h
+++ b/libcxx/include/__format/formatter_integer.h
@@ -67,7 +67,7 @@ template <__fmt_char_type _CharT>
 struct _LIBCPP_TEMPLATE_VIS formatter<long, _CharT> : public __formatter_integer<_CharT> {};
 template <__fmt_char_type _CharT>
 struct _LIBCPP_TEMPLATE_VIS formatter<long long, _CharT> : public __formatter_integer<_CharT> {};
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 template <__fmt_char_type _CharT>
 struct _LIBCPP_TEMPLATE_VIS formatter<__int128_t, _CharT> : public __formatter_integer<_CharT> {};
 #  endif
@@ -83,7 +83,7 @@ template <__fmt_char_type _CharT>
 struct _LIBCPP_TEMPLATE_VIS formatter<unsigned long, _CharT> : public __formatter_integer<_CharT> {};
 template <__fmt_char_type _CharT>
 struct _LIBCPP_TEMPLATE_VIS formatter<unsigned long long, _CharT> : public __formatter_integer<_CharT> {};
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 template <__fmt_char_type _CharT>
 struct _LIBCPP_TEMPLATE_VIS formatter<__uint128_t, _CharT> : public __formatter_integer<_CharT> {};
 #  endif
@@ -99,7 +99,7 @@ template <>
 inline constexpr bool enable_nonlocking_formatter_optimization<long> = true;
 template <>
 inline constexpr bool enable_nonlocking_formatter_optimization<long long> = true;
-#    ifndef _LIBCPP_HAS_NO_INT128
+#    if _LIBCPP_HAS_INT128
 template <>
 inline constexpr bool enable_nonlocking_formatter_optimization<__int128_t> = true;
 #    endif
@@ -114,7 +114,7 @@ template <>
 inline constexpr bool enable_nonlocking_formatter_optimization<unsigned long> = true;
 template <>
 inline constexpr bool enable_nonlocking_formatter_optimization<unsigned long long> = true;
-#    ifndef _LIBCPP_HAS_NO_INT128
+#    if _LIBCPP_HAS_INT128
 template <>
 inline constexpr bool enable_nonlocking_formatter_optimization<__uint128_t> = true;
 #    endif
diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h
index 0d95c9a6ef2bbd4f8085c4807630f28958f6a929..e3844f86dbd3f55e9643bac1e08f1f0f3d694179 100644
--- a/libcxx/include/__functional/function.h
+++ b/libcxx/include/__functional/function.h
@@ -79,7 +79,7 @@ public:
 _LIBCPP_DIAGNOSTIC_POP
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_function_call() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_function_call();
 #  else
   _LIBCPP_VERBOSE_ABORT("bad_function_call was thrown in -fno-exceptions mode");
@@ -123,7 +123,7 @@ _LIBCPP_HIDE_FROM_ABI bool __not_null(function<_Fp> const& __f) {
   return !!__f;
 }
 
-#  ifdef _LIBCPP_HAS_EXTENSION_BLOCKS
+#  if _LIBCPP_HAS_EXTENSION_BLOCKS
 template <class _Rp, class... _Args>
 _LIBCPP_HIDE_FROM_ABI bool __not_null(_Rp (^__p)(_Args...)) {
   return __p;
@@ -244,10 +244,10 @@ public:
   virtual void destroy() _NOEXCEPT            = 0;
   virtual void destroy_deallocate() _NOEXCEPT = 0;
   virtual _Rp operator()(_ArgTypes&&...)      = 0;
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
   virtual const void* target(const type_info&) const _NOEXCEPT = 0;
   virtual const std::type_info& target_type() const _NOEXCEPT  = 0;
-#  endif // _LIBCPP_HAS_NO_RTTI
+#  endif // _LIBCPP_HAS_RTTI
 };
 
 // __func implements __base for a given functor type.
@@ -273,10 +273,10 @@ public:
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy_deallocate() _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual _Rp operator()(_ArgTypes&&... __arg);
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const void* target(const type_info&) const _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const std::type_info& target_type() const _NOEXCEPT;
-#  endif // _LIBCPP_HAS_NO_RTTI
+#  endif // _LIBCPP_HAS_RTTI
 };
 
 template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
@@ -314,7 +314,7 @@ _Rp __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&&... __arg) {
   return __f_(std::forward<_ArgTypes>(__arg)...);
 }
 
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
 
 template <class _Fp, class _Alloc, class _Rp, class... _ArgTypes>
 const void* __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT {
@@ -328,7 +328,7 @@ const std::type_info& __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() cons
   return typeid(_Fp);
 }
 
-#  endif // _LIBCPP_HAS_NO_RTTI
+#  endif // _LIBCPP_HAS_RTTI
 
 // __value_func creates a value-type from a __func.
 
@@ -465,7 +465,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __f_ != nullptr; }
 
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
   _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT {
     if (__f_ == nullptr)
       return typeid(void);
@@ -478,7 +478,7 @@ public:
       return nullptr;
     return (const _Tp*)__f_->target(typeid(_Tp));
   }
-#  endif // _LIBCPP_HAS_NO_RTTI
+#  endif // _LIBCPP_HAS_RTTI
 };
 
 // Storage for a functor object, to be used with __policy to manage copy and
@@ -521,7 +521,7 @@ struct __policy {
         nullptr,
         nullptr,
         true,
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
         &typeid(void)
 #  else
         nullptr
@@ -548,7 +548,7 @@ private:
         &__large_clone<_Fun>,
         &__large_destroy<_Fun>,
         false,
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
         &typeid(typename _Fun::_Target)
 #  else
         nullptr
@@ -563,7 +563,7 @@ private:
         nullptr,
         nullptr,
         false,
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
         &typeid(typename _Fun::_Target)
 #  else
         nullptr
@@ -725,7 +725,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return !__policy_->__is_null; }
 
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
   _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT { return *__policy_->__type_info; }
 
   template <typename _Tp>
@@ -737,10 +737,10 @@ public:
     else
       return reinterpret_cast<const _Tp*>(&__buf_.__small);
   }
-#  endif // _LIBCPP_HAS_NO_RTTI
+#  endif // _LIBCPP_HAS_RTTI
 };
 
-#  if defined(_LIBCPP_HAS_BLOCKS_RUNTIME)
+#  if _LIBCPP_HAS_BLOCKS_RUNTIME
 
 extern "C" void* _Block_copy(const void*);
 extern "C" void _Block_release(const void*);
@@ -752,7 +752,7 @@ class __func<_Rp1 (^)(_ArgTypes1...), _Alloc, _Rp(_ArgTypes...)> : public __base
 
 public:
   _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type const& __f)
-#    ifdef _LIBCPP_HAS_OBJC_ARC
+#    if _LIBCPP_HAS_OBJC_ARC
       : __f_(__f)
 #    else
       : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
@@ -763,7 +763,7 @@ public:
   // [TODO] add && to save on a retain
 
   _LIBCPP_HIDE_FROM_ABI explicit __func(__block_type __f, const _Alloc& /* unused */)
-#    ifdef _LIBCPP_HAS_OBJC_ARC
+#    if _LIBCPP_HAS_OBJC_ARC
       : __f_(__f)
 #    else
       : __f_(reinterpret_cast<__block_type>(__f ? _Block_copy(__f) : nullptr))
@@ -785,7 +785,7 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual void destroy() _NOEXCEPT {
-#    ifndef _LIBCPP_HAS_OBJC_ARC
+#    if !_LIBCPP_HAS_OBJC_ARC
     if (__f_)
       _Block_release(__f_);
 #    endif
@@ -804,7 +804,7 @@ public:
     return std::__invoke(__f_, std::forward<_ArgTypes>(__arg)...);
   }
 
-#    ifndef _LIBCPP_HAS_NO_RTTI
+#    if _LIBCPP_HAS_RTTI
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const void* target(type_info const& __ti) const _NOEXCEPT {
     if (__ti == typeid(__func::__block_type))
       return &__f_;
@@ -814,7 +814,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual const std::type_info& target_type() const _NOEXCEPT {
     return typeid(__func::__block_type);
   }
-#    endif // _LIBCPP_HAS_NO_RTTI
+#    endif // _LIBCPP_HAS_RTTI
 };
 
 #  endif // _LIBCPP_HAS_EXTENSION_BLOCKS
@@ -906,14 +906,14 @@ public:
   // function invocation:
   _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes...) const;
 
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
   // function target access:
   _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT;
   template <typename _Tp>
   _LIBCPP_HIDE_FROM_ABI _Tp* target() _NOEXCEPT;
   template <typename _Tp>
   _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT;
-#  endif // _LIBCPP_HAS_NO_RTTI
+#  endif // _LIBCPP_HAS_RTTI
 };
 
 #  if _LIBCPP_STD_VER >= 17
@@ -990,7 +990,7 @@ _Rp function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const {
   return __f_(std::forward<_ArgTypes>(__arg)...);
 }
 
-#  ifndef _LIBCPP_HAS_NO_RTTI
+#  if _LIBCPP_HAS_RTTI
 
 template <class _Rp, class... _ArgTypes>
 const std::type_info& function<_Rp(_ArgTypes...)>::target_type() const _NOEXCEPT {
@@ -1009,7 +1009,7 @@ const _Tp* function<_Rp(_ArgTypes...)>::target() const _NOEXCEPT {
   return __f_.template target<_Tp>();
 }
 
-#  endif // _LIBCPP_HAS_NO_RTTI
+#  endif // _LIBCPP_HAS_RTTI
 
 template <class _Rp, class... _ArgTypes>
 inline _LIBCPP_HIDE_FROM_ABI bool operator==(const function<_Rp(_ArgTypes...)>& __f, nullptr_t) _NOEXCEPT {
diff --git a/libcxx/include/__functional/hash.h b/libcxx/include/__functional/hash.h
index 59dce7ac9cc03197d0baa03f32269607bf1c105b..8abec99f7290571e2295b5994de726f227ed4661 100644
--- a/libcxx/include/__functional/hash.h
+++ b/libcxx/include/__functional/hash.h
@@ -356,12 +356,12 @@ struct _LIBCPP_TEMPLATE_VIS hash<unsigned char> : public __unary_function<unsign
   _LIBCPP_HIDE_FROM_ABI size_t operator()(unsigned char __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
 };
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 template <>
 struct _LIBCPP_TEMPLATE_VIS hash<char8_t> : public __unary_function<char8_t, size_t> {
   _LIBCPP_HIDE_FROM_ABI size_t operator()(char8_t __v) const _NOEXCEPT { return static_cast<size_t>(__v); }
 };
-#endif // !_LIBCPP_HAS_NO_CHAR8_T
+#endif // _LIBCPP_HAS_CHAR8_T
 
 template <>
 struct _LIBCPP_TEMPLATE_VIS hash<char16_t> : public __unary_function<char16_t, size_t> {
@@ -416,7 +416,7 @@ struct _LIBCPP_TEMPLATE_VIS hash<long long> : public __scalar_hash<long long> {}
 template <>
 struct _LIBCPP_TEMPLATE_VIS hash<unsigned long long> : public __scalar_hash<unsigned long long> {};
 
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 
 template <>
 struct _LIBCPP_TEMPLATE_VIS hash<__int128_t> : public __scalar_hash<__int128_t> {};
diff --git a/libcxx/include/__fwd/string.h b/libcxx/include/__fwd/string.h
index 2418e1f9b23d0d92947edb9676f96fbfa7b72c24..f13a3594022e8be0f9b15d82387aecbf01dd42c8 100644
--- a/libcxx/include/__fwd/string.h
+++ b/libcxx/include/__fwd/string.h
@@ -24,7 +24,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits;
 template <>
 struct char_traits<char>;
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 template <>
 struct char_traits<char8_t>;
 #endif
@@ -48,7 +48,7 @@ using string = basic_string<char>;
 using wstring = basic_string<wchar_t>;
 #endif
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 using u8string = basic_string<char8_t>;
 #endif
 
@@ -67,7 +67,7 @@ using string _LIBCPP_AVAILABILITY_PMR = basic_string<char>;
 using wstring _LIBCPP_AVAILABILITY_PMR = basic_string<wchar_t>;
 #  endif
 
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 using u8string _LIBCPP_AVAILABILITY_PMR = basic_string<char8_t>;
 #  endif
 
@@ -83,7 +83,7 @@ class _LIBCPP_PREFERRED_NAME(string)
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
       _LIBCPP_PREFERRED_NAME(wstring)
 #endif
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
       _LIBCPP_PREFERRED_NAME(u8string)
 #endif
       _LIBCPP_PREFERRED_NAME(u16string)
@@ -93,7 +93,7 @@ class _LIBCPP_PREFERRED_NAME(string)
 #  ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
       _LIBCPP_PREFERRED_NAME(pmr::wstring)
 #  endif
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
       _LIBCPP_PREFERRED_NAME(pmr::u8string)
 #  endif
       _LIBCPP_PREFERRED_NAME(pmr::u16string)
diff --git a/libcxx/include/__fwd/string_view.h b/libcxx/include/__fwd/string_view.h
index 72a64be5b00b54bf02026d8cb708aa8b0a724ce1..e1acd8742561100d648394ee84637f34517077a5 100644
--- a/libcxx/include/__fwd/string_view.h
+++ b/libcxx/include/__fwd/string_view.h
@@ -23,7 +23,7 @@ template <class _CharT, class _Traits = char_traits<_CharT> >
 class _LIBCPP_TEMPLATE_VIS basic_string_view;
 
 typedef basic_string_view<char> string_view;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 typedef basic_string_view<char8_t> u8string_view;
 #endif
 typedef basic_string_view<char16_t> u16string_view;
@@ -38,7 +38,7 @@ class _LIBCPP_PREFERRED_NAME(string_view)
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
       _LIBCPP_PREFERRED_NAME(wstring_view)
 #endif
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
       _LIBCPP_PREFERRED_NAME(u8string_view)
 #endif
       _LIBCPP_PREFERRED_NAME(u16string_view)
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index 821d4edd8d87c0369a6d34822af402dff3e8e3fe..560e873adc3846b62cd6bc3036606aaddec2642b 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -1210,9 +1210,9 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u,
     max_load_factor() = __u.max_load_factor();
     if (bucket_count() != 0) {
       __next_pointer __cache = __detach();
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
       try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
         const_iterator __i = __u.begin();
         while (__cache != nullptr && __u.size() != 0) {
           __cache->__upcast()->__get_value() = std::move(__u.remove(__i++)->__get_value());
@@ -1220,12 +1220,12 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u,
           __node_insert_multi(__cache->__upcast());
           __cache = __next;
         }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         __deallocate_node(__cache);
         throw;
       }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       __deallocate_node(__cache);
     }
     const_iterator __i = __u.begin();
@@ -1256,21 +1256,21 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_unique(_InputIterator __
 
   if (bucket_count() != 0) {
     __next_pointer __cache = __detach();
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __cache != nullptr && __first != __last; ++__first) {
         __cache->__upcast()->__get_value() = *__first;
         __next_pointer __next              = __cache->__next_;
         __node_insert_unique(__cache->__upcast());
         __cache = __next;
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __deallocate_node(__cache);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __deallocate_node(__cache);
   }
   for (; __first != __last; ++__first)
@@ -1288,21 +1288,21 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_multi(_InputIterator __f
       " or the nodes value type");
   if (bucket_count() != 0) {
     __next_pointer __cache = __detach();
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __cache != nullptr && __first != __last; ++__first) {
         __cache->__upcast()->__get_value() = *__first;
         __next_pointer __next              = __cache->__next_;
         __node_insert_multi(__cache->__upcast());
         __cache = __next;
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __deallocate_node(__cache);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __deallocate_node(__cache);
   }
   for (; __first != __last; ++__first)
diff --git a/libcxx/include/__iterator/counted_iterator.h b/libcxx/include/__iterator/counted_iterator.h
index d461427ccea3ce83dbb1b35b45ddaaa895c4ab54..65e178bc0cf21c8e270489768efa0760181ee8af 100644
--- a/libcxx/include/__iterator/counted_iterator.h
+++ b/libcxx/include/__iterator/counted_iterator.h
@@ -132,7 +132,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator++(int) {
     _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end.");
     --__count_;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
       return __current_++;
     } catch (...) {
@@ -141,7 +141,7 @@ public:
     }
 #  else
     return __current_++;
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
   _LIBCPP_HIDE_FROM_ABI constexpr counted_iterator operator++(int)
diff --git a/libcxx/include/__locale b/libcxx/include/__locale
index 4b382764b446453e6db5bfce70d9f663e0049795..5fb0c19903cd5be5ba9f8a03a3c486fbff0b57c5 100644
--- a/libcxx/include/__locale
+++ b/libcxx/include/__locale
@@ -982,7 +982,7 @@ protected:
   virtual int do_max_length() const _NOEXCEPT;
 };
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t> // C++20
 
@@ -1145,7 +1145,7 @@ protected:
   virtual int do_max_length() const _NOEXCEPT;
 };
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t> // C++20
 
@@ -1255,7 +1255,7 @@ extern template class _LIBCPP_DEPRECATED_IN_CXX20
 _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char16_t, char, mbstate_t>; // deprecated in C++20
 extern template class _LIBCPP_DEPRECATED_IN_CXX20
 _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char32_t, char, mbstate_t>; // deprecated in C++20
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char16_t, char8_t, mbstate_t>; // C++20
 extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char32_t, char8_t, mbstate_t>; // C++20
 #endif
diff --git a/libcxx/include/__memory/addressof.h b/libcxx/include/__memory/addressof.h
index fa590212c49b96effe9ac549dacf617b716b7134..ecb68e0fe61e42bebaa3253e77448b7794b3834d 100644
--- a/libcxx/include/__memory/addressof.h
+++ b/libcxx/include/__memory/addressof.h
@@ -23,7 +23,7 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_NO_CFI _LIBCPP_HIDE_FROM_ABI _Tp* a
   return __builtin_addressof(__x);
 }
 
-#if defined(_LIBCPP_HAS_OBJC_ARC) && !defined(_LIBCPP_PREDEFINED_OBJC_ARC_ADDRESSOF)
+#if _LIBCPP_HAS_OBJC_ARC && !defined(_LIBCPP_PREDEFINED_OBJC_ARC_ADDRESSOF)
 // Objective-C++ Automatic Reference Counting uses qualified pointers
 // that require special addressof() signatures. When
 // _LIBCPP_PREDEFINED_OBJC_ARC_ADDRESSOF is defined, the compiler
@@ -33,7 +33,7 @@ inline _LIBCPP_HIDE_FROM_ABI __strong _Tp* addressof(__strong _Tp& __x) _NOEXCEP
   return &__x;
 }
 
-#  ifdef _LIBCPP_HAS_OBJC_ARC_WEAK
+#  if _LIBCPP_HAS_OBJC_ARC_WEAK
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI __weak _Tp* addressof(__weak _Tp& __x) _NOEXCEPT {
   return &__x;
diff --git a/libcxx/include/__memory/aligned_alloc.h b/libcxx/include/__memory/aligned_alloc.h
index cb424328bcafc174ef7d3df0d8388d591a519e76..33fe8af77df7b51af9302fd5197c295478b04659 100644
--- a/libcxx/include/__memory/aligned_alloc.h
+++ b/libcxx/include/__memory/aligned_alloc.h
@@ -19,7 +19,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#ifndef _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 
 // Low-level helpers to call the aligned allocation and deallocation functions
 // on the target platform. This is used to implement libc++'s own memory
@@ -30,7 +30,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_aligned_alloc(std::size_t __alignment, std::size_t __size) {
 #  if defined(_LIBCPP_MSVCRT_LIKE)
   return ::_aligned_malloc(__size, __alignment);
-#  elif _LIBCPP_STD_VER >= 17 && !defined(_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC)
+#  elif _LIBCPP_STD_VER >= 17 && _LIBCPP_HAS_C11_ALIGNED_ALLOC
   // aligned_alloc() requires that __size is a multiple of __alignment,
   // but for C++ [new.delete.general], only states "if the value of an
   // alignment argument passed to any of these functions is not a valid
@@ -57,7 +57,7 @@ inline _LIBCPP_HIDE_FROM_ABI void __libcpp_aligned_free(void* __ptr) {
 #  endif
 }
 
-#endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#endif // _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 20c1b69f45ae6613d9252e2b7c6140ecc4737a18..65870ba574c25bc7d77cbb09cb2b0ba88387bf2e 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -54,7 +54,7 @@
 #include <cstddef>
 #include <new>
 #include <typeinfo>
-#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#if _LIBCPP_HAS_ATOMIC_HEADER
 #  include <__atomic/memory_order.h>
 #endif
 
@@ -70,10 +70,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
 // should be sufficient for thread safety.
 // See https://llvm.org/PR22803
-#if defined(__clang__) && __has_builtin(__atomic_add_fetch) && defined(__ATOMIC_RELAXED) && defined(__ATOMIC_ACQ_REL)
-#  define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
-#elif defined(_LIBCPP_COMPILER_GCC)
-#  define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT
+#if (defined(__clang__) && __has_builtin(__atomic_add_fetch) && defined(__ATOMIC_RELAXED) &&                           \
+     defined(__ATOMIC_ACQ_REL)) ||                                                                                     \
+    defined(_LIBCPP_COMPILER_GCC)
+#  define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT 1
+#else
+#  define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT 0
 #endif
 
 template <class _ValueType>
@@ -98,7 +100,7 @@ inline _LIBCPP_HIDE_FROM_ABI _ValueType __libcpp_acquire_load(_ValueType const*
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_increment(_Tp& __t) _NOEXCEPT {
-#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+#if _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT && !defined(_LIBCPP_HAS_NO_THREADS)
   return __atomic_add_fetch(&__t, 1, __ATOMIC_RELAXED);
 #else
   return __t += 1;
@@ -107,7 +109,7 @@ inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_increment(_Tp& __t) _N
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_decrement(_Tp& __t) _NOEXCEPT {
-#if defined(_LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT) && !defined(_LIBCPP_HAS_NO_THREADS)
+#if _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT && !defined(_LIBCPP_HAS_NO_THREADS)
   return __atomic_add_fetch(&__t, -1, __ATOMIC_ACQ_REL);
 #else
   return __t -= 1;
@@ -124,7 +126,7 @@ public:
 };
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_weak_ptr() {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw bad_weak_ptr();
 #else
   _LIBCPP_VERBOSE_ABORT("bad_weak_ptr was thrown in -fno-exceptions mode");
@@ -206,7 +208,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI __shared_ptr_pointer(_Tp __p, _Dp __d, _Alloc __a)
       : __ptr_(__p), __deleter_(std::move(__d)), __alloc_(std::move(__a)) {}
 
-#ifndef _LIBCPP_HAS_NO_RTTI
+#if _LIBCPP_HAS_RTTI
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL const void* __get_deleter(const type_info&) const _NOEXCEPT override;
 #endif
 
@@ -215,14 +217,14 @@ private:
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared_weak() _NOEXCEPT override;
 };
 
-#ifndef _LIBCPP_HAS_NO_RTTI
+#if _LIBCPP_HAS_RTTI
 
 template <class _Tp, class _Dp, class _Alloc>
 const void* __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__get_deleter(const type_info& __t) const _NOEXCEPT {
   return __t == typeid(_Dp) ? std::addressof(__deleter_) : nullptr;
 }
 
-#endif // _LIBCPP_HAS_NO_RTTI
+#endif // _LIBCPP_HAS_RTTI
 
 template <class _Tp, class _Dp, class _Alloc>
 void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared() _NOEXCEPT {
@@ -455,9 +457,9 @@ public:
 
   template <class _Yp, class _Dp, __enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI shared_ptr(_Yp* __p, _Dp __d) : __ptr_(__p) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
       typedef __shared_ptr_pointer<_Yp*, _Dp, _AllocT> _CntrlBlk;
 #ifndef _LIBCPP_CXX03_LANG
@@ -466,12 +468,12 @@ public:
     __cntrl_ = new _CntrlBlk(__p, __d, _AllocT());
 #endif // not _LIBCPP_CXX03_LANG
       __enable_weak_this(__p, __p);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __d(__p);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
   template <class _Yp,
@@ -479,9 +481,9 @@ public:
             class _Alloc,
             __enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI shared_ptr(_Yp* __p, _Dp __d, _Alloc __a) : __ptr_(__p) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       typedef __shared_ptr_pointer<_Yp*, _Dp, _Alloc> _CntrlBlk;
       typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
       typedef __allocator_destructor<_A2> _D2;
@@ -495,12 +497,12 @@ public:
 #endif // not _LIBCPP_CXX03_LANG
       __cntrl_ = std::addressof(*__hold2.release());
       __enable_weak_this(__p, __p);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __d(__p);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
   template <class _Dp>
@@ -509,9 +511,9 @@ public:
       _Dp __d,
       __enable_if_t<__shared_ptr_nullptr_deleter_ctor_reqs<_Dp>::value, __nullptr_sfinae_tag> = __nullptr_sfinae_tag())
       : __ptr_(nullptr) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       typedef typename __shared_ptr_default_allocator<_Tp>::type _AllocT;
       typedef __shared_ptr_pointer<nullptr_t, _Dp, _AllocT> _CntrlBlk;
 #ifndef _LIBCPP_CXX03_LANG
@@ -519,12 +521,12 @@ public:
 #else
     __cntrl_ = new _CntrlBlk(__p, __d, _AllocT());
 #endif // not _LIBCPP_CXX03_LANG
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __d(__p);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
   template <class _Dp, class _Alloc>
@@ -534,9 +536,9 @@ public:
       _Alloc __a,
       __enable_if_t<__shared_ptr_nullptr_deleter_ctor_reqs<_Dp>::value, __nullptr_sfinae_tag> = __nullptr_sfinae_tag())
       : __ptr_(nullptr) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       typedef __shared_ptr_pointer<nullptr_t, _Dp, _Alloc> _CntrlBlk;
       typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
       typedef __allocator_destructor<_A2> _D2;
@@ -549,12 +551,12 @@ public:
         _CntrlBlk(__p, __d, __a);
 #endif // not _LIBCPP_CXX03_LANG
       __cntrl_ = std::addressof(*__hold2.release());
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __d(__p);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
   template <class _Yp>
@@ -767,12 +769,12 @@ public:
   }
 #endif
 
-#ifndef _LIBCPP_HAS_NO_RTTI
+#if _LIBCPP_HAS_RTTI
   template <class _Dp>
   _LIBCPP_HIDE_FROM_ABI _Dp* __get_deleter() const _NOEXCEPT {
     return static_cast<_Dp*>(__cntrl_ ? const_cast<void*>(__cntrl_->__get_deleter(typeid(_Dp))) : nullptr);
   }
-#endif // _LIBCPP_HAS_NO_RTTI
+#endif // _LIBCPP_HAS_RTTI
 
   template <class _Yp, class _CntrlBlk>
   _LIBCPP_HIDE_FROM_ABI static shared_ptr<_Tp> __create_with_control_block(_Yp* __p, _CntrlBlk* __cntrl) _NOEXCEPT {
@@ -1297,14 +1299,14 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(shared_ptr<_Up>&&
 }
 #endif
 
-#ifndef _LIBCPP_HAS_NO_RTTI
+#if _LIBCPP_HAS_RTTI
 
 template <class _Dp, class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _Dp* get_deleter(const shared_ptr<_Tp>& __p) _NOEXCEPT {
   return __p.template __get_deleter<_Dp>();
 }
 
-#endif // _LIBCPP_HAS_NO_RTTI
+#endif // _LIBCPP_HAS_RTTI
 
 template <class _Tp>
 class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr {
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index dd72f3c10cf15a86184397c87d9ff061f8e86f7e..54af1fa1a1cc55dca9ea9fcdffed1bb0f8cf2f69 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -59,12 +59,12 @@ template <class _ValueType, class _InputIterator, class _Sentinel1, class _Forwa
 inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_copy(
     _InputIterator __ifirst, _Sentinel1 __ilast, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
   _ForwardIterator __idx = __ofirst;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
 #endif
     for (; __ifirst != __ilast && !__stop_copying(__idx); ++__ifirst, (void)++__idx)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(*__ifirst);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__ofirst, __idx);
     throw;
@@ -89,12 +89,12 @@ template <class _ValueType, class _InputIterator, class _Size, class _ForwardIte
 inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
 __uninitialized_copy_n(_InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
   _ForwardIterator __idx = __ofirst;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
 #endif
     for (; __n > 0 && !__stop_copying(__idx); ++__ifirst, (void)++__idx, (void)--__n)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(*__ifirst);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__ofirst, __idx);
     throw;
@@ -119,12 +119,12 @@ template <class _ValueType, class _ForwardIterator, class _Sentinel, class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
 __uninitialized_fill(_ForwardIterator __first, _Sentinel __last, const _Tp& __x) {
   _ForwardIterator __idx = __first;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
 #endif
     for (; __idx != __last; ++__idx)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__x);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
     throw;
@@ -147,12 +147,12 @@ template <class _ValueType, class _ForwardIterator, class _Size, class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
 __uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) {
   _ForwardIterator __idx = __first;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
 #endif
     for (; __n > 0; ++__idx, (void)--__n)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__x);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
     throw;
@@ -177,12 +177,12 @@ template <class _ValueType, class _ForwardIterator, class _Sentinel>
 inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
 __uninitialized_default_construct(_ForwardIterator __first, _Sentinel __last) {
   auto __idx = __first;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
 #  endif
     for (; __idx != __last; ++__idx)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
     throw;
@@ -203,12 +203,12 @@ inline _LIBCPP_HIDE_FROM_ABI void uninitialized_default_construct(_ForwardIterat
 template <class _ValueType, class _ForwardIterator, class _Size>
 inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) {
   auto __idx = __first;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
 #  endif
     for (; __n > 0; ++__idx, (void)--__n)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
     throw;
@@ -230,12 +230,12 @@ template <class _ValueType, class _ForwardIterator, class _Sentinel>
 inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
 __uninitialized_value_construct(_ForwardIterator __first, _Sentinel __last) {
   auto __idx = __first;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
 #  endif
     for (; __idx != __last; ++__idx)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
     throw;
@@ -256,12 +256,12 @@ inline _LIBCPP_HIDE_FROM_ABI void uninitialized_value_construct(_ForwardIterator
 template <class _ValueType, class _ForwardIterator, class _Size>
 inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) {
   auto __idx = __first;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
 #  endif
     for (; __n > 0; ++__idx, (void)--__n)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
     throw;
@@ -292,13 +292,13 @@ inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitiali
     _EndPredicate __stop_moving,
     _IterMove __iter_move) {
   auto __idx = __ofirst;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
 #  endif
     for (; __ifirst != __ilast && !__stop_moving(__idx); ++__idx, (void)++__ifirst) {
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__iter_move(__ifirst));
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__ofirst, __idx);
     throw;
@@ -330,12 +330,12 @@ template <class _ValueType,
 inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_move_n(
     _InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_moving, _IterMove __iter_move) {
   auto __idx = __ofirst;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
 #  endif
     for (; __n > 0 && !__stop_moving(__idx); ++__idx, (void)++__ifirst, --__n)
       ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__iter_move(__ifirst));
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     std::__destroy(__ofirst, __idx);
     throw;
@@ -627,7 +627,7 @@ __uninitialized_allocator_relocate(_Alloc& __alloc, _Tp* __first, _Tp* __last, _
         std::__make_exception_guard(_AllocatorDestroyRangeReverse<_Alloc, _Tp*>(__alloc, __destruct_first, __result));
     auto __iter = __first;
     while (__iter != __last) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
       allocator_traits<_Alloc>::construct(__alloc, __result, std::move_if_noexcept(*__iter));
 #else
       allocator_traits<_Alloc>::construct(__alloc, __result, std::move(*__iter));
diff --git a/libcxx/include/__memory/unique_temporary_buffer.h b/libcxx/include/__memory/unique_temporary_buffer.h
index 001254e8fa4be1422deba6e34331be01ada69b22..8a8c327be69fe72888b1d2b2f907e761672cb7c3 100644
--- a/libcxx/include/__memory/unique_temporary_buffer.h
+++ b/libcxx/include/__memory/unique_temporary_buffer.h
@@ -62,7 +62,7 @@ __allocate_unique_temporary_buffer(ptrdiff_t __count) {
   if (__count > __max_count)
     __count = __max_count;
   while (__count > 0) {
-#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+#if _LIBCPP_HAS_ALIGNED_ALLOCATION
     if (__is_overaligned_for_new(_LIBCPP_ALIGNOF(_Tp))) {
       align_val_t __al = align_val_t(_LIBCPP_ALIGNOF(_Tp));
       __ptr            = static_cast<_Tp*>(::operator new(__count * sizeof(_Tp), __al, nothrow));
diff --git a/libcxx/include/__mutex/mutex.h b/libcxx/include/__mutex/mutex.h
index 1ed01547126f48eae4446a325b93d8b12a113968..9c0c9ad6b5997eaf08f5e39018084ea6502dd109 100644
--- a/libcxx/include/__mutex/mutex.h
+++ b/libcxx/include/__mutex/mutex.h
@@ -30,7 +30,7 @@ public:
   mutex(const mutex&)            = delete;
   mutex& operator=(const mutex&) = delete;
 
-#  if defined(_LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION)
+#  if _LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION
   _LIBCPP_HIDE_FROM_ABI ~mutex() = default;
 #  else
   ~mutex() _NOEXCEPT;
diff --git a/libcxx/include/__ostream/basic_ostream.h b/libcxx/include/__ostream/basic_ostream.h
index 1f7fe8338c5894d9ece847abf44bb48930852600..1b1c026706cdfd138c63d9dd2a234fa293638980 100644
--- a/libcxx/include/__ostream/basic_ostream.h
+++ b/libcxx/include/__ostream/basic_ostream.h
@@ -156,15 +156,15 @@ basic_ostream<_CharT, _Traits>::sentry::sentry(basic_ostream<_CharT, _Traits>& _
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>::sentry::~sentry() {
   if (__os_.rdbuf() && __os_.good() && (__os_.flags() & ios_base::unitbuf) && uncaught_exceptions() == 0) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       if (__os_.rdbuf()->pubsync() == -1)
         __os_.setstate(ios_base::badbit);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -185,15 +185,15 @@ basic_ostream<_CharT, _Traits>::~basic_ostream() {}
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>&
 basic_ostream<_CharT, _Traits>::operator<<(basic_streambuf<char_type, traits_type>* __sb) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       if (__sb) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
         try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
           typedef istreambuf_iterator<_CharT, _Traits> _Ip;
           typedef ostreambuf_iterator<_CharT, _Traits> _Op;
           _Ip __i(__sb);
@@ -207,27 +207,27 @@ basic_ostream<_CharT, _Traits>::operator<<(basic_streambuf<char_type, traits_typ
           }
           if (__c == 0)
             this->setstate(ios_base::failbit);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
         } catch (...) {
           this->__set_failbit_and_consider_rethrow();
         }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       } else
         this->setstate(ios_base::badbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(bool __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -235,19 +235,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(bool
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(short __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       ios_base::fmtflags __flags = ios_base::flags() & ios_base::basefield;
@@ -262,19 +262,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(short
               .failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned short __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -282,19 +282,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsig
       if (__f.put(*this, *this, this->fill(), static_cast<unsigned long>(__n)).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(int __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       ios_base::fmtflags __flags = ios_base::flags() & ios_base::basefield;
@@ -309,19 +309,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(int _
               .failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned int __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -329,19 +329,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsig
       if (__f.put(*this, *this, this->fill(), static_cast<unsigned long>(__n)).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -349,19 +349,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned long __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -369,19 +369,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsig
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long long __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -389,19 +389,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsigned long long __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -409,19 +409,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(unsig
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(float __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -429,19 +429,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(float
       if (__f.put(*this, *this, this->fill(), static_cast<double>(__n)).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(double __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -449,19 +449,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(doubl
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long double __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -469,19 +469,19 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(long
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(const void* __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef num_put<char_type, ostreambuf_iterator<char_type, traits_type> > _Fp;
@@ -489,20 +489,20 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::operator<<(const
       if (__f.put(*this, *this, this->fill(), __n).failed())
         this->setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
 __put_character_sequence(basic_ostream<_CharT, _Traits>& __os, const _CharT* __str, size_t __len) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
     if (__s) {
       typedef ostreambuf_iterator<_CharT, _Traits> _Ip;
@@ -516,11 +516,11 @@ __put_character_sequence(basic_ostream<_CharT, _Traits>& __os, const _CharT* __s
               .failed())
         __os.setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return __os;
 }
 
@@ -531,9 +531,9 @@ _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_
 
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_CharT, _Traits>& __os, char __cn) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
     if (__s) {
       _CharT __c = __os.widen(__cn);
@@ -548,11 +548,11 @@ _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& operator<<(basic_ostream<_
               .failed())
         __os.setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return __os;
 }
 
@@ -580,9 +580,9 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const _CharT* __str) {
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os, const char* __strn) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
     if (__s) {
       typedef ostreambuf_iterator<_CharT, _Traits> _Ip;
@@ -609,11 +609,11 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const char* __strn) {
               .failed())
         __os.setstate(ios_base::badbit | ios_base::failbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return __os;
 }
 
@@ -638,9 +638,9 @@ operator<<(basic_ostream<char, _Traits>& __os, const unsigned char* __str) {
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::put(char_type __c) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __s(*this);
     if (__s) {
       typedef ostreambuf_iterator<_CharT, _Traits> _Op;
@@ -649,37 +649,37 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::put(char_type __
       if (__o.failed())
         this->setstate(ios_base::badbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::write(const char_type* __s, streamsize __n) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     sentry __sen(*this);
     if (__sen && __n) {
       if (this->rdbuf()->sputn(__s, __n) != __n)
         this->setstate(ios_base::badbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
 template <class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::flush() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     if (this->rdbuf()) {
       sentry __s(*this);
       if (__s) {
@@ -687,11 +687,11 @@ basic_ostream<_CharT, _Traits>& basic_ostream<_CharT, _Traits>::flush() {
           this->setstate(ios_base::badbit);
       }
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return *this;
 }
 
@@ -823,7 +823,7 @@ basic_ostream<wchar_t, _Traits>& operator<<(basic_ostream<wchar_t, _Traits>&, co
 
 #    endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
 
-#    ifndef _LIBCPP_HAS_NO_CHAR8_T
+#    if _LIBCPP_HAS_CHAR8_T
 template <class _Traits>
 basic_ostream<char, _Traits>& operator<<(basic_ostream<char, _Traits>&, char8_t) = delete;
 
diff --git a/libcxx/include/__ostream/print.h b/libcxx/include/__ostream/print.h
index 6c82b11e1de96b5804d14f5f6ac0bbd2cdab89ee..82eb93a8457928b0c91ed1c5f673ea8b95010753 100644
--- a/libcxx/include/__ostream/print.h
+++ b/libcxx/include/__ostream/print.h
@@ -52,9 +52,9 @@ __vprint_nonunicode(ostream& __os, string_view __fmt, format_args __args, bool _
     const char* __str = __o.data();
     size_t __len      = __o.size();
 
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       typedef ostreambuf_iterator<char> _Ip;
       if (std::__pad_and_output(
               _Ip(__os),
@@ -66,11 +66,11 @@ __vprint_nonunicode(ostream& __os, string_view __fmt, format_args __args, bool _
               .failed())
         __os.setstate(ios_base::badbit | ios_base::failbit);
 
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __os.__set_badbit_and_consider_rethrow();
     }
-#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -115,9 +115,9 @@ _LIBCPP_HIDE_FROM_ABI void __vprint_unicode(ostream& __os, string_view __fmt, fo
   // This is the path for the native API, start with flushing.
   __os.flush();
 
-#        ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#        if _LIBCPP_HAS_EXCEPTIONS
   try {
-#        endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#        endif // _LIBCPP_HAS_EXCEPTIONS
     ostream::sentry __s(__os);
     if (__s) {
 #        ifndef _LIBCPP_WIN32API
@@ -129,11 +129,11 @@ _LIBCPP_HIDE_FROM_ABI void __vprint_unicode(ostream& __os, string_view __fmt, fo
 #        endif
     }
 
-#        ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#        if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#        endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#        endif // _LIBCPP_HAS_EXCEPTIONS
 #      endif   // _LIBCPP_AVAILABILITY_HAS_PRINT
 }
 
diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h
index 2d6ab49c8f7f3a2968519a4ac6f4c5b55d2a0d06..a92d0978e5c640409c9632b2a19014a04de0a9e0 100644
--- a/libcxx/include/__pstl/backends/libdispatch.h
+++ b/libcxx/include/__pstl/backends/libdispatch.h
@@ -142,11 +142,11 @@ struct __cpu_traits<__libdispatch_backend_tag> {
 
     unique_ptr<__merge_range_t[], decltype(__destroy)> __ranges(
         [&]() -> __merge_range_t* {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
           try {
 #  endif
             return std::allocator<__merge_range_t>().allocate(__n_ranges);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
           } catch (const std::bad_alloc&) {
             return nullptr;
           }
diff --git a/libcxx/include/__random/is_valid.h b/libcxx/include/__random/is_valid.h
index a3e0f143ae86a792f05a60c539e24231aacb64d6..f6679b3fdc427b1d2ac96f025e3a3c4fa0a65b0d 100644
--- a/libcxx/include/__random/is_valid.h
+++ b/libcxx/include/__random/is_valid.h
@@ -66,12 +66,12 @@ struct __libcpp_random_is_valid_inttype<unsigned long> : true_type {};
 template <>
 struct __libcpp_random_is_valid_inttype<unsigned long long> : true_type {};
 
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 template <>
 struct __libcpp_random_is_valid_inttype<__int128_t> : true_type {}; // extension
 template <>
 struct __libcpp_random_is_valid_inttype<__uint128_t> : true_type {}; // extension
-#endif                                                               // _LIBCPP_HAS_NO_INT128
+#endif                                                               // _LIBCPP_HAS_INT128
 
 // [rand.req.urng]/3:
 // A class G meets the uniform random bit generator requirements if G models
diff --git a/libcxx/include/__random/linear_congruential_engine.h b/libcxx/include/__random/linear_congruential_engine.h
index 9d77649e9cfc8e04b4f778d15b6f2100a9694f36..a0afda4945cdcf244ddafb7bfde6d0934ca22a9c 100644
--- a/libcxx/include/__random/linear_congruential_engine.h
+++ b/libcxx/include/__random/linear_congruential_engine.h
@@ -48,7 +48,7 @@ struct __lce_alg_picker {
       : _Schrage ? _LCE_Schrage
                  : _LCE_Promote;
 
-#ifdef _LIBCPP_HAS_NO_INT128
+#if !_LIBCPP_HAS_INT128
   static_assert(_Mp != (unsigned long long)(-1) || _Full || _Part || _Schrage,
                 "The current values for a, c, and m are not currently supported on platforms without __int128");
 #endif
@@ -63,7 +63,7 @@ struct __lce_ta;
 
 // 64
 
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 template <unsigned long long _Ap, unsigned long long _Cp, unsigned long long _Mp>
 struct __lce_ta<_Ap, _Cp, _Mp, (unsigned long long)(-1), _LCE_Promote> {
   typedef unsigned long long result_type;
diff --git a/libcxx/include/__random/log2.h b/libcxx/include/__random/log2.h
index 74b4889c6402b53064d3504fb5f250025155882d..c96a5247ff6dbe0919e0096646be6ca2a01e9819 100644
--- a/libcxx/include/__random/log2.h
+++ b/libcxx/include/__random/log2.h
@@ -38,7 +38,7 @@ struct __log2_imp<unsigned long long, 0, _Rp> {
   static const size_t value = _Rp + 1;
 };
 
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 
 template <__uint128_t _Xp, size_t _Rp>
 struct __log2_imp<__uint128_t, _Xp, _Rp> {
@@ -47,16 +47,16 @@ struct __log2_imp<__uint128_t, _Xp, _Rp> {
                   : __log2_imp<unsigned long long, _Xp, 63>::value;
 };
 
-#endif // _LIBCPP_HAS_NO_INT128
+#endif // _LIBCPP_HAS_INT128
 
 template <class _UIntType, _UIntType _Xp>
 struct __log2 {
   static const size_t value = __log2_imp<
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
       __conditional_t<sizeof(_UIntType) <= sizeof(unsigned long long), unsigned long long, __uint128_t>,
 #else
       unsigned long long,
-#endif // _LIBCPP_HAS_NO_INT128
+#endif // _LIBCPP_HAS_INT128
       _Xp,
       sizeof(_UIntType) * __CHAR_BIT__ - 1>::value;
 };
diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer
index 7916769bd83bfaf99bda682829fe476cb3a58f51..dfe552fbb45893bc5b7c481ce37006bea905f3e5 100644
--- a/libcxx/include/__split_buffer
+++ b/libcxx/include/__split_buffer
@@ -440,9 +440,9 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::reserve(size
 template <class _Tp, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
   if (capacity() > size()) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       __split_buffer<value_type, __alloc_rr&> __t(size(), 0, __alloc());
       __t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
       __t.__end_ = __t.__begin_ + (__end_ - __begin_);
@@ -450,10 +450,10 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::shrink_to_fi
       std::swap(__begin_, __t.__begin_);
       std::swap(__end_, __t.__end_);
       std::swap(__end_cap(), __t.__end_cap());
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
diff --git a/libcxx/include/__string/char_traits.h b/libcxx/include/__string/char_traits.h
index 9b7d567dd3c278a2ffffb3842179e467827ae571..107f0a96a022776d23bf6603ad53cd864b5ea2e4 100644
--- a/libcxx/include/__string/char_traits.h
+++ b/libcxx/include/__string/char_traits.h
@@ -257,7 +257,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits<wchar_t> : __char_traits_base<wchar_t, w
 };
 #endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 
 template <>
 struct _LIBCPP_TEMPLATE_VIS char_traits<char8_t>
@@ -277,7 +277,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits<char8_t>
   }
 };
 
-#endif // _LIBCPP_HAS_NO_CHAR8_T
+#endif // _LIBCPP_HAS_CHAR8_T
 
 template <>
 struct _LIBCPP_TEMPLATE_VIS char_traits<char16_t>
diff --git a/libcxx/include/__string/constexpr_c_functions.h b/libcxx/include/__string/constexpr_c_functions.h
index 9b8871e2e71a38f1b545334ec74b78358201efcb..e62a7b0cd1b3bb86f93c015b72b4957323a5382f 100644
--- a/libcxx/include/__string/constexpr_c_functions.h
+++ b/libcxx/include/__string/constexpr_c_functions.h
@@ -43,7 +43,7 @@ inline const bool __is_char_type = false;
 template <>
 inline const bool __is_char_type<char> = true;
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 template <>
 inline const bool __is_char_type<char8_t> = true;
 #endif
diff --git a/libcxx/include/__system_error/system_error.h b/libcxx/include/__system_error/system_error.h
index 3ffa1029ca5c2e33aa40085146086bd29993d362..2763aaba88f5fa1bd2da40bd8021f5cf19e8625a 100644
--- a/libcxx/include/__system_error/system_error.h
+++ b/libcxx/include/__system_error/system_error.h
@@ -41,7 +41,7 @@ public:
 
 [[__noreturn__]] _LIBCPP_EXPORTED_FROM_ABI void __throw_system_error(int __ev, const char* __what_arg);
 [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI inline void __throw_system_error(error_code __ec, const char* __what_arg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw system_error(__ec, __what_arg);
 #else
   _LIBCPP_VERBOSE_ABORT(
diff --git a/libcxx/include/__type_traits/container_traits.h b/libcxx/include/__type_traits/container_traits.h
new file mode 100644
index 0000000000000000000000000000000000000000..5262cef94c61efd395f325703e719e2d1fdf3496
--- /dev/null
+++ b/libcxx/include/__type_traits/container_traits.h
@@ -0,0 +1,43 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_CONTAINER_TRAITS_H
+#define _LIBCPP___TYPE_TRAITS_CONTAINER_TRAITS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// // __container_traits is a general purpose utility containing traits describing various containers operations.
+// It currently only has one trait: `__emplacement_has_strong_exception_safety_guarantee`, but it's
+// intended to be extended in the future.
+//
+// These traits should only be used for optimization or QoI purposes. In particular, since this is a libc++ internal
+// mechanism, no user-defined containers should be expected to specialize these traits (in fact it would be illegal for
+// them to do so). Hence, when using these traits to implement something, make sure that a container that fails to
+// specialize these traits does not result in non-conforming code.
+//
+// When a trait is nonsensical for a type, this class still provides a fallback value for that trait.
+// For example, `std::array` does not support `insert` or `emplace`, so
+// `__emplacement_has_strong_exception_safety_guarantee` is false for such types.
+template <class _Container>
+struct __container_traits {
+  // A trait that tells whether a single element insertion/emplacement via member function
+  // `insert(...)` or `emplace(...)` has strong exception guarantee, that is, if the function
+  // exits via an exception, the original container is unaffected
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = false;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_CONTAINER_TRAITS_H
diff --git a/libcxx/include/__type_traits/is_integral.h b/libcxx/include/__type_traits/is_integral.h
index 26969885af8dfd7aa199ee2a0a8255d889b92efd..61178720a5938c752e71a52f4a00af2ad86d0367 100644
--- a/libcxx/include/__type_traits/is_integral.h
+++ b/libcxx/include/__type_traits/is_integral.h
@@ -28,7 +28,7 @@ template <>          struct __libcpp_is_integral<unsigned char>      { enum { va
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
 template <>          struct __libcpp_is_integral<wchar_t>            { enum { value = 1 }; };
 #endif
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 template <>          struct __libcpp_is_integral<char8_t>            { enum { value = 1 }; };
 #endif
 template <>          struct __libcpp_is_integral<char16_t>           { enum { value = 1 }; };
@@ -41,7 +41,7 @@ template <>          struct __libcpp_is_integral<long>               { enum { va
 template <>          struct __libcpp_is_integral<unsigned long>      { enum { value = 1 }; };
 template <>          struct __libcpp_is_integral<long long>          { enum { value = 1 }; };
 template <>          struct __libcpp_is_integral<unsigned long long> { enum { value = 1 }; };
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 template <>          struct __libcpp_is_integral<__int128_t>         { enum { value = 1 }; };
 template <>          struct __libcpp_is_integral<__uint128_t>        { enum { value = 1 }; };
 #endif
diff --git a/libcxx/include/__type_traits/is_pointer.h b/libcxx/include/__type_traits/is_pointer.h
index 38eb7996c686424114429a3e957275a1c48aa440..9701e57807cf6f736d628bdcfeacc7de669eb4be 100644
--- a/libcxx/include/__type_traits/is_pointer.h
+++ b/libcxx/include/__type_traits/is_pointer.h
@@ -40,7 +40,7 @@ template <class _Tp>
 struct __libcpp_remove_objc_qualifiers {
   typedef _Tp type;
 };
-#  if defined(_LIBCPP_HAS_OBJC_ARC)
+#  if _LIBCPP_HAS_OBJC_ARC
 // clang-format off
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __strong> { typedef _Tp type; };
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __weak> { typedef _Tp type; };
diff --git a/libcxx/include/__type_traits/is_scalar.h b/libcxx/include/__type_traits/is_scalar.h
index 455200de472089989174151ebe049f489e710c08..242023a6877c9442838029221ee90e19f1900bf1 100644
--- a/libcxx/include/__type_traits/is_scalar.h
+++ b/libcxx/include/__type_traits/is_scalar.h
@@ -37,7 +37,7 @@ inline constexpr bool is_scalar_v = __is_scalar(_Tp);
 
 template <class _Tp>
 struct __is_block : false_type {};
-#  if defined(_LIBCPP_HAS_EXTENSION_BLOCKS)
+#  if _LIBCPP_HAS_EXTENSION_BLOCKS
 template <class _Rp, class... _Args>
 struct __is_block<_Rp (^)(_Args...)> : true_type {};
 #  endif
diff --git a/libcxx/include/__type_traits/is_signed_integer.h b/libcxx/include/__type_traits/is_signed_integer.h
index 467548d0aaafbd20054ec2938e5049e34dd9ccf3..a3e19a66f2c74a9884f6e4cc7ea3a663659d5d13 100644
--- a/libcxx/include/__type_traits/is_signed_integer.h
+++ b/libcxx/include/__type_traits/is_signed_integer.h
@@ -25,7 +25,7 @@ template <>          struct __libcpp_is_signed_integer<signed short>     : publi
 template <>          struct __libcpp_is_signed_integer<signed int>       : public true_type {};
 template <>          struct __libcpp_is_signed_integer<signed long>      : public true_type {};
 template <>          struct __libcpp_is_signed_integer<signed long long> : public true_type {};
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 template <>          struct __libcpp_is_signed_integer<__int128_t>       : public true_type {};
 #endif
 // clang-format on
diff --git a/libcxx/include/__type_traits/is_unsigned_integer.h b/libcxx/include/__type_traits/is_unsigned_integer.h
index 265894b32d4fc1dd7595c26c5ceb16b44bae825f..86e42a1e8156aa100fd5de51e88ae9aebd2d7d79 100644
--- a/libcxx/include/__type_traits/is_unsigned_integer.h
+++ b/libcxx/include/__type_traits/is_unsigned_integer.h
@@ -25,7 +25,7 @@ template <>          struct __libcpp_is_unsigned_integer<unsigned short>     : p
 template <>          struct __libcpp_is_unsigned_integer<unsigned int>       : public true_type {};
 template <>          struct __libcpp_is_unsigned_integer<unsigned long>      : public true_type {};
 template <>          struct __libcpp_is_unsigned_integer<unsigned long long> : public true_type {};
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 template <>          struct __libcpp_is_unsigned_integer<__uint128_t>        : public true_type {};
 #endif
 // clang-format on
diff --git a/libcxx/include/__type_traits/make_32_64_or_128_bit.h b/libcxx/include/__type_traits/make_32_64_or_128_bit.h
index f7f2e81735dafd43d40f66a271a27a69a276adfe..70f84fcd18686cd84f286b17a4e899329bdf44dc 100644
--- a/libcxx/include/__type_traits/make_32_64_or_128_bit.h
+++ b/libcxx/include/__type_traits/make_32_64_or_128_bit.h
@@ -35,7 +35,7 @@ using __make_32_64_or_128_bit_t =
     __copy_unsigned_t<_Tp,
         __conditional_t<sizeof(_Tp) <= sizeof(int32_t),    int32_t,
         __conditional_t<sizeof(_Tp) <= sizeof(int64_t),    int64_t,
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
         __conditional_t<sizeof(_Tp) <= sizeof(__int128_t), __int128_t,
         /* else */                                         void>
 #else
diff --git a/libcxx/include/__type_traits/make_signed.h b/libcxx/include/__type_traits/make_signed.h
index d09d6ed4a1e7cc7c626039bfaaa7ce1578475284..8070690b3a7a90d010cd2e74ab7054eaf4df8830 100644
--- a/libcxx/include/__type_traits/make_signed.h
+++ b/libcxx/include/__type_traits/make_signed.h
@@ -35,11 +35,11 @@ typedef __type_list<signed char,
         __type_list<signed int,
         __type_list<signed long,
         __type_list<signed long long,
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
         __type_list<__int128_t,
 #  endif
         __nat
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
         >
 #  endif
         > > > > > __signed_types;
@@ -63,7 +63,7 @@ template <> struct __make_signed<  signed long,      true> {typedef long      ty
 template <> struct __make_signed<unsigned long,      true> {typedef long      type;};
 template <> struct __make_signed<  signed long long, true> {typedef long long type;};
 template <> struct __make_signed<unsigned long long, true> {typedef long long type;};
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 template <> struct __make_signed<__int128_t,         true> {typedef __int128_t type;};
 template <> struct __make_signed<__uint128_t,        true> {typedef __int128_t type;};
 #  endif
diff --git a/libcxx/include/__type_traits/make_unsigned.h b/libcxx/include/__type_traits/make_unsigned.h
index 8757f451eb807bd39facf3a4115ca20c8d79fd54..562f7bab8a7fbf65ab8d93c35ee38f761aa5ebca 100644
--- a/libcxx/include/__type_traits/make_unsigned.h
+++ b/libcxx/include/__type_traits/make_unsigned.h
@@ -37,11 +37,11 @@ typedef __type_list<unsigned char,
         __type_list<unsigned int,
         __type_list<unsigned long,
         __type_list<unsigned long long,
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
         __type_list<__uint128_t,
 #  endif
         __nat
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
         >
 #  endif
         > > > > > __unsigned_types;
@@ -65,7 +65,7 @@ template <> struct __make_unsigned<  signed long,      true> {typedef unsigned l
 template <> struct __make_unsigned<unsigned long,      true> {typedef unsigned long      type;};
 template <> struct __make_unsigned<  signed long long, true> {typedef unsigned long long type;};
 template <> struct __make_unsigned<unsigned long long, true> {typedef unsigned long long type;};
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
 template <> struct __make_unsigned<__int128_t,         true> {typedef __uint128_t        type;};
 template <> struct __make_unsigned<__uint128_t,        true> {typedef __uint128_t        type;};
 #  endif
diff --git a/libcxx/include/__type_traits/promote.h b/libcxx/include/__type_traits/promote.h
index 2b2a6843b9150219166884e613ae5dafb059b7a5..0f545bc507398f9c57573899a3d0828621816cc8 100644
--- a/libcxx/include/__type_traits/promote.h
+++ b/libcxx/include/__type_traits/promote.h
@@ -39,7 +39,7 @@ class __promote {
   static double __test(unsigned long);
   static double __test(long long);
   static double __test(unsigned long long);
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   static double __test(__int128_t);
   static double __test(__uint128_t);
 #  endif
@@ -63,7 +63,7 @@ struct __numeric_type {
   static double __test(unsigned long);
   static double __test(long long);
   static double __test(unsigned long long);
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   static double __test(__int128_t);
   static double __test(__uint128_t);
 #  endif
diff --git a/libcxx/include/__utility/convert_to_integral.h b/libcxx/include/__utility/convert_to_integral.h
index f1fcdd98010cca26fec39066ee261ae070016483..8947c349d830209cd87ceea30dcc87e2e4a0b62a 100644
--- a/libcxx/include/__utility/convert_to_integral.h
+++ b/libcxx/include/__utility/convert_to_integral.h
@@ -42,7 +42,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long long __convert_to_integral(_
   return __val;
 }
 
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __int128_t __convert_to_integral(__int128_t __val) { return __val; }
 
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __uint128_t __convert_to_integral(__uint128_t __val) { return __val; }
diff --git a/libcxx/include/__utility/exception_guard.h b/libcxx/include/__utility/exception_guard.h
index 9f732ca265c86e29e5678e649ffd0a485bc6e325..a03bd7e8f35227767a2c1c3822c303d07ec7c863 100644
--- a/libcxx/include/__utility/exception_guard.h
+++ b/libcxx/include/__utility/exception_guard.h
@@ -44,7 +44,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // less common, especially one that tries to catch an exception through -fno-exceptions code.
 //
 // __exception_guard can help greatly simplify code that would normally be cluttered by
-// `#if _LIBCPP_HAS_NO_EXCEPTIONS`. For example:
+// `#if _LIBCPP_HAS_EXCEPTIONS`. For example:
 //
 //    template <class Iterator, class Size, class OutputIterator>
 //    Iterator uninitialized_copy_n(Iterator iter, Size n, OutputIterator out) {
@@ -124,7 +124,7 @@ private:
 
 _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__exception_guard_noexceptions);
 
-#ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#if !_LIBCPP_HAS_EXCEPTIONS
 template <class _Rollback>
 using __exception_guard = __exception_guard_noexceptions<_Rollback>;
 #else
diff --git a/libcxx/include/any b/libcxx/include/any
index 6e4ff31ff9b620c4267dd79bbcc663eea76dbefc..e32aa7f8e8a42093ee1bcd58409e9d9786d4d296 100644
--- a/libcxx/include/any
+++ b/libcxx/include/any
@@ -128,7 +128,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if _LIBCPP_STD_VER >= 17
 
 [[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST void __throw_bad_any_cast() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_any_cast();
 #  else
   _LIBCPP_VERBOSE_ABORT("bad_any_cast was thrown in -fno-exceptions mode");
@@ -176,7 +176,7 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr const void* __get_fallback_typeid() {
 
 template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI bool __compare_typeid(type_info const* __id, const void* __fallback_id) {
-#  if !defined(_LIBCPP_HAS_NO_RTTI)
+#  if _LIBCPP_HAS_RTTI
   if (__id && *__id == typeid(_Tp))
     return true;
 #  endif
@@ -266,7 +266,7 @@ public:
   // 6.3.4 any observers
   _LIBCPP_HIDE_FROM_ABI bool has_value() const _NOEXCEPT { return __h_ != nullptr; }
 
-#  if !defined(_LIBCPP_HAS_NO_RTTI)
+#  if _LIBCPP_HAS_RTTI
   _LIBCPP_HIDE_FROM_ABI const type_info& type() const _NOEXCEPT {
     if (__h_) {
       return *static_cast<type_info const*>(this->__call(_Action::_TypeInfo));
@@ -372,7 +372,7 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI static void* __type_info() {
-#  if !defined(_LIBCPP_HAS_NO_RTTI)
+#  if _LIBCPP_HAS_RTTI
     return const_cast<void*>(static_cast<void const*>(&typeid(_Tp)));
 #  else
     return nullptr;
@@ -444,7 +444,7 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI static void* __type_info() {
-#  if !defined(_LIBCPP_HAS_NO_RTTI)
+#  if _LIBCPP_HAS_RTTI
     return const_cast<void*>(static_cast<void const*>(&typeid(_Tp)));
 #  else
     return nullptr;
@@ -579,7 +579,7 @@ _LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any* __any) _NOEXCEPT {
     void* __p = __any->__call(
         _Action::_Get,
         nullptr,
-#  if !defined(_LIBCPP_HAS_NO_RTTI)
+#  if _LIBCPP_HAS_RTTI
         &typeid(_ValueType),
 #  else
         nullptr,
diff --git a/libcxx/include/atomic b/libcxx/include/atomic
index bf65750672bb0a5bb8da3856e8247d3af6523a0e..ebd46238eec986083327699a8c7ec91217f56ef3 100644
--- a/libcxx/include/atomic
+++ b/libcxx/include/atomic
@@ -613,7 +613,7 @@ template <class T>
 #  pragma GCC system_header
 #endif
 
-#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER
+#if !_LIBCPP_HAS_ATOMIC_HEADER
 #  error <atomic> is not implemented
 #endif
 
diff --git a/libcxx/include/cuchar b/libcxx/include/cuchar
index bfc26f03aaf782688f21f5ec7371f05e4cbea39c..1d156eb7aaaf35e746b7e3fe591615ec6e741202 100644
--- a/libcxx/include/cuchar
+++ b/libcxx/include/cuchar
@@ -59,7 +59,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
 
-#  if !defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8)
+#  if _LIBCPP_HAS_C8RTOMB_MBRTOC8
 using ::mbrtoc8 _LIBCPP_USING_IF_EXISTS;
 using ::c8rtomb _LIBCPP_USING_IF_EXISTS;
 #  endif
diff --git a/libcxx/include/deque b/libcxx/include/deque
index bab0526629f0f8c02d54f9d40d3b2d7aeec8b2cb..11219d1a99244ee4533a37d3bad80c5bc05a17f4 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -220,6 +220,8 @@ template <class T, class Allocator, class Predicate>
 #include <__ranges/size.h>
 #include <__split_buffer>
 #include <__type_traits/conditional.h>
+#include <__type_traits/container_traits.h>
+#include <__type_traits/disjunction.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_convertible.h>
@@ -918,7 +920,7 @@ private:
     (void)__end;
     (void)__annotation_type;
     (void)__place;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     // __beg - index of the first item to annotate
     // __end - index behind the last item to annotate (so last item + 1)
     // __annotation_type - __asan_unposion or __asan_poison
@@ -1011,23 +1013,23 @@ private:
       std::__annotate_double_ended_contiguous_container<_Allocator>(
           __mem_beg, __mem_end, __old_beg, __old_end, __new_beg, __new_end);
     }
-#endif // !_LIBCPP_HAS_NO_ASAN
+#endif // _LIBCPP_HAS_ASAN
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
     (void)__current_size;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     if (__current_size == 0)
       __annotate_from_to(0, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
     else {
       __annotate_from_to(0, __start_, __asan_poison, __asan_front_moved);
       __annotate_from_to(__start_ + __current_size, __map_.size() * __block_size, __asan_poison, __asan_back_moved);
     }
-#endif
+#endif // _LIBCPP_HAS_ASAN
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     if (empty()) {
       for (size_t __i = 0; __i < __map_.size(); ++__i) {
         __annotate_whole_block(__i, __asan_unposion);
@@ -1036,19 +1038,19 @@ private:
       __annotate_from_to(0, __start_, __asan_unposion, __asan_front_moved);
       __annotate_from_to(__start_ + size(), __map_.size() * __block_size, __asan_unposion, __asan_back_moved);
     }
-#endif
+#endif // _LIBCPP_HAS_ASAN
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_increase_front(size_type __n) const _NOEXCEPT {
     (void)__n;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_from_to(__start_ - __n, __start_, __asan_unposion, __asan_front_moved);
 #endif
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_increase_back(size_type __n) const _NOEXCEPT {
     (void)__n;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_from_to(__start_ + size(), __start_ + size() + __n, __asan_unposion, __asan_back_moved);
 #endif
   }
@@ -1056,7 +1058,7 @@ private:
   _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_front(size_type __old_size, size_type __old_start) const _NOEXCEPT {
     (void)__old_size;
     (void)__old_start;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_from_to(__old_start, __old_start + (__old_size - size()), __asan_poison, __asan_front_moved);
 #endif
   }
@@ -1064,7 +1066,7 @@ private:
   _LIBCPP_HIDE_FROM_ABI void __annotate_shrink_back(size_type __old_size, size_type __old_start) const _NOEXCEPT {
     (void)__old_size;
     (void)__old_start;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_from_to(__old_start + size(), __old_start + __old_size, __asan_poison, __asan_back_moved);
 #endif
   }
@@ -1077,7 +1079,7 @@ private:
   __annotate_whole_block(size_t __block_index, __asan_annotation_type __annotation_type) const _NOEXCEPT {
     (void)__block_index;
     (void)__annotation_type;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __map_const_iterator __block_it = __map_.begin() + __block_index;
     const void* __block_start       = std::__to_address(*__block_it);
     const void* __block_end         = std::__to_address(*__block_it + __block_size);
@@ -1090,7 +1092,7 @@ private:
     }
 #endif
   }
-#if !defined(_LIBCPP_HAS_NO_ASAN)
+#if _LIBCPP_HAS_ASAN
 
 public:
   _LIBCPP_HIDE_FROM_ABI bool __verify_asan_annotations() const _NOEXCEPT {
@@ -1152,7 +1154,7 @@ public:
   }
 
 private:
-#endif // _LIBCPP_VERIFY_ASAN_DEQUE_ANNOTATIONS
+#endif // _LIBCPP_HAS_ASAN
   _LIBCPP_HIDE_FROM_ABI bool __maybe_remove_front_spare(bool __keep_one = true) {
     if (__front_spare_blocks() >= 2 || (!__keep_one && __front_spare_blocks())) {
       __annotate_whole_block(0, __asan_unposion);
@@ -2123,22 +2125,22 @@ void deque<_Tp, _Allocator>::__add_front_capacity(size_type __n) {
     size_type __ds = (__nb + __back_capacity) * __block_size - __map_.empty();
     __split_buffer<pointer, __pointer_allocator&> __buf(
         std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()), 0, __map_.__alloc());
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __nb > 0; --__nb) {
         __buf.push_back(__alloc_traits::allocate(__a, __block_size));
         // ASan: this is empty container, we have to poison whole block
         __annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __annotate_delete();
       for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
         __alloc_traits::deallocate(__a, *__i, __block_size);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     for (; __back_capacity > 0; --__back_capacity) {
       __buf.push_back(__map_.back());
       __map_.pop_back();
@@ -2248,22 +2250,22 @@ void deque<_Tp, _Allocator>::__add_back_capacity(size_type __n) {
         std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()),
         __map_.size() - __front_capacity,
         __map_.__alloc());
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __nb > 0; --__nb) {
         __buf.push_back(__alloc_traits::allocate(__a, __block_size));
         // ASan: this is an empty container, we have to poison the whole block
         __annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __annotate_delete();
       for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
         __alloc_traits::deallocate(__a, *__i, __block_size);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     for (; __front_capacity > 0; --__front_capacity) {
       __buf.push_back(__map_.front());
       __map_.pop_front();
@@ -2609,6 +2611,17 @@ inline constexpr bool __format::__enable_insertable<std::deque<wchar_t>> = true;
 
 #endif // _LIBCPP_STD_VER >= 20
 
+template <class _Tp, class _Allocator>
+struct __container_traits<deque<_Tp, _Allocator> > {
+  // http://eel.is/c++draft/deque.modifiers#3
+  //  If an exception is thrown other than by the copy constructor, move constructor, assignment operator, or move
+  //  assignment operator of T, there are no effects. If an exception is thrown while inserting a single element at
+  //  either end, there are no effects. Otherwise, if an exception is thrown by the move constructor of a
+  //  non-Cpp17CopyInsertable T, the effects are unspecified.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
+      _Or<is_nothrow_move_constructible<_Tp>, __is_cpp17_copy_insertable<_Allocator> >::value;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/experimental/__simd/utility.h b/libcxx/include/experimental/__simd/utility.h
index 0103b06b719532e622f0b0867fd605c54e3a4032..01736925d155c4cbf41b8f639d91a84edcddcec1 100644
--- a/libcxx/include/experimental/__simd/utility.h
+++ b/libcxx/include/experimental/__simd/utility.h
@@ -47,7 +47,7 @@ _LIBCPP_HIDE_FROM_ABI auto __choose_mask_type() {
   } else if constexpr (sizeof(_Tp) == 8) {
     return uint64_t{};
   }
-#  ifndef _LIBCPP_HAS_NO_INT128
+#  if _LIBCPP_HAS_INT128
   else if constexpr (sizeof(_Tp) == 16) {
     return __uint128_t{};
   }
diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list
index c5ae8add511cf83877754cdea9c6086e869a7e15..d3262fb8eaed1f527f442427bf52b8f49a85b74d 100644
--- a/libcxx/include/forward_list
+++ b/libcxx/include/forward_list
@@ -218,6 +218,7 @@ template <class T, class Allocator, class Predicate>
 #include <__ranges/container_compatible_range.h>
 #include <__ranges/from_range.h>
 #include <__type_traits/conditional.h>
+#include <__type_traits/container_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_const.h>
@@ -1122,13 +1123,13 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, size_type __n, const
   if (__n > 0) {
     __node_pointer __first = this->__create_node(/* next = */ nullptr, __v);
     __node_pointer __last  = __first;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, __last = __last->__next_) {
         __last->__next_ = this->__create_node(/* next = */ nullptr, __v);
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (__first != nullptr) {
         __node_pointer __next = __first->__next_;
@@ -1137,7 +1138,7 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, size_type __n, const
       }
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
     __r             = static_cast<__begin_node_pointer>(__last);
@@ -1162,13 +1163,13 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
     __node_pointer __first = this->__create_node(/* next = */ nullptr, *__f);
     __node_pointer __last  = __first;
 
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (++__f; __f != __l; ++__f, ((void)(__last = __last->__next_))) {
         __last->__next_ = this->__create_node(/* next = */ nullptr, *__f);
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (__first != nullptr) {
         __node_pointer __next = __first->__next_;
@@ -1177,7 +1178,7 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
       }
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
@@ -1544,6 +1545,17 @@ erase(forward_list<_Tp, _Allocator>& __c, const _Up& __v) {
 }
 #endif
 
+template <class _Tp, class _Allocator>
+struct __container_traits<forward_list<_Tp, _Allocator> > {
+  // http://eel.is/c++draft/container.reqmts
+  // Unless otherwise specified (see [associative.reqmts.except], [unord.req.except], [deque.modifiers],
+  // [inplace.vector.modifiers], and [vector.modifiers]) all container types defined in this Clause meet the following
+  // additional requirements:
+  // - If an exception is thrown by an insert() or emplace() function while inserting a single element, that
+  // function has no effects.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/fstream b/libcxx/include/fstream
index de7c614836babdb21bdfbdb238b3a1da056c5be5..dce0efc71d8e5f72a45d00602de49763070675a6 100644
--- a/libcxx/include/fstream
+++ b/libcxx/include/fstream
@@ -211,8 +211,10 @@ typedef basic_fstream<wchar_t> wfstream;
 _LIBCPP_PUSH_MACROS
 #include <__undef_macros>
 
-#if defined(_LIBCPP_MSVCRT) || defined(_NEWLIB_VERSION)
-#  define _LIBCPP_HAS_NO_OFF_T_FUNCTIONS
+#if !defined(_LIBCPP_MSVCRT) && !defined(_NEWLIB_VERSION)
+#  define _LIBCPP_HAS_OFF_T_FUNCTIONS 1
+#else
+#  define _LIBCPP_HAS_OFF_T_FUNCTIONS 0
 #endif
 
 #if !defined(_LIBCPP_HAS_NO_FILESYSTEM) && !defined(_LIBCPP_HAS_NO_LOCALIZATION)
@@ -254,7 +256,7 @@ public:
   // 27.9.1.4 Members:
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   basic_filebuf* open(const char* __s, ios_base::openmode __mode);
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   basic_filebuf* open(const wchar_t* __s, ios_base::openmode __mode);
 #  endif
   _LIBCPP_HIDE_FROM_ABI basic_filebuf* open(const string& __s, ios_base::openmode __mode);
@@ -281,7 +283,7 @@ public:
 #  endif //  _LIBCPP_STD_VER >= 26
 
   _LIBCPP_HIDE_FROM_ABI inline static const char* __make_mdstring(ios_base::openmode __mode) _NOEXCEPT;
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI inline static const wchar_t* __make_mdwstring(ios_base::openmode __mode) _NOEXCEPT;
 #  endif
 
@@ -485,14 +487,14 @@ inline basic_filebuf<_CharT, _Traits>& basic_filebuf<_CharT, _Traits>::operator=
 
 template <class _CharT, class _Traits>
 basic_filebuf<_CharT, _Traits>::~basic_filebuf() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     close();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   if (__owns_eb_)
     delete[] __extbuf_;
   if (__owns_ib_)
@@ -630,7 +632,7 @@ const char* basic_filebuf<_CharT, _Traits>::__make_mdstring(ios_base::openmode _
   __libcpp_unreachable();
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 const wchar_t* basic_filebuf<_CharT, _Traits>::__make_mdwstring(ios_base::openmode __mode) _NOEXCEPT {
   switch (__mode & ~ios_base::ate) {
@@ -705,7 +707,7 @@ inline basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::__open(in
   return __do_open(fdopen(__fd, __mdstr), __mode);
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 // This is basically the same as the char* overload except that it uses _wfopen
 // and long mode strings.
 template <class _CharT, class _Traits>
@@ -929,7 +931,7 @@ basic_filebuf<_CharT, _Traits>::seekoff(off_type __off, ios_base::seekdir __way,
   default:
     return pos_type(off_type(-1));
   }
-#  if defined(_LIBCPP_HAS_NO_OFF_T_FUNCTIONS)
+#  if !_LIBCPP_HAS_OFF_T_FUNCTIONS
   if (fseek(__file_, __width > 0 ? __width * __off : 0, __whence))
     return pos_type(off_type(-1));
   pos_type __r = ftell(__file_);
@@ -947,7 +949,7 @@ typename basic_filebuf<_CharT, _Traits>::pos_type
 basic_filebuf<_CharT, _Traits>::seekpos(pos_type __sp, ios_base::openmode) {
   if (__file_ == nullptr || sync())
     return pos_type(off_type(-1));
-#  if defined(_LIBCPP_HAS_NO_OFF_T_FUNCTIONS)
+#  if !_LIBCPP_HAS_OFF_T_FUNCTIONS
   if (fseek(__file_, __sp, SEEK_SET))
     return pos_type(off_type(-1));
 #  else
@@ -1000,7 +1002,7 @@ int basic_filebuf<_CharT, _Traits>::sync() {
         }
       }
     }
-#  if defined(_LIBCPP_HAS_NO_OFF_T_FUNCTIONS)
+#  if !_LIBCPP_HAS_OFF_T_FUNCTIONS
     if (fseek(__file_, -__c, SEEK_CUR))
       return -1;
 #  else
@@ -1098,7 +1100,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI basic_ifstream();
   _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in);
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const wchar_t* __s, ios_base::openmode __mode = ios_base::in);
 #  endif
   _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const string& __s, ios_base::openmode __mode = ios_base::in);
@@ -1118,7 +1120,7 @@ public:
 #  endif
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   void open(const char* __s, ios_base::openmode __mode = ios_base::in);
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in);
 #  endif
   void open(const string& __s, ios_base::openmode __mode = ios_base::in);
@@ -1147,7 +1149,7 @@ inline basic_ifstream<_CharT, _Traits>::basic_ifstream(const char* __s, ios_base
     this->setstate(ios_base::failbit);
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 inline basic_ifstream<_CharT, _Traits>::basic_ifstream(const wchar_t* __s, ios_base::openmode __mode)
     : basic_istream<char_type, traits_type>(std::addressof(__sb_)) {
@@ -1206,7 +1208,7 @@ void basic_ifstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode _
     this->setstate(ios_base::failbit);
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 void basic_ifstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
   if (__sb_.open(__s, __mode | ios_base::in))
@@ -1254,7 +1256,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI basic_ofstream();
   _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const char* __s, ios_base::openmode __mode = ios_base::out);
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const wchar_t* __s, ios_base::openmode __mode = ios_base::out);
 #  endif
   _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const string& __s, ios_base::openmode __mode = ios_base::out);
@@ -1276,7 +1278,7 @@ public:
 #  endif
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   void open(const char* __s, ios_base::openmode __mode = ios_base::out);
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::out);
 #  endif
   void open(const string& __s, ios_base::openmode __mode = ios_base::out);
@@ -1306,7 +1308,7 @@ inline basic_ofstream<_CharT, _Traits>::basic_ofstream(const char* __s, ios_base
     this->setstate(ios_base::failbit);
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 inline basic_ofstream<_CharT, _Traits>::basic_ofstream(const wchar_t* __s, ios_base::openmode __mode)
     : basic_ostream<char_type, traits_type>(std::addressof(__sb_)) {
@@ -1365,7 +1367,7 @@ void basic_ofstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode _
     this->setstate(ios_base::failbit);
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 void basic_ofstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
   if (__sb_.open(__s, __mode | ios_base::out))
@@ -1414,7 +1416,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI basic_fstream();
   _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const char* __s,
                                                ios_base::openmode __mode = ios_base::in | ios_base::out);
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const wchar_t* __s,
                                                ios_base::openmode __mode = ios_base::in | ios_base::out);
 #  endif
@@ -1440,7 +1442,7 @@ public:
 #  endif
   _LIBCPP_HIDE_FROM_ABI bool is_open() const;
   _LIBCPP_HIDE_FROM_ABI void open(const char* __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
   void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
 #  endif
   _LIBCPP_HIDE_FROM_ABI void open(const string& __s, ios_base::openmode __mode = ios_base::in | ios_base::out);
@@ -1469,7 +1471,7 @@ inline basic_fstream<_CharT, _Traits>::basic_fstream(const char* __s, ios_base::
     this->setstate(ios_base::failbit);
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 inline basic_fstream<_CharT, _Traits>::basic_fstream(const wchar_t* __s, ios_base::openmode __mode)
     : basic_iostream<char_type, traits_type>(std::addressof(__sb_)) {
@@ -1528,7 +1530,7 @@ void basic_fstream<_CharT, _Traits>::open(const char* __s, ios_base::openmode __
     this->setstate(ios_base::failbit);
 }
 
-#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+#  if _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
 void basic_fstream<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
   if (__sb_.open(__s, __mode))
diff --git a/libcxx/include/future b/libcxx/include/future
index 7dc38a3354bb4bfd08e9ef90d587045f625dd1c8..dfa373d6593c7909187e22633ad9af938a4bd428 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -500,7 +500,7 @@ public:
 
 // Declared above std::future_error
 void __throw_future_error(future_errc __ev) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw future_error(make_error_code(__ev));
 #  else
   (void)__ev;
@@ -779,15 +779,15 @@ inline __deferred_assoc_state<_Rp, _Fp>::__deferred_assoc_state(_Fp&& __f) : __f
 
 template <class _Rp, class _Fp>
 void __deferred_assoc_state<_Rp, _Fp>::__execute() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->set_value(__func_());
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Fp>
@@ -809,16 +809,16 @@ inline __deferred_assoc_state<void, _Fp>::__deferred_assoc_state(_Fp&& __f) : __
 
 template <class _Fp>
 void __deferred_assoc_state<void, _Fp>::__execute() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __func_();
     this->set_value();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class _Fp>
@@ -840,15 +840,15 @@ inline __async_assoc_state<_Rp, _Fp>::__async_assoc_state(_Fp&& __f) : __func_(s
 
 template <class _Rp, class _Fp>
 void __async_assoc_state<_Rp, _Fp>::__execute() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->set_value(__func_());
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class _Fp>
@@ -876,16 +876,16 @@ inline __async_assoc_state<void, _Fp>::__async_assoc_state(_Fp&& __f) : __func_(
 
 template <class _Fp>
 void __async_assoc_state<void, _Fp>::__execute() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __func_();
     this->set_value();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     this->set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Fp>
@@ -1656,15 +1656,15 @@ void packaged_task<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __args) {
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __p_.set_value(__f_(std::forward<_ArgTypes>(__args)...));
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class... _ArgTypes>
@@ -1673,15 +1673,15 @@ void packaged_task<_Rp(_ArgTypes...)>::make_ready_at_thread_exit(_ArgTypes... __
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __p_.set_value_at_thread_exit(__f_(std::forward<_ArgTypes>(__args)...));
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception_at_thread_exit(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _Rp, class... _ArgTypes>
@@ -1758,16 +1758,16 @@ void packaged_task<void(_ArgTypes...)>::operator()(_ArgTypes... __args) {
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __f_(std::forward<_ArgTypes>(__args)...);
     __p_.set_value();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class... _ArgTypes>
@@ -1776,16 +1776,16 @@ void packaged_task<void(_ArgTypes...)>::make_ready_at_thread_exit(_ArgTypes... _
     __throw_future_error(future_errc::no_state);
   if (__p_.__state_->__has_value())
     __throw_future_error(future_errc::promise_already_satisfied);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __f_(std::forward<_ArgTypes>(__args)...);
     __p_.set_value_at_thread_exit();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __p_.set_exception_at_thread_exit(current_exception());
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class... _ArgTypes>
@@ -1857,13 +1857,13 @@ async(launch __policy, _Fp&& __f, _Args&&... __args) {
   typedef __async_func<__decay_t<_Fp>, __decay_t<_Args>...> _BF;
   typedef typename _BF::_Rp _Rp;
 
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   try {
 #    endif
     if (__does_policy_contain(__policy, launch::async))
       return std::__make_async_assoc_state<_Rp>(
           _BF(_LIBCPP_AUTO_CAST(std::forward<_Fp>(__f)), _LIBCPP_AUTO_CAST(std::forward<_Args>(__args))...));
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     if (__policy == launch::async)
       throw;
diff --git a/libcxx/include/iomanip b/libcxx/include/iomanip
index 2c4023d4ee44a15856383ad006776a1c2cf27071..538fc413bb8ee24821375a168d9a83745048a42c 100644
--- a/libcxx/include/iomanip
+++ b/libcxx/include/iomanip
@@ -236,9 +236,9 @@ public:
 template <class _CharT, class _Traits, class _MoneyT>
 _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
 operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t7<_MoneyT>& __x) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_istream<_CharT, _Traits>::sentry __s(__is);
     if (__s) {
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
@@ -248,11 +248,11 @@ operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t7<_MoneyT>& __x) {
       __mf.get(_Ip(__is), _Ip(), __x.__intl_, __is, __err, __x.__mon_);
       __is.setstate(__err);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __is.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return __is;
 }
 
@@ -285,9 +285,9 @@ public:
 template <class _CharT, class _Traits, class _MoneyT>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t8<_MoneyT>& __x) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
     if (__s) {
       typedef ostreambuf_iterator<_CharT, _Traits> _Op;
@@ -296,11 +296,11 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t8<_MoneyT>& __x) {
       if (__mf.put(_Op(__os), __x.__intl_, __os, __os.fill(), __x.__mon_).failed())
         __os.setstate(ios_base::badbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return __os;
 }
 
@@ -333,9 +333,9 @@ public:
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>&
 operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t9<_CharT>& __x) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_istream<_CharT, _Traits>::sentry __s(__is);
     if (__s) {
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
@@ -345,11 +345,11 @@ operator>>(basic_istream<_CharT, _Traits>& __is, const __iom_t9<_CharT>& __x) {
       __tf.get(_Ip(__is), _Ip(), __is, __err, __x.__tm_, __x.__fmt_, __x.__fmt_ + _Traits::length(__x.__fmt_));
       __is.setstate(__err);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __is.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return __is;
 }
 
@@ -382,9 +382,9 @@ public:
 template <class _CharT, class _Traits>
 _LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t10<_CharT>& __x) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     typename basic_ostream<_CharT, _Traits>::sentry __s(__os);
     if (__s) {
       typedef ostreambuf_iterator<_CharT, _Traits> _Op;
@@ -394,11 +394,11 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const __iom_t10<_CharT>& __x) {
               .failed())
         __os.setstate(ios_base::badbit);
     }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __os.__set_badbit_and_consider_rethrow();
   }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   return __os;
 }
 
diff --git a/libcxx/include/ios b/libcxx/include/ios
index d4f15a269a11a6546f82184a2e3168e687fa7c24..546c5462f222dbdf3e94b70c3fdfc6d3929091b0 100644
--- a/libcxx/include/ios
+++ b/libcxx/include/ios
@@ -232,7 +232,7 @@ storage-class-specifier const error_category& iostream_category() noexcept;
 // [ios.syn]
 #  include <iosfwd>
 
-#  if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#  if _LIBCPP_HAS_ATOMIC_HEADER
 #    include <__atomic/atomic.h> // for __xindex_
 #  endif
 
@@ -441,7 +441,7 @@ public:
 };
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_failure(char const* __msg) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw ios_base::failure(__msg);
 #  else
   _LIBCPP_VERBOSE_ABORT("ios_base::failure was thrown in -fno-exceptions mode with message \"%s\"", __msg);
diff --git a/libcxx/include/iosfwd b/libcxx/include/iosfwd
index eeafcc37c598ef216ea85fbc54d0e28b9f1e428e..9079ce21512f90571a4c3a3b7ad9f53e0d671305 100644
--- a/libcxx/include/iosfwd
+++ b/libcxx/include/iosfwd
@@ -134,7 +134,7 @@ typedef fpos<mbstate_t> streampos;
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
 typedef fpos<mbstate_t> wstreampos;
 #endif
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 typedef fpos<mbstate_t> u8streampos;
 #endif
 typedef fpos<mbstate_t> u16streampos;
diff --git a/libcxx/include/istream b/libcxx/include/istream
index 8ee29ba3302f2fa2da5ff12d5d96e59cc7f12c3f..45ca95369c51d3ce29aca27ba2c5d2352d107625 100644
--- a/libcxx/include/istream
+++ b/libcxx/include/istream
@@ -357,13 +357,13 @@ __input_arithmetic(basic_istream<_CharT, _Traits>& __is, _Tp& __n) {
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __s(__is);
   if (__s) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
       typedef num_get<_CharT, _Ip> _Fp;
       std::use_facet<_Fp>(__is.getloc()).get(_Ip(__is), _Ip(), __is, __state, __n);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -438,9 +438,9 @@ __input_arithmetic_with_numeric_limits(basic_istream<_CharT, _Traits>& __is, _Tp
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __s(__is);
   if (__s) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       typedef istreambuf_iterator<_CharT, _Traits> _Ip;
       typedef num_get<_CharT, _Ip> _Fp;
       long __temp;
@@ -454,7 +454,7 @@ __input_arithmetic_with_numeric_limits(basic_istream<_CharT, _Traits>& __is, _Tp
       } else {
         __n = static_cast<_Tp>(__temp);
       }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -462,7 +462,7 @@ __input_arithmetic_with_numeric_limits(basic_istream<_CharT, _Traits>& __is, _Tp
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __is.setstate(__state);
   }
   return __is;
@@ -484,7 +484,7 @@ __input_c_string(basic_istream<_CharT, _Traits>& __is, _CharT* __p, size_t __n)
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
 #  endif
       _CharT* __s               = __p;
@@ -505,7 +505,7 @@ __input_c_string(basic_istream<_CharT, _Traits>& __is, _CharT* __p, size_t __n)
       __is.width(0);
       if (__s == __p)
         __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -572,7 +572,7 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& operator>>(basic_istream<_
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
 #  endif
       typename _Traits::int_type __i = __is.rdbuf()->sbumpc();
@@ -580,7 +580,7 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& operator>>(basic_istream<_
         __state |= ios_base::eofbit | ios_base::failbit;
       else
         __c = _Traits::to_char_type(__i);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -614,9 +614,9 @@ basic_istream<_CharT, _Traits>::operator>>(basic_streambuf<char_type, traits_typ
   sentry __s(*this, true);
   if (__s) {
     if (__sb) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
         while (true) {
           typename traits_type::int_type __i = this->rdbuf()->sgetc();
           if (traits_type::eq_int_type(__i, _Traits::eof())) {
@@ -630,7 +630,7 @@ basic_istream<_CharT, _Traits>::operator>>(basic_streambuf<char_type, traits_typ
         }
         if (__gc_ == 0)
           __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         __state |= ios_base::badbit;
         if (__gc_ == 0)
@@ -641,7 +641,7 @@ basic_istream<_CharT, _Traits>::operator>>(basic_streambuf<char_type, traits_typ
           throw;
         }
       }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     } else {
       __state |= ios_base::failbit;
     }
@@ -657,7 +657,7 @@ typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>
   int_type __r              = traits_type::eof();
   sentry __s(*this, true);
   if (__s) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
 #  endif
       __r = this->rdbuf()->sbumpc();
@@ -665,7 +665,7 @@ typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>
         __state |= ios_base::failbit | ios_base::eofbit;
       else
         __gc_ = 1;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       this->__setstate_nothrow(this->rdstate() | ios_base::badbit);
       if (this->exceptions() & ios_base::badbit) {
@@ -685,7 +685,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::get(char_type* _
   sentry __sen(*this, true);
   if (__sen) {
     if (__n > 0) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       try {
 #  endif
         while (__gc_ < __n - 1) {
@@ -703,7 +703,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::get(char_type* _
         }
         if (__gc_ == 0)
           __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         __state |= ios_base::badbit;
         this->__setstate_nothrow(__state);
@@ -734,9 +734,9 @@ basic_istream<_CharT, _Traits>::get(basic_streambuf<char_type, traits_type>& __s
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       while (true) {
         typename traits_type::int_type __i = this->rdbuf()->sgetc();
         if (traits_type::eq_int_type(__i, traits_type::eof())) {
@@ -751,12 +751,12 @@ basic_istream<_CharT, _Traits>::get(basic_streambuf<char_type, traits_type>& __s
         __inc_gcount();
         this->rdbuf()->sbumpc();
       }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       // according to the spec, exceptions here are caught but not rethrown
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     if (__gc_ == 0)
       __state |= ios_base::failbit;
     this->setstate(__state);
@@ -771,9 +771,9 @@ basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_typ
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       while (true) {
         typename traits_type::int_type __i = this->rdbuf()->sgetc();
         if (traits_type::eq_int_type(__i, traits_type::eof())) {
@@ -794,7 +794,7 @@ basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_typ
         this->rdbuf()->sbumpc();
         __inc_gcount();
       }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -806,7 +806,7 @@ basic_istream<_CharT, _Traits>::getline(char_type* __s, streamsize __n, char_typ
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
   if (__n > 0)
     *__s = char_type();
@@ -822,9 +822,9 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::ignore(streamsiz
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       if (__n == numeric_limits<streamsize>::max()) {
         while (true) {
           typename traits_type::int_type __i = this->rdbuf()->sbumpc();
@@ -848,7 +848,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::ignore(streamsiz
             break;
         }
       }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -856,7 +856,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::ignore(streamsiz
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return *this;
@@ -869,13 +869,13 @@ typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>
   int_type __r              = traits_type::eof();
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       __r = this->rdbuf()->sgetc();
       if (traits_type::eq_int_type(__r, traits_type::eof()))
         __state |= ios_base::eofbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -883,7 +883,7 @@ typename basic_istream<_CharT, _Traits>::int_type basic_istream<_CharT, _Traits>
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return __r;
@@ -895,13 +895,13 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::read(char_type*
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       __gc_ = this->rdbuf()->sgetn(__s, __n);
       if (__gc_ != __n)
         __state |= ios_base::failbit | ios_base::eofbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -909,7 +909,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::read(char_type*
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -923,9 +923,9 @@ streamsize basic_istream<_CharT, _Traits>::readsome(char_type* __s, streamsize _
   __gc_                     = 0;
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       streamsize __c = this->rdbuf()->in_avail();
       switch (__c) {
       case -1:
@@ -940,7 +940,7 @@ streamsize basic_istream<_CharT, _Traits>::readsome(char_type* __s, streamsize _
           __state |= ios_base::failbit | ios_base::eofbit;
         break;
       }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -948,7 +948,7 @@ streamsize basic_istream<_CharT, _Traits>::readsome(char_type* __s, streamsize _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -963,12 +963,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::putback(char_typ
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf() == nullptr || this->rdbuf()->sputbackc(__c) == traits_type::eof())
         __state |= ios_base::badbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -976,7 +976,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::putback(char_typ
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -991,12 +991,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::unget() {
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf() == nullptr || this->rdbuf()->sungetc() == traits_type::eof())
         __state |= ios_base::badbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1004,7 +1004,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::unget() {
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   } else {
     __state |= ios_base::failbit;
   }
@@ -1021,14 +1021,14 @@ int basic_istream<_CharT, _Traits>::sync() {
 
   int __r = 0;
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf()->pubsync() == -1) {
         __state |= ios_base::badbit;
         __r = -1;
       }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1036,7 +1036,7 @@ int basic_istream<_CharT, _Traits>::sync() {
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return __r;
@@ -1048,11 +1048,11 @@ typename basic_istream<_CharT, _Traits>::pos_type basic_istream<_CharT, _Traits>
   pos_type __r(-1);
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       __r = this->rdbuf()->pubseekoff(0, ios_base::cur, ios_base::in);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1060,7 +1060,7 @@ typename basic_istream<_CharT, _Traits>::pos_type basic_istream<_CharT, _Traits>
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return __r;
@@ -1072,12 +1072,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(pos_type _
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf()->pubseekpos(__pos, ios_base::in) == pos_type(-1))
         __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1085,7 +1085,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(pos_type _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return *this;
@@ -1097,12 +1097,12 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(off_type _
   this->clear(__state);
   sentry __sen(*this, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       if (this->rdbuf()->pubseekoff(__off, __dir, ios_base::in) == pos_type(-1))
         __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       this->__setstate_nothrow(__state);
@@ -1110,7 +1110,7 @@ basic_istream<_CharT, _Traits>& basic_istream<_CharT, _Traits>::seekg(off_type _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     this->setstate(__state);
   }
   return *this;
@@ -1121,9 +1121,9 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& ws(basic_istream<_CharT, _
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       const ctype<_CharT>& __ct = std::use_facet<ctype<_CharT> >(__is.getloc());
       while (true) {
         typename _Traits::int_type __i = __is.rdbuf()->sgetc();
@@ -1135,7 +1135,7 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& ws(basic_istream<_CharT, _
           break;
         __is.rdbuf()->sbumpc();
       }
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -1143,7 +1143,7 @@ _LIBCPP_HIDE_FROM_ABI basic_istream<_CharT, _Traits>& ws(basic_istream<_CharT, _
         throw;
       }
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     __is.setstate(__state);
   }
   return __is;
@@ -1211,7 +1211,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
 #  endif
       __str.clear();
@@ -1243,7 +1243,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _
       __is.width(0);
       if (__c == 0)
         __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -1263,7 +1263,7 @@ getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _All
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is, true);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
 #  endif
       __str.clear();
@@ -1286,7 +1286,7 @@ getline(basic_istream<_CharT, _Traits>& __is, basic_string<_CharT, _Traits, _All
       }
       if (__extr == 0)
         __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
@@ -1324,7 +1324,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x) {
   ios_base::iostate __state = ios_base::goodbit;
   typename basic_istream<_CharT, _Traits>::sentry __sen(__is);
   if (__sen) {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
 #  endif
       basic_string<_CharT, _Traits> __str;
@@ -1348,7 +1348,7 @@ operator>>(basic_istream<_CharT, _Traits>& __is, bitset<_Size>& __x) {
       __x = bitset<_Size>(__str);
       if (_Size > 0 && __c == 0)
         __state |= ios_base::failbit;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __state |= ios_base::badbit;
       __is.__setstate_nothrow(__state);
diff --git a/libcxx/include/list b/libcxx/include/list
index 05234f7696c6f4a1db875ae5be4f0aa5922b4b51..4a169b08d8cddbb62292f4cf3e487d3c0efb9fab 100644
--- a/libcxx/include/list
+++ b/libcxx/include/list
@@ -225,6 +225,7 @@ template <class T, class Allocator, class Predicate>
 #include <__ranges/container_compatible_range.h>
 #include <__ranges/from_range.h>
 #include <__type_traits/conditional.h>
+#include <__type_traits/container_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_nothrow_assignable.h>
@@ -1157,13 +1158,13 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _
     ++__ds;
     __r          = iterator(__node->__as_link());
     iterator __e = __r;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, __x)->__as_link();
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1175,7 +1176,7 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _
       }
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
     base::__sz() += __ds;
   }
@@ -1199,13 +1200,13 @@ list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Se
     ++__ds;
     __r          = iterator(__node->__as_link());
     iterator __e = __r;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (++__f; __f != __l; ++__f, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, *__f)->__as_link();
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1217,7 +1218,7 @@ list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Se
       }
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
     base::__sz() += __ds;
   }
@@ -1370,13 +1371,13 @@ void list<_Tp, _Alloc>::resize(size_type __n) {
     ++__ds;
     iterator __r = iterator(__node->__as_link());
     iterator __e = __r;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr)->__as_link();
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1388,7 +1389,7 @@ void list<_Tp, _Alloc>::resize(size_type __n) {
       }
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes_at_back(__r.__ptr_, __e.__ptr_);
     base::__sz() += __ds;
   }
@@ -1406,13 +1407,13 @@ void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
     __base_pointer __nl = __node->__as_link();
     iterator __r        = iterator(__nl);
     iterator __e        = __r;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
         __e.__ptr_->__next_ = this->__create_node(/* prev = */ __e.__ptr_, /* next = */ nullptr, __x)->__as_link();
       }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       while (true) {
         __base_pointer __prev    = __e.__ptr_->__prev_;
@@ -1424,7 +1425,7 @@ void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
       }
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __link_nodes(base::__end_as_link(), __r.__ptr_, __e.__ptr_);
     base::__sz() += __ds;
   }
@@ -1715,6 +1716,17 @@ inline constexpr bool __format::__enable_insertable<std::list<wchar_t>> = true;
 
 #endif // _LIBCPP_STD_VER >= 20
 
+template <class _Tp, class _Allocator>
+struct __container_traits<list<_Tp, _Allocator> > {
+  // http://eel.is/c++draft/container.reqmts
+  // Unless otherwise specified (see [associative.reqmts.except], [unord.req.except], [deque.modifiers],
+  // [inplace.vector.modifiers], and [vector.modifiers]) all container types defined in this Clause meet the following
+  // additional requirements:
+  // - If an exception is thrown by an insert() or emplace() function while inserting a single element, that
+  // function has no effects.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/locale b/libcxx/include/locale
index 573910a85bef540f48cf24f98a7a0226c1ae778e..782475ea7e0eb845e2191f4868ea701e5b2a99a7 100644
--- a/libcxx/include/locale
+++ b/libcxx/include/locale
@@ -223,7 +223,11 @@ template <class charT> class messages_byname;
 #    if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
 #      define _LIBCPP_HAS_CATOPEN 1
 #      include <nl_types.h>
+#    else
+#      define _LIBCPP_HAS_CATOPEN 0
 #    endif
+#  else
+#    define _LIBCPP_HAS_CATOPEN 0
 #  endif
 
 #  ifdef _LIBCPP_LOCALE__L_EXTENSIONS
@@ -3074,7 +3078,7 @@ locale::id messages<_CharT>::id;
 
 template <class _CharT>
 typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<char>& __nm, const locale&) const {
-#  ifdef _LIBCPP_HAS_CATOPEN
+#  if _LIBCPP_HAS_CATOPEN
   return (catalog)catopen(__nm.c_str(), NL_CAT_LOCALE);
 #  else  // !_LIBCPP_HAS_CATOPEN
   (void)__nm;
@@ -3085,7 +3089,7 @@ typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<
 template <class _CharT>
 typename messages<_CharT>::string_type
 messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
-#  ifdef _LIBCPP_HAS_CATOPEN
+#  if _LIBCPP_HAS_CATOPEN
   string __ndflt;
   __narrow_to_utf8<sizeof(char_type) * __CHAR_BIT__>()(
       std::back_inserter(__ndflt), __dflt.c_str(), __dflt.c_str() + __dflt.size());
@@ -3105,7 +3109,7 @@ messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type&
 
 template <class _CharT>
 void messages<_CharT>::do_close(catalog __c) const {
-#  ifdef _LIBCPP_HAS_CATOPEN
+#  if _LIBCPP_HAS_CATOPEN
   catclose((nl_catd)__c);
 #  else  // !_LIBCPP_HAS_CATOPEN
   (void)__c;
diff --git a/libcxx/include/map b/libcxx/include/map
index 7fca4c8a0872fd5535077b2ec86f329b151408f4..fabf39512ab75fa4770b675bdbccd7edfa92b1eb 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -594,6 +594,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 #include <__ranges/container_compatible_range.h>
 #include <__ranges/from_range.h>
 #include <__tree>
+#include <__type_traits/container_traits.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/remove_const.h>
 #include <__type_traits/type_identity.h>
@@ -1644,6 +1645,14 @@ erase_if(map<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) {
 }
 #endif
 
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+struct __container_traits<map<_Key, _Tp, _Compare, _Allocator> > {
+  // http://eel.is/c++draft/associative.reqmts.except#2
+  // For associative containers, if an exception is thrown by any operation from within
+  // an insert or emplace function inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+};
+
 template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
 class _LIBCPP_TEMPLATE_VIS multimap {
 public:
@@ -2158,6 +2167,14 @@ erase_if(multimap<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) {
 }
 #endif
 
+template <class _Key, class _Tp, class _Compare, class _Allocator>
+struct __container_traits<multimap<_Key, _Tp, _Compare, _Allocator> > {
+  // http://eel.is/c++draft/associative.reqmts.except#2
+  // For associative containers, if an exception is thrown by any operation from within
+  // an insert or emplace function inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 3ea91274a9cc9a19f0b0bb1db6429d29fa65d90c..5a0e199394d01852cc2fab92c0122b9f5672eae2 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -79,6 +79,7 @@ module std_core [system] {
     }
     module conditional                                { header "__type_traits/conditional.h" }
     module conjunction                                { header "__type_traits/conjunction.h" }
+    module container_traits                           { header "__type_traits/container_traits.h" }
     module copy_cv                                    { header "__type_traits/copy_cv.h" }
     module copy_cvref                                 { header "__type_traits/copy_cvref.h" }
     module datasizeof                                 { header "__type_traits/datasizeof.h" }
diff --git a/libcxx/include/new b/libcxx/include/new
index 3252b0bb1abcdb2dc057c98a724d6f964ac704b8..2947ee179510a5af26ebf28a2d0914b6318be860 100644
--- a/libcxx/include/new
+++ b/libcxx/include/new
@@ -103,16 +103,22 @@ void  operator delete[](void* ptr, void*) noexcept;
 #  pragma GCC system_header
 #endif
 
-#if !defined(__cpp_sized_deallocation) || __cpp_sized_deallocation < 201309L
-#  define _LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION
+#if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
+#  define _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION 1
+#else
+#  define _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION 0
 #endif
 
-#if !defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_STD_VER < 14 && defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
-#  define _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+#if _LIBCPP_STD_VER >= 14 || _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION
+#  define _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION 1
+#else
+#  define _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION 0
 #endif
 
-#if defined(_LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION) || defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
-#  define _LIBCPP_HAS_NO_SIZED_DEALLOCATION
+#if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION && _LIBCPP_HAS_LANGUAGE_SIZED_DEALLOCATION
+#  define _LIBCPP_HAS_SIZED_DEALLOCATION 1
+#else
+#  define _LIBCPP_HAS_SIZED_DEALLOCATION 0
 #endif
 
 namespace std // purposefully not using versioning namespace
@@ -169,14 +175,14 @@ public:
 [[__noreturn__]] _LIBCPP_EXPORTED_FROM_ABI void __throw_bad_alloc(); // not in C++ spec
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_array_new_length() {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw bad_array_new_length();
 #else
   _LIBCPP_VERBOSE_ABORT("bad_array_new_length was thrown in -fno-exceptions mode");
 #endif
 }
 
-#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) && !defined(_LIBCPP_ABI_VCRUNTIME)
+#if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION && !defined(_LIBCPP_ABI_VCRUNTIME)
 #  ifndef _LIBCPP_CXX03_LANG
 enum class align_val_t : size_t {};
 #  else
@@ -208,7 +214,7 @@ inline constexpr destroying_delete_t destroying_delete{};
     _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t&) _NOEXCEPT;
-#  ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+#  if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz) _NOEXCEPT;
 #  endif
 
@@ -217,17 +223,17 @@ _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz) _
     _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t&) _NOEXCEPT;
-#  ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+#  if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz) _NOEXCEPT;
 #  endif
 
-#  ifndef _LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#  if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void* operator new(std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
 [[__nodiscard__]] _LIBCPP_OVERRIDABLE_FUNC_VIS void*
 operator new(std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
-#    ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+#    if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
 #    endif
 
@@ -237,7 +243,7 @@ operator new[](std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
 operator new[](std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t) _NOEXCEPT;
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
-#    ifndef _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
+#    if _LIBCPP_HAS_LIBRARY_SIZED_DEALLOCATION
 _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
 #    endif
 #  endif
@@ -284,7 +290,7 @@ _LIBCPP_HIDE_FROM_ABI void __libcpp_operator_delete(_Args... __args) {
 }
 
 inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_allocate(size_t __size, size_t __align) {
-#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+#if _LIBCPP_HAS_ALIGNED_ALLOCATION
   if (__is_overaligned_for_new(__align)) {
     const align_val_t __align_val = static_cast<align_val_t>(__align);
     return __libcpp_operator_new(__size, __align_val);
@@ -297,7 +303,7 @@ inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_allocate(size_t __size, size_t __ali
 
 template <class... _Args>
 _LIBCPP_HIDE_FROM_ABI void __do_deallocate_handle_size(void* __ptr, size_t __size, _Args... __args) {
-#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION
+#if !_LIBCPP_HAS_SIZED_DEALLOCATION
   (void)__size;
   return std::__libcpp_operator_delete(__ptr, __args...);
 #else
@@ -306,7 +312,7 @@ _LIBCPP_HIDE_FROM_ABI void __do_deallocate_handle_size(void* __ptr, size_t __siz
 }
 
 inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate(void* __ptr, size_t __size, size_t __align) {
-#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+#if !_LIBCPP_HAS_ALIGNED_ALLOCATION
   (void)__align;
   return __do_deallocate_handle_size(__ptr, __size);
 #else
@@ -320,7 +326,7 @@ inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate(void* __ptr, size_t __size
 }
 
 inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate_unsized(void* __ptr, size_t __align) {
-#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+#if !_LIBCPP_HAS_ALIGNED_ALLOCATION
   (void)__align;
   return __libcpp_operator_delete(__ptr);
 #else
diff --git a/libcxx/include/optional b/libcxx/include/optional
index 4e44ef990f5d29761004d36c70de18d4b0807ad6..b9dcf9053633f062ca7050d9e81084ca357ebf1f 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -260,7 +260,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 [[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_OPTIONAL_ACCESS void
 __throw_bad_optional_access() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_optional_access();
 #  else
   _LIBCPP_VERBOSE_ABORT("bad_optional_access was thrown in -fno-exceptions mode");
diff --git a/libcxx/include/regex b/libcxx/include/regex
index d6b8768a257e3403536956100aebd3544a19e2bf..074b19865cb4ff93ea41f5021323be30c69ca54b 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -984,7 +984,7 @@ public:
 
 template <regex_constants::error_type _Ev>
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_regex_error() {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw regex_error(_Ev);
 #else
   _LIBCPP_VERBOSE_ABORT("regex_error was thrown in -fno-exceptions mode");
diff --git a/libcxx/include/set b/libcxx/include/set
index 0c2ca64139e0d392b27d4aa876ae973d46536570..5db0db8086dec6a6aa60805a4470fb7ab3a156ca 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -531,6 +531,7 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
 #include <__ranges/container_compatible_range.h>
 #include <__ranges/from_range.h>
 #include <__tree>
+#include <__type_traits/container_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_nothrow_assignable.h>
@@ -1022,6 +1023,14 @@ erase_if(set<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
 }
 #endif
 
+template <class _Key, class _Compare, class _Allocator>
+struct __container_traits<set<_Key, _Compare, _Allocator> > {
+  // http://eel.is/c++draft/associative.reqmts.except#2
+  // For associative containers, if an exception is thrown by any operation from within
+  // an insert or emplace function inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+};
+
 template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
 class _LIBCPP_TEMPLATE_VIS multiset {
 public:
@@ -1481,6 +1490,14 @@ erase_if(multiset<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
 }
 #endif
 
+template <class _Key, class _Compare, class _Allocator>
+struct __container_traits<multiset<_Key, _Compare, _Allocator> > {
+  // http://eel.is/c++draft/associative.reqmts.except#2
+  // For associative containers, if an exception is thrown by any operation from within
+  // an insert or emplace function inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee = true;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/sstream b/libcxx/include/sstream
index c29abab9c5fc3bf8b6a912b9f8035ecae0b30424..df039194996d74c727775d50e4118573b0e7fe4a 100644
--- a/libcxx/include/sstream
+++ b/libcxx/include/sstream
@@ -786,9 +786,9 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::overflow(int_type __c) {
     if (this->pptr() == this->epptr()) {
       if (!(__mode_ & ios_base::out))
         return traits_type::eof();
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
         ptrdiff_t __nout = this->pptr() - this->pbase();
         ptrdiff_t __hm   = __hm_ - this->pbase();
         __str_.push_back(char_type());
@@ -797,11 +797,11 @@ basic_stringbuf<_CharT, _Traits, _Allocator>::overflow(int_type __c) {
         this->setp(__p, __p + __str_.size());
         this->__pbump(__nout);
         __hm_ = this->pbase() + __hm;
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         return traits_type::eof();
       }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
     }
     __hm_ = std::max(this->pptr() + 1, __hm_);
     if (__mode_ & ios_base::in) {
diff --git a/libcxx/include/stdatomic.h b/libcxx/include/stdatomic.h
index 3206f7da8c8c459805bd32532175c3ef2a04b8cb..2f25b057df0874ab435e13b4ed258b1bd8434071 100644
--- a/libcxx/include/stdatomic.h
+++ b/libcxx/include/stdatomic.h
@@ -156,7 +156,7 @@ using std::atomic_long _LIBCPP_USING_IF_EXISTS;
 using std::atomic_ulong _LIBCPP_USING_IF_EXISTS;
 using std::atomic_llong _LIBCPP_USING_IF_EXISTS;
 using std::atomic_ullong _LIBCPP_USING_IF_EXISTS;
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 using std::atomic_char8_t _LIBCPP_USING_IF_EXISTS;
 #  endif
 using std::atomic_char16_t _LIBCPP_USING_IF_EXISTS;
diff --git a/libcxx/include/stdexcept b/libcxx/include/stdexcept
index bdfc27aeac374291f3ad9102c7d3bb9ef4732431..daa7b501a86999b3bf80dfb2d04986c864ee28de 100644
--- a/libcxx/include/stdexcept
+++ b/libcxx/include/stdexcept
@@ -212,7 +212,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 [[__noreturn__]] _LIBCPP_EXPORTED_FROM_ABI void __throw_runtime_error(const char*);
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_logic_error(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw logic_error(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("logic_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
@@ -220,7 +220,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_domain_error(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw domain_error(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("domain_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
@@ -228,7 +228,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_invalid_argument(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw invalid_argument(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("invalid_argument was thrown in -fno-exceptions mode with message \"%s\"", __msg);
@@ -236,7 +236,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_length_error(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw length_error(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("length_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
@@ -244,7 +244,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_out_of_range(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw out_of_range(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("out_of_range was thrown in -fno-exceptions mode with message \"%s\"", __msg);
@@ -252,7 +252,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_range_error(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw range_error(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("range_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
@@ -260,7 +260,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_overflow_error(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw overflow_error(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("overflow_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
@@ -268,7 +268,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 }
 
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_underflow_error(const char* __msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw underflow_error(__msg);
 #else
   _LIBCPP_VERBOSE_ABORT("underflow_error was thrown in -fno-exceptions mode with message \"%s\"", __msg);
diff --git a/libcxx/include/string b/libcxx/include/string
index 685dbcbd4418e8a15f5a6c8c8d2a968e98f0919e..4b5017f5e7753f45661c43bcfdcd919167aa7007 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -672,7 +672,7 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
 _LIBCPP_PUSH_MACROS
 #include <__undef_macros>
 
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
 #  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS __attribute__((__no_sanitize__("address")))
 // This macro disables AddressSanitizer (ASan) instrumentation for a specific function,
 // allowing memory accesses that would normally trigger ASan errors to proceed without crashing.
@@ -785,7 +785,7 @@ public:
   //
   // This string implementation doesn't contain any references into itself. It only contains a bit that says whether
   // it is in small or large string mode, so the entire structure is trivially relocatable if its members are.
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
   // When compiling with AddressSanitizer (ASan), basic_string cannot be trivially
   // relocatable. Because the object's memory might be poisoned when its content
   // is kept inside objects memory (short string optimization), instead of in allocated
@@ -799,7 +799,8 @@ public:
       basic_string,
       void>;
 #endif
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __asan_volatile_wrapper(pointer const& __ptr) const {
     if (__libcpp_is_constant_evaluated())
       return __ptr;
@@ -2018,7 +2019,7 @@ private:
   __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
     (void)__old_mid;
     (void)__new_mid;
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
 #  if defined(__APPLE__)
     // TODO: remove after addressing issue #96099 (https://github.com/llvm/llvm-project/issues/96099)
     if (!__is_long())
@@ -2030,14 +2031,14 @@ private:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_new(size_type __current_size) const _NOEXCEPT {
     (void)__current_size;
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + capacity() + 1, data() + __current_size + 1);
 #endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_delete() const _NOEXCEPT {
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + size() + 1, data() + capacity() + 1);
 #endif
@@ -2045,7 +2046,7 @@ private:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_increase(size_type __n) const _NOEXCEPT {
     (void)__n;
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + size() + 1, data() + size() + 1 + __n);
 #endif
@@ -2053,7 +2054,7 @@ private:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
     (void)__old_size;
-#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if _LIBCPP_HAS_ASAN && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
     if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + __old_size + 1, data() + size() + 1);
 #endif
@@ -2416,19 +2417,19 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputIterator _
   __rep_ = __rep();
   __annotate_new(0);
 
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     for (; __first != __last; ++__first)
       push_back(*__first);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     __annotate_delete();
     if (__is_long())
       __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
     throw;
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2463,18 +2464,18 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_size(_InputIterator __fir
     __set_long_size(__sz);
   }
 
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__p));
     traits_type::assign(*__end, value_type());
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     if (__is_long())
       __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
     throw;
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   __annotate_new(__sz);
 }
 
@@ -3376,9 +3377,9 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
       // Shrink
       // - called from shrink_to_fit should not throw.
       // - called from reserve may throw but is not required to.
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
       try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
         auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
 
         // The Standard mandates shrink_to_fit() does not increase the capacity.
@@ -3391,11 +3392,11 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
         }
         __new_data        = __allocation.ptr;
         __target_capacity = __allocation.count - 1;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         return;
       }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     }
     __begin_lifetime(__new_data, __target_capacity + 1);
     __now_long = true;
@@ -4234,7 +4235,7 @@ struct __string_hash : public __unary_function<basic_string<_CharT, char_traits<
 template <class _Allocator>
 struct hash<basic_string<char, char_traits<char>, _Allocator> > : __string_hash<char, _Allocator> {};
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 template <class _Allocator>
 struct hash<basic_string<char8_t, char_traits<char8_t>, _Allocator> > : __string_hash<char8_t, _Allocator> {};
 #endif
@@ -4308,7 +4309,7 @@ operator""s(const wchar_t* __str, size_t __len) {
 }
 #  endif
 
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string<char8_t> operator""s(const char8_t* __str, size_t __len) {
   return basic_string<char8_t>(__str, __len);
 }
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index 3b32117386388c0821a2bfc5e64587b4c6eec47a..5beac404fb5b54f51b947bde5cbcde9f8825f182 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -901,7 +901,7 @@ struct __string_view_hash : public __unary_function<basic_string_view<_CharT, ch
 template <>
 struct hash<basic_string_view<char, char_traits<char> > > : __string_view_hash<char> {};
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 template <>
 struct hash<basic_string_view<char8_t, char_traits<char8_t> > > : __string_view_hash<char8_t> {};
 #endif
@@ -931,7 +931,7 @@ operator""sv(const wchar_t* __str, size_t __len) noexcept {
 }
 #  endif
 
-#  ifndef _LIBCPP_HAS_NO_CHAR8_T
+#  if _LIBCPP_HAS_CHAR8_T
 inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char8_t>
 operator""sv(const char8_t* __str, size_t __len) noexcept {
   return basic_string_view<char8_t>(__str, __len);
diff --git a/libcxx/include/syncstream b/libcxx/include/syncstream
index a2e1c7edadc5defba0a1b1f14b92e02468d44894..72a5f6ca5c86c13acde7dc63c3d71ff40c5fe2ea 100644
--- a/libcxx/include/syncstream
+++ b/libcxx/include/syncstream
@@ -273,14 +273,14 @@ public:
   }
 
   _LIBCPP_HIDE_FROM_ABI ~basic_syncbuf() {
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     try {
-#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
       emit();
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
     }
-#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#    endif // _LIBCPP_HAS_EXCEPTIONS
     __dec_reference();
   }
 
@@ -337,7 +337,7 @@ protected:
       return traits_type::not_eof(__c);
 
     if (this->pptr() == this->epptr()) {
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       try {
 #    endif
         size_t __size = __str_.size();
@@ -348,7 +348,7 @@ protected:
         this->setp(__p, __p + __str_.size());
         this->pbump(__size);
 
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         return traits_type::eof();
       }
@@ -480,13 +480,13 @@ public:
     // TODO validate other unformatted output functions.
     typename basic_ostream<char_type, traits_type>::sentry __s(*this);
     if (__s) {
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       try {
 #    endif
 
         if (__sb_.emit() == false)
           this->setstate(ios::badbit);
-#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if _LIBCPP_HAS_EXCEPTIONS
       } catch (...) {
         this->__set_badbit_and_consider_rethrow();
       }
diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo
index a44fa4d73ee58092141d417729d784222e2a6150..252afe59a0aa6d5861142a2fd53171c98edd33cf 100644
--- a/libcxx/include/typeinfo
+++ b/libcxx/include/typeinfo
@@ -374,7 +374,7 @@ private:
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 [[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_cast() {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw bad_cast();
 #else
   _LIBCPP_VERBOSE_ABORT("bad_cast was thrown in -fno-exceptions mode");
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index 0d71a51ee9e72998d02d83eb2f4285c727a05a8f..05aa01a3b7c30835db30238286709dd153596c92 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -604,7 +604,9 @@ template <class Key, class T, class Hash, class Pred, class Alloc>
 #include <__ranges/concepts.h>
 #include <__ranges/container_compatible_range.h>
 #include <__ranges/from_range.h>
+#include <__type_traits/container_traits.h>
 #include <__type_traits/enable_if.h>
+#include <__type_traits/invoke.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/remove_const.h>
@@ -1830,6 +1832,16 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_map<_Key, _Tp, _Has
 
 #endif
 
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+struct __container_traits<unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc> > {
+  // http://eel.is/c++draft/unord.req.except#2
+  //  For unordered associative containers, if an exception is thrown by any operation
+  //  other than the container's hash function from within an insert or emplace function
+  //  inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
+      __nothrow_invokable<_Hash, const _Key&>::value;
+};
+
 template <class _Key,
           class _Tp,
           class _Hash  = hash<_Key>,
@@ -2520,6 +2532,16 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multimap<_Key, _Tp,
 
 #endif
 
+template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
+struct __container_traits<unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc> > {
+  // http://eel.is/c++draft/unord.req.except#2
+  //  For unordered associative containers, if an exception is thrown by any operation
+  //  other than the container's hash function from within an insert or emplace function
+  //  inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
+      __nothrow_invokable<_Hash, const _Key&>::value;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/unordered_set b/libcxx/include/unordered_set
index 2b09c72b866b03ce3b05337840d94bc54a7a75e5..7ab1c651b8c956579bae4c46254ae9669e235fcb 100644
--- a/libcxx/include/unordered_set
+++ b/libcxx/include/unordered_set
@@ -550,7 +550,9 @@ template <class Value, class Hash, class Pred, class Alloc>
 #include <__ranges/concepts.h>
 #include <__ranges/container_compatible_range.h>
 #include <__ranges/from_range.h>
+#include <__type_traits/container_traits.h>
 #include <__type_traits/enable_if.h>
+#include <__type_traits/invoke.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/is_nothrow_assignable.h>
@@ -1183,6 +1185,16 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_set<_Value, _Hash,
 
 #endif
 
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+struct __container_traits<unordered_set<_Value, _Hash, _Pred, _Alloc> > {
+  // http://eel.is/c++draft/unord.req.except#2
+  //  For unordered associative containers, if an exception is thrown by any operation
+  //  other than the container's hash function from within an insert or emplace function
+  //  inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
+      __nothrow_invokable<_Hash, const _Value&>::value;
+};
+
 template <class _Value, class _Hash = hash<_Value>, class _Pred = equal_to<_Value>, class _Alloc = allocator<_Value> >
 class _LIBCPP_TEMPLATE_VIS unordered_multiset {
 public:
@@ -1793,6 +1805,16 @@ inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const unordered_multiset<_Value, _H
 
 #endif
 
+template <class _Value, class _Hash, class _Pred, class _Alloc>
+struct __container_traits<unordered_multiset<_Value, _Hash, _Pred, _Alloc> > {
+  // http://eel.is/c++draft/unord.req.except#2
+  //  For unordered associative containers, if an exception is thrown by any operation
+  //  other than the container's hash function from within an insert or emplace function
+  //  inserting a single element, the insertion has no effect.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
+      __nothrow_invokable<_Hash, const _Value&>::value;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/valarray b/libcxx/include/valarray
index 44341eb2ba6c14d6c1d9765ab5c720b727504991..b3b48958f92bd76ff5a7cbf930428888fca209ae 100644
--- a/libcxx/include/valarray
+++ b/libcxx/include/valarray
@@ -1984,17 +1984,17 @@ template <class _Tp>
 inline valarray<_Tp>::valarray(size_t __n) : __begin_(nullptr), __end_(nullptr) {
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
         ::new ((void*)__end_) value_type();
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2007,17 +2007,17 @@ template <class _Tp>
 valarray<_Tp>::valarray(const value_type* __p, size_t __n) : __begin_(nullptr), __end_(nullptr) {
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (size_t __n_left = __n; __n_left; ++__end_, ++__p, --__n_left)
         ::new ((void*)__end_) value_type(*__p);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2025,17 +2025,17 @@ template <class _Tp>
 valarray<_Tp>::valarray(const valarray& __v) : __begin_(nullptr), __end_(nullptr) {
   if (__v.size()) {
     __begin_ = __end_ = allocator<value_type>().allocate(__v.size());
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (value_type* __p = __v.__begin_; __p != __v.__end_; ++__end_, ++__p)
         ::new ((void*)__end_) value_type(*__p);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__v.size());
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2051,18 +2051,18 @@ valarray<_Tp>::valarray(initializer_list<value_type> __il) : __begin_(nullptr),
   const size_t __n = __il.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     try {
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
       size_t __n_left = __n;
       for (const value_type* __p = __il.begin(); __n_left; ++__end_, ++__p, --__n_left)
         ::new ((void*)__end_) value_type(*__p);
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2073,18 +2073,18 @@ valarray<_Tp>::valarray(const slice_array<value_type>& __sa) : __begin_(nullptr)
   const size_t __n = __sa.__size_;
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       size_t __n_left = __n;
       for (const value_type* __p = __sa.__vp_; __n_left; ++__end_, __p += __sa.__stride_, --__n_left)
         ::new ((void*)__end_) value_type(*__p);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2093,19 +2093,19 @@ valarray<_Tp>::valarray(const gslice_array<value_type>& __ga) : __begin_(nullptr
   const size_t __n = __ga.__1d_.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       typedef const size_t* _Ip;
       const value_type* __s = __ga.__vp_;
       for (_Ip __i = __ga.__1d_.__begin_, __e = __ga.__1d_.__end_; __i != __e; ++__i, ++__end_)
         ::new ((void*)__end_) value_type(__s[*__i]);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2114,19 +2114,19 @@ valarray<_Tp>::valarray(const mask_array<value_type>& __ma) : __begin_(nullptr),
   const size_t __n = __ma.__1d_.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       typedef const size_t* _Ip;
       const value_type* __s = __ma.__vp_;
       for (_Ip __i = __ma.__1d_.__begin_, __e = __ma.__1d_.__end_; __i != __e; ++__i, ++__end_)
         ::new ((void*)__end_) value_type(__s[*__i]);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2135,19 +2135,19 @@ valarray<_Tp>::valarray(const indirect_array<value_type>& __ia) : __begin_(nullp
   const size_t __n = __ia.__1d_.size();
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       typedef const size_t* _Ip;
       const value_type* __s = __ia.__vp_;
       for (_Ip __i = __ia.__1d_.__begin_, __e = __ia.__1d_.__end_; __i != __e; ++__i, ++__end_)
         ::new ((void*)__end_) value_type(__s[*__i]);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2636,17 +2636,17 @@ void valarray<_Tp>::resize(size_t __n, value_type __x) {
   __clear(size());
   if (__n) {
     __begin_ = __end_ = allocator<value_type>().allocate(__n);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
         ::new ((void*)__end_) value_type(__x);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       __clear(__n);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
diff --git a/libcxx/include/variant b/libcxx/include/variant
index fd3aebaea0e14afbe39d158cc59cbd55086e797a..2e158a4eea31457e43493dcc22521ba26b009207 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -307,7 +307,7 @@ struct __farray {
 
 [[noreturn]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS void
 __throw_bad_variant_access() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_variant_access();
 #  else
   _LIBCPP_VERBOSE_ABORT("bad_variant_access was thrown in -fno-exceptions mode");
@@ -1062,7 +1062,7 @@ public:
         std::swap(__lhs, __rhs);
       }
       __impl __tmp(std::move(*__rhs));
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
       if constexpr (__all<is_nothrow_move_constructible_v<_Types>...>::value) {
         this->__generic_construct(*__rhs, std::move(*__lhs));
       } else {
@@ -1300,7 +1300,7 @@ public:
     __impl_.__swap(__that.__impl_);
   }
 
-#  if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_HAS_EXPLICIT_THIS_PARAMETER)
+#  if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
   // Helper class to implement [variant.visit]/10
   //   Constraints: The call to visit does not use an explicit template-argument-list
   //   that begins with a type template-argument.
diff --git a/libcxx/include/vector b/libcxx/include/vector
index 7d3aac5989a48c826631834171edbc93a15e1ccc..dc31f31838264cefab2809dba48ff59c5b33464b 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -356,6 +356,8 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
 #include <__ranges/size.h>
 #include <__split_buffer>
 #include <__type_traits/conditional.h>
+#include <__type_traits/container_traits.h>
+#include <__type_traits/disjunction.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_allocator.h>
 #include <__type_traits/is_constructible.h>
@@ -917,27 +919,27 @@ private:
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
     (void)__current_size;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_contiguous_container(data() + capacity(), data() + __current_size);
 #endif
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT {
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_contiguous_container(data() + size(), data() + capacity());
 #endif
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_increase(size_type __n) const _NOEXCEPT {
     (void)__n;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_contiguous_container(data() + size(), data() + size() + __n);
 #endif
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
     (void)__old_size;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
     __annotate_contiguous_container(data() + __old_size, data() + size());
 #endif
   }
@@ -945,14 +947,14 @@ private:
   struct _ConstructTransaction {
     _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(vector& __v, size_type __n)
         : __v_(__v), __pos_(__v.__end_), __new_end_(__v.__end_ + __n) {
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
       __v_.__annotate_increase(__n);
 #endif
     }
 
     _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() {
       __v_.__end_ = __pos_;
-#ifndef _LIBCPP_HAS_NO_ASAN
+#if _LIBCPP_HAS_ASAN
       if (__pos_ != __new_end_) {
         __v_.__annotate_shrink(__new_end_ - __v_.__begin_);
       }
@@ -1482,9 +1484,9 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::reserve(size_type __
 template <class _Tp, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
   if (capacity() > size()) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       allocator_type& __a = this->__alloc();
       __split_buffer<value_type, allocator_type&> __v(size(), size(), __a);
       // The Standard mandates shrink_to_fit() does not increase the capacity.
@@ -1492,10 +1494,10 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOE
       // due to swapping the elements.
       if (__v.capacity() < capacity())
         __swap_out_circular_buffer(__v);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -1733,21 +1735,21 @@ vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _Inpu
   }
   __split_buffer<value_type, allocator_type&> __v(__a);
   if (__first != __last) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
       difference_type __old_size = __old_last - this->__begin_;
       difference_type __old_p    = __p - this->__begin_;
       reserve(__recommend(size() + __v.size()));
       __p        = this->__begin_ + __old_p;
       __old_last = this->__begin_ + __old_size;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       erase(__make_iter(__old_last), end());
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
   __p = std::rotate(__p, __old_last, this->__end_);
   insert(__make_iter(__p), std::make_move_iterator(__v.begin()), std::make_move_iterator(__v.end()));
@@ -2190,18 +2192,18 @@ private:
   template <class _InputIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
   __init_with_sentinel(_InputIterator __first, _Sentinel __last) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __first != __last; ++__first)
         push_back(*__first);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       if (__begin_ != nullptr)
         __storage_traits::deallocate(__alloc(), __begin_, __cap());
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
   template <class _Iterator, class _Sentinel>
@@ -2648,14 +2650,14 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::reserve(size_type _
 template <class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::shrink_to_fit() _NOEXCEPT {
   if (__external_cap_to_internal(size()) > __cap()) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       vector(*this, allocator_type(__alloc())).swap(*this);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
 }
 
@@ -2744,21 +2746,21 @@ vector<bool, _Allocator>::__insert_with_sentinel(const_iterator __position, _Inp
   }
   vector __v(get_allocator());
   if (__first != __last) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       __v.__assign_with_sentinel(std::move(__first), std::move(__last));
       difference_type __old_size = static_cast<difference_type>(__old_end - begin());
       difference_type __old_p    = __p - begin();
       reserve(__recommend(size() + __v.size()));
       __p       = begin() + __old_p;
       __old_end = begin() + __old_size;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       erase(__old_end, end());
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   }
   __p = std::rotate(__p, __old_end, end());
   insert(__p, __v.begin(), __v.end());
@@ -3005,6 +3007,18 @@ public:
 };
 #endif // _LIBCPP_STD_VER >= 23
 
+template <class _Tp, class _Allocator>
+struct __container_traits<vector<_Tp, _Allocator> > {
+  // http://eel.is/c++draft/vector.modifiers#2
+  //  If an exception is thrown other than by the copy constructor, move constructor, assignment operator, or move
+  //  assignment operator of T or by any InputIterator operation, there are no effects. If an exception is thrown while
+  //  inserting a single element at the end and T is Cpp17CopyInsertable or is_nothrow_move_constructible_v<T> is true,
+  //  there are no effects. Otherwise, if an exception is thrown by the move constructor of a non-Cpp17CopyInsertable T,
+  //  the effects are unspecified.
+  static _LIBCPP_CONSTEXPR const bool __emplacement_has_strong_exception_safety_guarantee =
+      _Or<is_nothrow_move_constructible<_Tp>, __is_cpp17_copy_insertable<_Allocator> >::value;
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/version b/libcxx/include/version
index 88387e311636c208fe1e0af80e2ab9f798dd558f..5ab4f28a04d880e30bc99b7b03175988e699a31a 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -384,7 +384,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_bit_cast                             201806L
 # define __cpp_lib_bitops                               201907L
 # define __cpp_lib_bounded_array_traits                 201902L
-# if !defined(_LIBCPP_HAS_NO_CHAR8_T)
+# if _LIBCPP_HAS_CHAR8_T
 #   define __cpp_lib_char8_t                            201907L
 # endif
 # define __cpp_lib_concepts                             202002L
diff --git a/libcxx/modules/std.compat/cuchar.inc b/libcxx/modules/std.compat/cuchar.inc
index d1a511cadef18461122330f771b335de9d536e0a..119a8511252bfca8ac5c8797781c97e01902e97a 100644
--- a/libcxx/modules/std.compat/cuchar.inc
+++ b/libcxx/modules/std.compat/cuchar.inc
@@ -17,7 +17,7 @@ export {
   // size_t is conditionally here, but always present in cstddef.cppm. To avoid
   // conflicing declarations omit the using here.
 
-#if !defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8)
+#if _LIBCPP_HAS_C8RTOMB_MBRTOC8
   using ::mbrtoc8 _LIBCPP_USING_IF_EXISTS;
   using ::c8rtomb _LIBCPP_USING_IF_EXISTS;
 #endif
diff --git a/libcxx/modules/std.cppm.in b/libcxx/modules/std.cppm.in
index 0575774f6f42fdd449b0502a45eb89179e7deb8e..64ed8d4088cc018f7d3279739228fda9cee78720 100644
--- a/libcxx/modules/std.cppm.in
+++ b/libcxx/modules/std.cppm.in
@@ -20,7 +20,7 @@ module;
 #include <algorithm>
 #include <any>
 #include <array>
-#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
+#if _LIBCPP_HAS_ATOMIC_HEADER
 #  include <atomic>
 #endif
 #include <barrier>
diff --git a/libcxx/modules/std/atomic.inc b/libcxx/modules/std/atomic.inc
index e8cf90d012588df3cbe7f7cb8aff4d33735261c6..6ed0f7e9c4f32853e66afd1302a80ac6bd1ceb5f 100644
--- a/libcxx/modules/std/atomic.inc
+++ b/libcxx/modules/std/atomic.inc
@@ -60,7 +60,7 @@ export namespace std {
   using std::atomic_char _LIBCPP_USING_IF_EXISTS;
   using std::atomic_char16_t _LIBCPP_USING_IF_EXISTS;
   using std::atomic_char32_t _LIBCPP_USING_IF_EXISTS;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
   using std::atomic_char8_t _LIBCPP_USING_IF_EXISTS;
 #endif
   using std::atomic_int _LIBCPP_USING_IF_EXISTS;
diff --git a/libcxx/modules/std/cuchar.inc b/libcxx/modules/std/cuchar.inc
index fd80110d11ee8484b35405be2c54616644d15686..4a0b21c17b7e1299e75df76e75265c21d33f1195 100644
--- a/libcxx/modules/std/cuchar.inc
+++ b/libcxx/modules/std/cuchar.inc
@@ -17,7 +17,7 @@ export namespace std {
   // size_t is conditionally here, but always present in cstddef.cppm. To avoid
   // conflicing declarations omit the using here.
 
-#if !defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8)
+#if _LIBCPP_HAS_C8RTOMB_MBRTOC8
   using std::mbrtoc8 _LIBCPP_USING_IF_EXISTS;
   using std::c8rtomb _LIBCPP_USING_IF_EXISTS;
 #endif
diff --git a/libcxx/modules/std/iosfwd.inc b/libcxx/modules/std/iosfwd.inc
index 090d990d061e5eb77b1f7ebe99955d972d841371..952057d691fc40f511f6c304339ea6cc9a8863c8 100644
--- a/libcxx/modules/std/iosfwd.inc
+++ b/libcxx/modules/std/iosfwd.inc
@@ -14,7 +14,7 @@ export namespace std {
 #endif
   using std::u16streampos;
   using std::u32streampos;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
   using std::u8streampos;
 #endif
 
diff --git a/libcxx/modules/std/memory.inc b/libcxx/modules/std/memory.inc
index c15f37a21239ce325ad6e59018e2da2b5044d64c..4c4cccfd7cf2fe97e2141184183eca06b89bdb0d 100644
--- a/libcxx/modules/std/memory.inc
+++ b/libcxx/modules/std/memory.inc
@@ -154,9 +154,9 @@ export namespace std {
   using std::reinterpret_pointer_cast;
   using std::static_pointer_cast;
 
-#ifndef _LIBCPP_HAS_NO_RTTI
+#if _LIBCPP_HAS_RTTI
   using std::get_deleter;
-#endif // _LIBCPP_HAS_NO_RTTI
+#endif // _LIBCPP_HAS_RTTI
 
   // [util.smartptr.shared.io], shared_ptr I/O
 
diff --git a/libcxx/modules/std/string.inc b/libcxx/modules/std/string.inc
index 9808a96215a18291d1df5faafef909f12ef0e8b0..322317c98c01808dc43fb4c7e6a7ce60f5a6f5b2 100644
--- a/libcxx/modules/std/string.inc
+++ b/libcxx/modules/std/string.inc
@@ -34,7 +34,7 @@ export namespace std {
   using std::string;
   using std::u16string;
   using std::u32string;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
   using std::u8string;
 #endif
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
@@ -60,7 +60,7 @@ export namespace std {
     using std::pmr::string;
     using std::pmr::u16string;
     using std::pmr::u32string;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
     using std::pmr::u8string;
 #endif
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
diff --git a/libcxx/modules/std/string_view.inc b/libcxx/modules/std/string_view.inc
index f4f9d80ddb83da557464f87eae3ee9a893f274a1..1237f4516e7d00482532e1558bfb2cd911444b88 100644
--- a/libcxx/modules/std/string_view.inc
+++ b/libcxx/modules/std/string_view.inc
@@ -27,7 +27,7 @@ export namespace std {
   using std::string_view;
   using std::u16string_view;
   using std::u32string_view;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
   using std::u8string_view;
 #endif
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
diff --git a/libcxx/src/condition_variable_destructor.cpp b/libcxx/src/condition_variable_destructor.cpp
index 59811ed7ff3564a9aab431ad06391c8db98fdd84..f6ffe336859906ec21f26eabc7338536d942cea6 100644
--- a/libcxx/src/condition_variable_destructor.cpp
+++ b/libcxx/src/condition_variable_destructor.cpp
@@ -14,7 +14,7 @@
 #include <__config>
 #include <__thread/support.h>
 
-#if _LIBCPP_ABI_VERSION == 1 || !defined(_LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION)
+#if _LIBCPP_ABI_VERSION == 1 || !_LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
 #  define NEEDS_CONDVAR_DESTRUCTOR
 #endif
 
diff --git a/libcxx/src/filesystem/error.h b/libcxx/src/filesystem/error.h
index 09020fbede9b948126bcd43e6e5d925c6039bada..07ba7fc3eef251a0ce689a75b2e8f3227a1657c0 100644
--- a/libcxx/src/filesystem/error.h
+++ b/libcxx/src/filesystem/error.h
@@ -186,16 +186,16 @@ struct ErrorHandler {
   T report(const error_code& ec, const char* msg, ...) const {
     va_list ap;
     va_start(ap, msg);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       report_impl(ec, msg, ap);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       va_end(ap);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     va_end(ap);
     return error_value<T>();
   }
@@ -206,16 +206,16 @@ struct ErrorHandler {
   T report(errc const& err, const char* msg, ...) const {
     va_list ap;
     va_start(ap, msg);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
       report_impl(make_error_code(err), msg, ap);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     } catch (...) {
       va_end(ap);
       throw;
     }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     va_end(ap);
     return error_value<T>();
   }
diff --git a/libcxx/src/filesystem/format_string.h b/libcxx/src/filesystem/format_string.h
index 81c5a95ae31e5f89a2900d2bdfdf96d9c6194e2c..ad6c57579a0a6214aab9bacc9a8ad958a95c04de 100644
--- a/libcxx/src/filesystem/format_string.h
+++ b/libcxx/src/filesystem/format_string.h
@@ -56,16 +56,16 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 2) string format_string(const cha
   string ret;
   va_list ap;
   va_start(ap, msg);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     ret = detail::vformat_string(msg, ap);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     va_end(ap);
     throw;
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
   va_end(ap);
   return ret;
 }
diff --git a/libcxx/src/filesystem/int128_builtins.cpp b/libcxx/src/filesystem/int128_builtins.cpp
index 72b7cb4a8d1c09eb3dcb547d592c6e23219dedda..da6f39e7d78b60278941d7d3f3247a842a66b740 100644
--- a/libcxx/src/filesystem/int128_builtins.cpp
+++ b/libcxx/src/filesystem/int128_builtins.cpp
@@ -16,7 +16,7 @@
 #include <__config>
 #include <climits>
 
-#if !defined(_LIBCPP_HAS_NO_INT128)
+#if _LIBCPP_HAS_INT128
 
 extern "C" __attribute__((no_sanitize("undefined"))) _LIBCPP_EXPORTED_FROM_ABI __int128_t
 __muloti4(__int128_t a, __int128_t b, int* overflow) {
diff --git a/libcxx/src/future.cpp b/libcxx/src/future.cpp
index e2c14c8321dddb45edd102d8916e1286950bc3ac..04e6fb8db645964c6ee1f2399668bbaea4741c83 100644
--- a/libcxx/src/future.cpp
+++ b/libcxx/src/future.cpp
@@ -142,10 +142,10 @@ promise<void>::promise() : __state_(new __assoc_sub_state) {}
 
 promise<void>::~promise() {
   if (__state_) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
     if (!__state_->__has_value() && __state_->use_count() > 1)
       __state_->set_exception(make_exception_ptr(future_error(future_errc::broken_promise)));
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     __state_->__release_shared();
   }
 }
diff --git a/libcxx/src/ios.cpp b/libcxx/src/ios.cpp
index a727855c4655e89d949297bf7fef25175fa7a15c..74d33b4b2f1bce144085ca85aa68db97f9e0ba3e 100644
--- a/libcxx/src/ios.cpp
+++ b/libcxx/src/ios.cpp
@@ -361,18 +361,18 @@ void ios_base::swap(ios_base& rhs) noexcept {
 
 void ios_base::__set_badbit_and_consider_rethrow() {
   __rdstate_ |= badbit;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   if (__exceptions_ & badbit)
     throw;
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 void ios_base::__set_failbit_and_consider_rethrow() {
   __rdstate_ |= failbit;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   if (__exceptions_ & failbit)
     throw;
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 bool ios_base::sync_with_stdio(bool sync) {
diff --git a/libcxx/src/locale.cpp b/libcxx/src/locale.cpp
index 484963dcede9d51a2f2a4efbc634a012fdcce357..99a2d50f207ed15499560e5f886b5f43714c1c20 100644
--- a/libcxx/src/locale.cpp
+++ b/libcxx/src/locale.cpp
@@ -174,7 +174,7 @@ locale::__imp::__imp(size_t refs) : facet(refs), facets_(N), name_("C") {
   install(&make<codecvt<char16_t, char, mbstate_t> >(1u));
   install(&make<codecvt<char32_t, char, mbstate_t> >(1u));
   _LIBCPP_SUPPRESS_DEPRECATED_POP
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
   install(&make<codecvt<char16_t, char8_t, mbstate_t> >(1u));
   install(&make<codecvt<char32_t, char8_t, mbstate_t> >(1u));
 #endif
@@ -219,9 +219,9 @@ locale::__imp::__imp(size_t refs) : facet(refs), facets_(N), name_("C") {
 }
 
 locale::__imp::__imp(const string& name, size_t refs) : facet(refs), facets_(N), name_(name) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     facets_ = locale::classic().__locale_->facets_;
     for (unsigned i = 0; i < facets_.size(); ++i)
       if (facets_[i])
@@ -242,7 +242,7 @@ locale::__imp::__imp(const string& name, size_t refs) : facet(refs), facets_(N),
     install(new codecvt_byname<char16_t, char, mbstate_t>(name_));
     install(new codecvt_byname<char32_t, char, mbstate_t>(name_));
     _LIBCPP_SUPPRESS_DEPRECATED_POP
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
     install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name_));
     install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name_));
 #endif
@@ -268,14 +268,14 @@ locale::__imp::__imp(const string& name, size_t refs) : facet(refs), facets_(N),
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
     install(new messages_byname<wchar_t>(name_));
 #endif
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     for (unsigned i = 0; i < facets_.size(); ++i)
       if (facets_[i])
         facets_[i]->__release_shared();
     throw;
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 locale::__imp::__imp(const __imp& other) : facets_(max<size_t>(N, other.facets_.size())), name_(other.name_) {
@@ -291,9 +291,9 @@ locale::__imp::__imp(const __imp& other, const string& name, locale::category c)
   for (unsigned i = 0; i < facets_.size(); ++i)
     if (facets_[i])
       facets_[i]->__add_shared();
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     if (c & locale::collate) {
       install(new collate_byname<char>(name));
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
@@ -313,7 +313,7 @@ locale::__imp::__imp(const __imp& other, const string& name, locale::category c)
       install(new codecvt_byname<char16_t, char, mbstate_t>(name));
       install(new codecvt_byname<char32_t, char, mbstate_t>(name));
       _LIBCPP_SUPPRESS_DEPRECATED_POP
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
       install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name));
       install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name));
 #endif
@@ -348,14 +348,14 @@ locale::__imp::__imp(const __imp& other, const string& name, locale::category c)
       install(new messages_byname<wchar_t>(name));
 #endif
     }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     for (unsigned i = 0; i < facets_.size(); ++i)
       if (facets_[i])
         facets_[i]->__release_shared();
     throw;
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 template <class F>
@@ -370,9 +370,9 @@ locale::__imp::__imp(const __imp& other, const __imp& one, locale::category c)
   for (unsigned i = 0; i < facets_.size(); ++i)
     if (facets_[i])
       facets_[i]->__add_shared();
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     if (c & locale::collate) {
       install_from<std::collate<char> >(one);
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
@@ -389,7 +389,7 @@ locale::__imp::__imp(const __imp& other, const __imp& one, locale::category c)
       install_from<std::codecvt<char16_t, char, mbstate_t> >(one);
       install_from<std::codecvt<char32_t, char, mbstate_t> >(one);
       _LIBCPP_SUPPRESS_DEPRECATED_POP
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
       install_from<std::codecvt<char16_t, char8_t, mbstate_t> >(one);
       install_from<std::codecvt<char32_t, char8_t, mbstate_t> >(one);
 #endif
@@ -443,14 +443,14 @@ locale::__imp::__imp(const __imp& other, const __imp& one, locale::category c)
       install_from<std::messages<wchar_t> >(one);
 #endif
     }
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     for (unsigned i = 0; i < facets_.size(); ++i)
       if (facets_[i])
         facets_[i]->__release_shared();
     throw;
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 locale::__imp::__imp(const __imp& other, facet* f, long id)
@@ -2815,7 +2815,7 @@ int codecvt<char16_t, char, mbstate_t>::do_length(
 
 int codecvt<char16_t, char, mbstate_t>::do_max_length() const noexcept { return 4; }
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
 
@@ -2949,7 +2949,7 @@ int codecvt<char32_t, char, mbstate_t>::do_length(
 
 int codecvt<char32_t, char, mbstate_t>::do_max_length() const noexcept { return 4; }
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
 
@@ -5707,7 +5707,7 @@ template class _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_
     codecvt_byname<char16_t, char, mbstate_t>;
 template class _LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS
     codecvt_byname<char32_t, char, mbstate_t>;
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
 template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<char16_t, char8_t, mbstate_t>;
 template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<char32_t, char8_t, mbstate_t>;
 #endif
diff --git a/libcxx/src/memory_resource.cpp b/libcxx/src/memory_resource.cpp
index d2ff3509c5a31ef01cc4925b7dfa2642ea85f71a..299f810948fd14348aacef0ff8a98d39e2b0706e 100644
--- a/libcxx/src/memory_resource.cpp
+++ b/libcxx/src/memory_resource.cpp
@@ -9,7 +9,7 @@
 #include <memory>
 #include <memory_resource>
 
-#ifndef _LIBCPP_HAS_NO_ATOMIC_HEADER
+#if _LIBCPP_HAS_ATOMIC_HEADER
 #  include <atomic>
 #elif !defined(_LIBCPP_HAS_NO_THREADS)
 #  include <mutex>
@@ -28,7 +28,7 @@ memory_resource::~memory_resource() = default;
 
 // new_delete_resource()
 
-#ifdef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+#if !_LIBCPP_HAS_ALIGNED_ALLOCATION
 static bool is_aligned_to(void* ptr, size_t align) {
   void* p2     = ptr;
   size_t space = 1;
@@ -39,7 +39,7 @@ static bool is_aligned_to(void* ptr, size_t align) {
 
 class _LIBCPP_EXPORTED_FROM_ABI __new_delete_memory_resource_imp : public memory_resource {
   void* do_allocate(size_t bytes, size_t align) override {
-#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+#if _LIBCPP_HAS_ALIGNED_ALLOCATION
     return std::__libcpp_allocate(bytes, align);
 #else
     if (bytes == 0)
@@ -91,7 +91,7 @@ memory_resource* null_memory_resource() noexcept { return &res_init.resources.nu
 // default_memory_resource()
 
 static memory_resource* __default_memory_resource(bool set = false, memory_resource* new_res = nullptr) noexcept {
-#ifndef _LIBCPP_HAS_NO_ATOMIC_HEADER
+#if _LIBCPP_HAS_ATOMIC_HEADER
   static constinit atomic<memory_resource*> __res{&res_init.resources.new_delete_res};
   if (set) {
     new_res = new_res ? new_res : new_delete_resource();
diff --git a/libcxx/src/mutex_destructor.cpp b/libcxx/src/mutex_destructor.cpp
index a6ceaaaf23ffed50017a61250b58c5248a30a8e3..9f991721f083fee8422461ec4fa8640d1391dc51 100644
--- a/libcxx/src/mutex_destructor.cpp
+++ b/libcxx/src/mutex_destructor.cpp
@@ -19,7 +19,7 @@
 #include <__config>
 #include <__thread/support.h>
 
-#if _LIBCPP_ABI_VERSION == 1 || !defined(_LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION)
+#if _LIBCPP_ABI_VERSION == 1 || !_LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION
 #  define NEEDS_MUTEX_DESTRUCTOR
 #endif
 
diff --git a/libcxx/src/new.cpp b/libcxx/src/new.cpp
index b0c731678cac30c1d0db337a17cc3d18f48232e8..e010fe4c4f1912f8f0ea78c11a299a7ed41fa92c 100644
--- a/libcxx/src/new.cpp
+++ b/libcxx/src/new.cpp
@@ -51,7 +51,7 @@ _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new(std
 }
 
 _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
-#  ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new)),
@@ -79,7 +79,7 @@ _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new[](s
 }
 
 _LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept {
-#  ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new[])),
@@ -114,7 +114,7 @@ _LIBCPP_WEAK void operator delete[](void* ptr, const std::nothrow_t&) noexcept {
 
 _LIBCPP_WEAK void operator delete[](void* ptr, size_t) noexcept { ::operator delete[](ptr); }
 
-#  if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
+#  if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 
 static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignment) {
   if (size == 0)
@@ -145,7 +145,7 @@ operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
 }
 
 _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
-#    ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if !_LIBCPP_HAS_EXCEPTIONS
 #      if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new)),
@@ -174,7 +174,7 @@ operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
 }
 
 _LIBCPP_WEAK void* operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
-#    ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#    if !_LIBCPP_HAS_EXCEPTIONS
 #      if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new[])),
@@ -220,7 +220,7 @@ _LIBCPP_WEAK void operator delete[](void* ptr, size_t, std::align_val_t alignmen
   ::operator delete[](ptr, alignment);
 }
 
-#  endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#  endif // _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 // ------------------ END COPY ------------------
 
 #endif // !__GLIBCXX__ && !_LIBCPP_ABI_VCRUNTIME
diff --git a/libcxx/src/new_helpers.cpp b/libcxx/src/new_helpers.cpp
index 6560d0188ee32b59acf9d85bce68a90787de43a9..2119d82b3e00f94063d8010a51037996fbcf35f5 100644
--- a/libcxx/src/new_helpers.cpp
+++ b/libcxx/src/new_helpers.cpp
@@ -18,7 +18,7 @@ const nothrow_t nothrow{};
 #ifndef LIBSTDCXX
 
 void __throw_bad_alloc() {
-#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if _LIBCPP_HAS_EXCEPTIONS
   throw bad_alloc();
 #  else
   _LIBCPP_VERBOSE_ABORT("bad_alloc was thrown in -fno-exceptions mode");
diff --git a/libcxx/src/ostream.cpp b/libcxx/src/ostream.cpp
index e1a9a4bc1de7185680012907a88c328c5fc692dd..f036aaf29dfb25c94a180e24b0d559ebce95671c 100644
--- a/libcxx/src/ostream.cpp
+++ b/libcxx/src/ostream.cpp
@@ -24,7 +24,7 @@ _LIBCPP_EXPORTED_FROM_ABI FILE* __get_ostream_file(ostream& __os) {
   // Returning a nullptr means the stream is not considered a terminal and the
   // special terminal handling is not done. The terminal handling is mainly of
   // importance on Windows.
-#ifndef _LIBCPP_HAS_NO_RTTI
+#if _LIBCPP_HAS_RTTI
   auto* __rdbuf = __os.rdbuf();
 #  ifndef _LIBCPP_HAS_NO_FILESYSTEM
   if (auto* __buffer = dynamic_cast<filebuf*>(__rdbuf))
@@ -33,7 +33,7 @@ _LIBCPP_EXPORTED_FROM_ABI FILE* __get_ostream_file(ostream& __os) {
 
   if (auto* __buffer = dynamic_cast<__stdoutbuf<char>*>(__rdbuf))
     return __buffer->__file_;
-#endif // _LIBCPP_HAS_NO_RTTI
+#endif // _LIBCPP_HAS_RTTI
 
   return nullptr;
 }
diff --git a/libcxx/src/ryu/d2s.cpp b/libcxx/src/ryu/d2s.cpp
index 32d617cb55324836d72e40fd471b728bc1c0a656..5b80ed8a709d8e6050f896ba5c17a8bfc44ccd0a 100644
--- a/libcxx/src/ryu/d2s.cpp
+++ b/libcxx/src/ryu/d2s.cpp
@@ -478,7 +478,7 @@ struct __floating_decimal_64 {
           36893488u, 7378697u, 1475739u, 295147u, 59029u, 11805u, 2361u, 472u, 94u, 18u, 3u };
 
         unsigned long _Trailing_zero_bits;
-#ifdef _LIBCPP_HAS_BITSCAN64
+#if _LIBCPP_HAS_BITSCAN64
         (void) _BitScanForward64(&_Trailing_zero_bits, __v.__mantissa); // __v.__mantissa is guaranteed nonzero
 #else // ^^^ 64-bit ^^^ / vvv 32-bit vvv
         const uint32_t _Low_mantissa = static_cast<uint32_t>(__v.__mantissa);
diff --git a/libcxx/src/stdexcept.cpp b/libcxx/src/stdexcept.cpp
index 134d28efb750f3bd86ce35cbfe56a870934d422f..0ee438bef02e1032012394502f73e86a1d973a5e 100644
--- a/libcxx/src/stdexcept.cpp
+++ b/libcxx/src/stdexcept.cpp
@@ -20,7 +20,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 void __throw_runtime_error(const char* msg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw runtime_error(msg);
 #else
   _LIBCPP_VERBOSE_ABORT("runtime_error was thrown in -fno-exceptions mode with message \"%s\"", msg);
diff --git a/libcxx/src/support/runtime/exception_fallback.ipp b/libcxx/src/support/runtime/exception_fallback.ipp
index bedb16e58566bcfa37d566d73ff0e3a58546c739..ba283aee2290188ff19d19eb174e92732a0d5c74 100644
--- a/libcxx/src/support/runtime/exception_fallback.ipp
+++ b/libcxx/src/support/runtime/exception_fallback.ipp
@@ -34,18 +34,18 @@ terminate_handler set_terminate(terminate_handler func) noexcept {
 terminate_handler get_terminate() noexcept { return __libcpp_atomic_load(&__terminate_handler); }
 
 [[noreturn]] void terminate() noexcept {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     (*get_terminate())();
     // handler should not return
     __libcpp_verbose_abort("terminate_handler unexpectedly returned\n");
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     // handler should not throw exception
     __libcpp_verbose_abort("terminate_handler unexpectedly threw an exception\n");
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 bool uncaught_exception() noexcept { return uncaught_exceptions() > 0; }
diff --git a/libcxx/src/support/runtime/exception_msvc.ipp b/libcxx/src/support/runtime/exception_msvc.ipp
index 869b64d2f533e498d3f48587c301550f5c4e584d..2ae004bb02e5d9e0d9ee76bd3471a73721d5eac9 100644
--- a/libcxx/src/support/runtime/exception_msvc.ipp
+++ b/libcxx/src/support/runtime/exception_msvc.ipp
@@ -42,18 +42,18 @@ terminate_handler set_terminate(terminate_handler func) noexcept { return ::set_
 terminate_handler get_terminate() noexcept { return ::_get_terminate(); }
 
 [[noreturn]] void terminate() noexcept {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
     (*get_terminate())();
     // handler should not return
     __libcpp_verbose_abort("terminate_handler unexpectedly returned\n");
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   } catch (...) {
     // handler should not throw exception
     __libcpp_verbose_abort("terminate_handler unexpectedly threw an exception\n");
   }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+#endif // _LIBCPP_HAS_EXCEPTIONS
 }
 
 bool uncaught_exception() noexcept { return uncaught_exceptions() > 0; }
diff --git a/libcxx/src/system_error.cpp b/libcxx/src/system_error.cpp
index 3367bd56bd78802f2c08232e8e7d0297f610fc97..476729310d868b9c8969dd288861050cf7ac11ed 100644
--- a/libcxx/src/system_error.cpp
+++ b/libcxx/src/system_error.cpp
@@ -211,7 +211,7 @@ system_error::system_error(int ev, const error_category& ecat)
 system_error::~system_error() noexcept {}
 
 void __throw_system_error(int ev, const char* what_arg) {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   std::__throw_system_error(error_code(ev, system_category()), what_arg);
 #else
   // The above could also handle the no-exception case, but for size, avoid referencing system_category() unnecessarily.
diff --git a/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp b/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp
index 55b5929847748fce2ef72735af46aaa6576f42b4..563580b68795522e78e155ecfc223afdae5240d6 100644
--- a/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp
+++ b/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_integer.compile.pass.cpp
@@ -30,7 +30,7 @@ static_assert(std::__libcpp_integer<unsigned int>);
 static_assert(std::__libcpp_integer<unsigned long int>);
 static_assert(std::__libcpp_integer<unsigned long long int>);
 static_assert(std::__libcpp_integer<unsigned short int>);
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 static_assert(std::__libcpp_integer<__uint128_t>);
 #endif
 // Signed
@@ -40,7 +40,7 @@ static_assert(std::__libcpp_integer<int>);
 static_assert(std::__libcpp_integer<long int>);
 static_assert(std::__libcpp_integer<long long int>);
 static_assert(std::__libcpp_integer<short int>);
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 static_assert(std::__libcpp_integer<__int128_t>);
 #endif
 // Non-integer
diff --git a/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp b/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp
index 5bf7aa526f0a26ff3ebeaa3426f5252d7b55bdb5..d1e21ee96b0736e7b3f75f9b7a2bdceb794191e6 100644
--- a/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp
+++ b/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_signed_integer.compile.pass.cpp
@@ -30,7 +30,7 @@ static_assert(!std::__libcpp_signed_integer<unsigned int>);
 static_assert(!std::__libcpp_signed_integer<unsigned long int>);
 static_assert(!std::__libcpp_signed_integer<unsigned long long int>);
 static_assert(!std::__libcpp_signed_integer<unsigned short int>);
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 static_assert(!std::__libcpp_signed_integer<__uint128_t>);
 #endif
 // Signed
@@ -40,7 +40,7 @@ static_assert(std::__libcpp_signed_integer<int>);
 static_assert(std::__libcpp_signed_integer<long int>);
 static_assert(std::__libcpp_signed_integer<long long int>);
 static_assert(std::__libcpp_signed_integer<short int>);
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 static_assert(std::__libcpp_signed_integer<__int128_t>);
 #endif
 // Non-integer
diff --git a/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp b/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp
index 163ec53e17bc1e4c55caaf351cdb2c14f33b45d3..c671f03cbfce46bfb9fb8267c1f5c705393ffae7 100644
--- a/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp
+++ b/libcxx/test/libcxx/concepts/concepts.arithmetic/__libcpp_unsigned_integer.compile.pass.cpp
@@ -30,7 +30,7 @@ static_assert(std::__libcpp_unsigned_integer<unsigned int>);
 static_assert(std::__libcpp_unsigned_integer<unsigned long int>);
 static_assert(std::__libcpp_unsigned_integer<unsigned long long int>);
 static_assert(std::__libcpp_unsigned_integer<unsigned short int>);
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 static_assert(std::__libcpp_unsigned_integer<__uint128_t>);
 #endif
 // Signed
@@ -40,7 +40,7 @@ static_assert(!std::__libcpp_unsigned_integer<int>);
 static_assert(!std::__libcpp_unsigned_integer<long int>);
 static_assert(!std::__libcpp_unsigned_integer<long long int>);
 static_assert(!std::__libcpp_unsigned_integer<short int>);
-#ifndef _LIBCPP_HAS_NO_INT128
+#if _LIBCPP_HAS_INT128
 static_assert(!std::__libcpp_unsigned_integer<__int128_t>);
 #endif
 // Non-integer
diff --git a/libcxx/test/libcxx/containers/container_traits.compile.pass.cpp b/libcxx/test/libcxx/containers/container_traits.compile.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1452bfbaf3960ea659d38880db0f0900b8d1343d
--- /dev/null
+++ b/libcxx/test/libcxx/containers/container_traits.compile.pass.cpp
@@ -0,0 +1,163 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// <__type_traits/container_traits.h>
+//
+
+#include <__type_traits/container_traits.h>
+
+#include <deque>
+#include <forward_list>
+#include <list>
+#include <vector>
+#include <map>
+#include <set>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "test_allocator.h"
+#include "test_macros.h"
+#include "MoveOnly.h"
+
+struct ThrowOnMove {
+  ThrowOnMove();
+  ThrowOnMove(const ThrowOnMove&) TEST_NOEXCEPT_COND(false);
+  ThrowOnMove(ThrowOnMove&&) TEST_NOEXCEPT_COND(false);
+  ThrowOnMove& operator=(ThrowOnMove&&) TEST_NOEXCEPT_COND(false);
+  ThrowOnMove& operator=(const ThrowOnMove&) TEST_NOEXCEPT_COND(false);
+
+  bool operator<(ThrowOnMove const&) const;
+  bool operator==(ThrowOnMove const&) const;
+};
+
+struct NonCopyThrowOnMove {
+  NonCopyThrowOnMove();
+  NonCopyThrowOnMove(NonCopyThrowOnMove&&) TEST_NOEXCEPT_COND(false);
+  NonCopyThrowOnMove(const NonCopyThrowOnMove&) = delete;
+  NonCopyThrowOnMove& operator=(NonCopyThrowOnMove&&) TEST_NOEXCEPT_COND(false);
+  NonCopyThrowOnMove& operator=(const NonCopyThrowOnMove&) = delete;
+
+  bool operator<(NonCopyThrowOnMove const&) const;
+  bool operator==(NonCopyThrowOnMove const&) const;
+};
+
+struct ThrowingHash {
+  template <class T>
+  std::size_t operator()(const T&) const TEST_NOEXCEPT_COND(false);
+};
+
+struct NoThrowHash {
+  template <class T>
+  std::size_t operator()(const T&) const TEST_NOEXCEPT;
+};
+
+template <bool Expected, class Container>
+void check() {
+  static_assert(
+      std::__container_traits<Container>::__emplacement_has_strong_exception_safety_guarantee == Expected, "");
+}
+
+void test() {
+  check<true, std::list<int> >();
+  check<true, std::list<int, test_allocator<int> > >();
+  check<true, std::list<MoveOnly> >();
+  check<true, std::list<ThrowOnMove> >();
+  check<true, std::list<NonCopyThrowOnMove> >();
+
+  check<true, std::forward_list<int> >();
+  check<true, std::forward_list<int, test_allocator<int> > >();
+  check<true, std::forward_list<MoveOnly> >();
+  check<true, std::forward_list<ThrowOnMove> >();
+  check<true, std::forward_list<NonCopyThrowOnMove> >();
+
+  check<true, std::deque<int> >();
+  check<true, std::deque<int, test_allocator<int> > >();
+  check<true, std::deque<MoveOnly> >();
+  check<true, std::deque<ThrowOnMove> >();
+  check<false, std::deque<NonCopyThrowOnMove> >();
+
+  check<true, std::vector<int> >();
+  check<true, std::vector<int, test_allocator<int> > >();
+  check<true, std::vector<MoveOnly> >();
+  check<true, std::vector<ThrowOnMove> >();
+  check<false, std::vector<NonCopyThrowOnMove> >();
+
+  check<true, std::set<int> >();
+  check<true, std::set<int, std::less<int>, test_allocator<int> > >();
+  check<true, std::set<MoveOnly> >();
+  check<true, std::set<ThrowOnMove> >();
+  check<true, std::set<NonCopyThrowOnMove> >();
+
+  check<true, std::multiset<int> >();
+  check<true, std::multiset<int, std::less<int>, test_allocator<int> > >();
+  check<true, std::multiset<MoveOnly> >();
+  check<true, std::multiset<ThrowOnMove> >();
+  check<true, std::multiset<NonCopyThrowOnMove> >();
+
+  check<true, std::map<int, int> >();
+  check<true, std::map<int, int, std::less<int>, test_allocator<int> > >();
+  check<true, std::map<MoveOnly, MoveOnly> >();
+  check<true, std::map<ThrowOnMove, ThrowOnMove> >();
+  check<true, std::map<NonCopyThrowOnMove, NonCopyThrowOnMove> >();
+
+  check<true, std::multimap<int, int> >();
+  check<true, std::multimap<int, int, std::less<int>, test_allocator<int> > >();
+  check<true, std::multimap<MoveOnly, MoveOnly> >();
+  check<true, std::multimap<ThrowOnMove, ThrowOnMove> >();
+  check<true, std::multimap<NonCopyThrowOnMove, NonCopyThrowOnMove> >();
+
+#if TEST_STD_VER < 11
+  check<false, std::unordered_set<int> >();
+  check<false, std::unordered_set<int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_set<MoveOnly> >();
+  check<false, std::unordered_set<MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_set<MoveOnly, ThrowingHash> >();
+
+  check<false, std::unordered_multiset<int> >();
+  check<false, std::unordered_multiset<int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_multiset<MoveOnly> >();
+  check<false, std::unordered_multiset<MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_multiset<MoveOnly, ThrowingHash> >();
+
+  check<false, std::unordered_map<int, int> >();
+  check<false, std::unordered_map<int, int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_map<MoveOnly, MoveOnly> >();
+  check<false, std::unordered_map<MoveOnly, MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_map<MoveOnly, MoveOnly, ThrowingHash> >();
+
+  check<false, std::unordered_multimap<int, int> >();
+  check<false, std::unordered_multimap<int, int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_multimap<MoveOnly, MoveOnly> >();
+  check<false, std::unordered_multimap<MoveOnly, MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_multimap<MoveOnly, MoveOnly, ThrowingHash> >();
+#else
+  check<true, std::unordered_set<int> >();
+  check<true, std::unordered_set<int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_set<MoveOnly> >();
+  check<true, std::unordered_set<MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_set<MoveOnly, ThrowingHash> >();
+
+  check<true, std::unordered_multiset<int> >();
+  check<true, std::unordered_multiset<int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_multiset<MoveOnly> >();
+  check<true, std::unordered_multiset<MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_multiset<MoveOnly, ThrowingHash> >();
+
+  check<true, std::unordered_map<int, int> >();
+  check<true, std::unordered_map<int, int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_map<MoveOnly, MoveOnly> >();
+  check<true, std::unordered_map<MoveOnly, MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_map<MoveOnly, MoveOnly, ThrowingHash> >();
+
+  check<true, std::unordered_multimap<int, int> >();
+  check<true, std::unordered_multimap<int, int, std::hash<int>, std::less<int>, test_allocator<int> > >();
+  check<false, std::unordered_multimap<MoveOnly, MoveOnly> >();
+  check<true, std::unordered_multimap<MoveOnly, MoveOnly, NoThrowHash> >();
+  check<false, std::unordered_multimap<MoveOnly, MoveOnly, ThrowingHash> >();
+#endif
+}
diff --git a/libcxx/test/libcxx/diagnostics/new.nodiscard.verify.cpp b/libcxx/test/libcxx/diagnostics/new.nodiscard.verify.cpp
index 578afaabbd26de113deee6e329b614b056be48c9..505618c0b88d70e41ceea6331d62b8dcdc835e06 100644
--- a/libcxx/test/libcxx/diagnostics/new.nodiscard.verify.cpp
+++ b/libcxx/test/libcxx/diagnostics/new.nodiscard.verify.cpp
@@ -21,12 +21,12 @@ void test() {
   ::operator new(0, std::nothrow);                        // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
   ::operator new[](0);                                    // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
   ::operator new[](0, std::nothrow);                      // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+#if _LIBCPP_HAS_ALIGNED_ALLOCATION
   ::operator new(0, std::align_val_t{1});                 // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
   ::operator new(0, std::align_val_t{1}, std::nothrow);   // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
   ::operator new[](0, std::align_val_t{1});               // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
   ::operator new[](0, std::align_val_t{1}, std::nothrow); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
-#endif // _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+#endif // _LIBCPP_HAS_ALIGNED_ALLOCATION
 
 #if TEST_STD_VER >= 17
   int* ptr = nullptr;
diff --git a/libcxx/test/libcxx/memory/aligned_allocation_macro.compile.pass.cpp b/libcxx/test/libcxx/memory/aligned_allocation_macro.compile.pass.cpp
index 34a3146e26c95895da1bb78235ff98b0433a9757..6249c52814781a3fe0aeb15b39f0ac170eec485f 100644
--- a/libcxx/test/libcxx/memory/aligned_allocation_macro.compile.pass.cpp
+++ b/libcxx/test/libcxx/memory/aligned_allocation_macro.compile.pass.cpp
@@ -12,7 +12,6 @@
 
 #include "test_macros.h"
 
-
-#ifdef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
-#   error "libc++ should have aligned allocation in C++17 and up when targeting a platform that supports it"
+#if !_LIBCPP_HAS_ALIGNED_ALLOCATION
+#  error "libc++ should have aligned allocation in C++17 and up when targeting a platform that supports it"
 #endif
diff --git a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
index 64d2c54e7dd84f8e08054f6cd06fa14981774bb3..8bd82bacc5d1fe16cba8cdb8706a96459c76b63f 100644
--- a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
+++ b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
@@ -79,7 +79,7 @@ static_assert(!std::__libcpp_is_trivially_relocatable<std::array<NotTriviallyCop
 static_assert(std::__libcpp_is_trivially_relocatable<std::array<std::unique_ptr<int>, 1> >::value, "");
 
 // basic_string
-#if defined(_LIBCPP_HAS_NO_ASAN) || !defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#if !_LIBCPP_HAS_ASAN || !defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
 struct MyChar {
   char c;
 };
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp
index 00ba267a4fde09360bbf4877d63eef59b85b1054..d2b62a9b6ab9947ff444bacbe67948f1a45a965e 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp
@@ -121,7 +121,7 @@ void test() {
     }
   }
 // These diagnostics happen when we try to construct bad_expected_access from the non copy-constructible error type.
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   // expected-error-re@*:* {{call to deleted constructor of{{.*}}}}
   // expected-error-re@*:* {{call to deleted constructor of{{.*}}}}
   // expected-error-re@*:* {{call to deleted constructor of{{.*}}}}
diff --git a/libcxx/test/std/atomics/stdatomic.h.syn/types.compile.pass.cpp b/libcxx/test/std/atomics/stdatomic.h.syn/types.compile.pass.cpp
index 85d93379878359c95e53f3b87d6ab345c054ce5a..50bc157157c5c904f37b9f6dfeb02aea95ca7ea7 100644
--- a/libcxx/test/std/atomics/stdatomic.h.syn/types.compile.pass.cpp
+++ b/libcxx/test/std/atomics/stdatomic.h.syn/types.compile.pass.cpp
@@ -168,7 +168,7 @@ void f() {
   static_assert(std::is_same_v<std::atomic<long long>, ::atomic_llong>);
   static_assert(std::is_same_v<std::atomic<unsigned long long>, ::atomic_ullong>);
 
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
   static_assert(std::is_same_v<std::atomic<char8_t>,  ::atomic_char8_t>);
 #endif
   static_assert(std::is_same_v<std::atomic<char16_t>, ::atomic_char16_t>);
diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/alg.pass.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/alg.pass.cpp
index 159cb19f65468b5708a2d2be5155913a17e78201..772c3c563087ecd8180651ff154c5e50c778022e 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/alg.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/alg.pass.cpp
@@ -82,7 +82,7 @@ int main(int, char**)
     assert(e4() < 0x7fffffffu);
     assert(e4() < 0x7fffffffu);
 
-#ifndef _LIBCPP_HAS_NO_INT128
+#ifndef TEST_HAS_NO_INT128
     // m might overflow. The overflow is not OK, result will be in bounds,
     // and Schrage's algorithm is incompatible here. Need to use 128 bit arithmetic.
     typedef std::linear_congruential_engine<T, 0x100000001ull, 0ull, (1ull << 61) - 1ull> E5;
diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/default.pass.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/default.pass.cpp
index a85c63d26ead6292b6fdaef1df116545470e7870..423f4d021ac64dfc620457053edfb54690489f36 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/default.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/default.pass.cpp
@@ -84,7 +84,7 @@ int main(int, char**)
     test_ext<unsigned long>();
     test<unsigned long long>();
     // This isn't implemented on platforms without __int128
-#ifndef _LIBCPP_HAS_NO_INT128
+#ifndef TEST_HAS_NO_INT128
     test_ext<unsigned long long>();
 #endif
 
diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/values.pass.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/values.pass.cpp
index 66b4e6ab4d6a86c5acf582fdce7b4d1c0d702bcb..f238c12cff1eb54d5c48e11b323636564d788d1a 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/values.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/values.pass.cpp
@@ -117,7 +117,7 @@ int main(int, char**)
     test_ext<unsigned long>();
     test<unsigned long long>();
     // This isn't implemented on platforms without __int128
-#ifndef _LIBCPP_HAS_NO_INT128
+#ifndef TEST_HAS_NO_INT128
     test_ext<unsigned long long>();
 #endif
 
diff --git a/libcxx/test/std/strings/c.strings/no_c8rtomb_mbrtoc8.verify.cpp b/libcxx/test/std/strings/c.strings/no_c8rtomb_mbrtoc8.verify.cpp
index bc058d400f7496fb8d6d61a393cea57c5b6ef620..1d4a225668d80ae1e7ec217a52c8db7b65a332f2 100644
--- a/libcxx/test/std/strings/c.strings/no_c8rtomb_mbrtoc8.verify.cpp
+++ b/libcxx/test/std/strings/c.strings/no_c8rtomb_mbrtoc8.verify.cpp
@@ -19,7 +19,7 @@
 #if !defined(TEST_HAS_NO_CHAR8_T)
 using U = decltype(::c8rtomb);
 using V = decltype(::mbrtoc8);
-#  if defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8)
+#  if !_LIBCPP_HAS_C8RTOMB_MBRTOC8
 // expected-error@-3 {{no member named 'c8rtomb' in the global namespace}}
 // expected-error@-3 {{no member named 'mbrtoc8' in the global namespace}}
 #  else
diff --git a/libcxx/test/support/test.support/make_string_header.pass.cpp b/libcxx/test/support/test.support/make_string_header.pass.cpp
index 8fcc05ec49d7c1dd093caa12941ba2a727999b54..45aaa2cfef207ff20303b29176397c06d1d3f351 100644
--- a/libcxx/test/support/test.support/make_string_header.pass.cpp
+++ b/libcxx/test/support/test.support/make_string_header.pass.cpp
@@ -31,7 +31,7 @@ int main(int, char**) {
     ==  L" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMN"
              "OPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~");
 #endif
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
+#if _LIBCPP_HAS_CHAR8_T
   assert(MAKE_STRING(char8_t,
          " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMN"
              "OPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~")
diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h
index ae171b3f223ab3d8cb2dc36a7fc85ea091ba67ed..5dcd775e8641e18f3748e0fa6d06e1424212075d 100644
--- a/libcxx/test/support/test_macros.h
+++ b/libcxx/test/support/test_macros.h
@@ -214,10 +214,10 @@
 #define TEST_IS_EXECUTED_IN_A_SLOW_ENVIRONMENT
 #endif
 
-#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) || \
-  (!(TEST_STD_VER > 14 || \
-    (defined(__cpp_aligned_new) && __cpp_aligned_new >= 201606L)))
-#define TEST_HAS_NO_ALIGNED_ALLOCATION
+#if defined(_LIBCPP_VERSION) && !_LIBCPP_HAS_ALIGNED_ALLOCATION
+#  define TEST_HAS_NO_ALIGNED_ALLOCATION
+#elif TEST_STD_VER < 17 && (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606L)
+#  define TEST_HAS_NO_ALIGNED_ALLOCATION
 #endif
 
 #if TEST_STD_VER > 17
@@ -397,12 +397,12 @@ inline Tp const& DoNotOptimize(Tp const& value) {
 #   define TEST_HAS_NO_UNICODE
 #endif
 
-#if defined(_LIBCPP_HAS_OPEN_WITH_WCHAR)
+#if defined(_LIBCPP_VERSION) && _LIBCPP_HAS_OPEN_WITH_WCHAR
 #  define TEST_HAS_OPEN_WITH_WCHAR
 #endif
 
-#if defined(_LIBCPP_HAS_NO_INT128) || defined(_MSVC_STL_VERSION)
-#   define TEST_HAS_NO_INT128
+#if (defined(_LIBCPP_VERSION) && !_LIBCPP_HAS_INT128) || defined(_MSVC_STL_VERSION)
+#  define TEST_HAS_NO_INT128
 #endif
 
 #if defined(_LIBCPP_HAS_NO_LOCALIZATION)
@@ -421,7 +421,7 @@ inline Tp const& DoNotOptimize(Tp const& value) {
 #  define TEST_HAS_NO_FILESYSTEM
 #endif
 
-#if defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8)
+#if defined(_LIBCPP_VERSION) && !_LIBCPP_HAS_C8RTOMB_MBRTOC8
 #  define TEST_HAS_NO_C8RTOMB_MBRTOC8
 #endif
 
@@ -495,7 +495,7 @@ inline Tp const& DoNotOptimize(Tp const& value) {
 #endif
 
 // Clang-18 has support for deducing this, but it does not set the FTM.
-#ifdef _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
+#if defined(_LIBCPP_VERSION) && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
 #  define TEST_HAS_EXPLICIT_THIS_PARAMETER
 #endif
 
diff --git a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
index 0e1d3506a997355524e7b7bddd306d8c1ac5d433..efe63e0f625f2e30eda17c7e7886895bf4b839e5 100644
--- a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
+++ b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
@@ -94,6 +94,7 @@ set(SOURCES
     abi_tag_on_virtual.cpp
     header_exportable_declarations.cpp
     hide_from_abi.cpp
+    internal_ftm_use.cpp
     proper_version_checks.cpp
     qualify_declval.cpp
     robust_against_adl.cpp
diff --git a/libcxx/test/tools/clang_tidy_checks/internal_ftm_use.cpp b/libcxx/test/tools/clang_tidy_checks/internal_ftm_use.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b5e849e2a0cb9016ccd0a490916218e625e79d9f
--- /dev/null
+++ b/libcxx/test/tools/clang_tidy_checks/internal_ftm_use.cpp
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// This clang-tidy check ensures that we don't use any _LIBCPP_HAS_FOO macro
+// inside `#ifdef`, `#ifndef` & friends, since the intent is to always use `#if` instead.
+
+#include "internal_ftm_use.hpp"
+
+#include <clang/Lex/Lexer.h>
+#include <clang/Lex/PPCallbacks.h>
+#include <clang/Lex/Preprocessor.h>
+
+#include <string>
+
+namespace libcpp {
+namespace {
+std::array valid_macros{
+    // Public API macros
+    "_LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS",
+
+    // Testing macros
+    "_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER",
+
+    // TODO: Why does this macro even exist?
+    "_LIBCPP_HAS_NO_TREE_BARRIER",
+
+    // Configuration macros
+    "_LIBCPP_HAS_NO_TIME_ZONE_DATABASE",
+    "_LIBCPP_HAS_NO_FILESYSTEM",
+    "_LIBCPP_HAS_NO_LOCALIZATION",
+    "_LIBCPP_HAS_NO_THREADS",
+    "_LIBCPP_HAS_NO_MONOTONIC_CLOCK",
+    "_LIBCPP_HAS_MUSL_LIBC",
+    "_LIBCPP_HAS_NO_WIDE_CHARACTERS",
+    "_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS",
+    "_LIBCPP_HAS_NO_RANDOM_DEVICE",
+    "_LIBCPP_HAS_NO_UNICODE",
+    "_LIBCPP_HAS_NO_TERMINAL",
+
+    // Atomic API macros
+    "_LIBCPP_HAS_EXTERNAL_ATOMIC_IMP",
+    "_LIBCPP_HAS_GCC_ATOMIC_IMP",
+    "_LIBCPP_HAS_C_ATOMIC_IMP",
+
+    // Thread API macros
+    "_LIBCPP_HAS_THREAD_API_PTHREAD",
+    "_LIBCPP_HAS_THREAD_API_WIN32",
+    "_LIBCPP_HAS_THREAD_API_EXTERNAL",
+    "_LIBCPP_HAS_THREAD_API_C11",
+
+    // Experimental features
+    "_LIBCPP_HAS_NO_EXPERIMENTAL_TZDB",
+    "_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM",
+    "_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN",
+    "_LIBCPP_HAS_NO_INCOMPLETE_PSTL",
+};
+
+class internal_ftm_use_callbacks : public clang::PPCallbacks {
+public:
+  internal_ftm_use_callbacks(clang::tidy::ClangTidyCheck& check) : check_(check) {}
+
+  void Defined(const clang::Token& token,
+               const clang::MacroDefinition& macro_definition,
+               clang::SourceRange location) override {
+    check_macro(token.getIdentifierInfo()->getName(), location.getBegin());
+  }
+
+  void Ifdef(clang::SourceLocation location, const clang::Token& token, const clang::MacroDefinition&) override {
+    check_macro(token.getIdentifierInfo()->getName(), location);
+  }
+
+  void Elifdef(clang::SourceLocation location, const clang::Token& token, const clang::MacroDefinition&) override {
+    check_macro(token.getIdentifierInfo()->getName(), location);
+  }
+
+  void Ifndef(clang::SourceLocation location, const clang::Token& token, const clang::MacroDefinition&) override {
+    check_macro(token.getIdentifierInfo()->getName(), location);
+  }
+
+  void Elifndef(clang::SourceLocation location, const clang::Token& token, const clang::MacroDefinition&) override {
+    check_macro(token.getIdentifierInfo()->getName(), location);
+  }
+
+private:
+  void check_macro(std::string_view macro, clang::SourceLocation location) {
+    if (macro.starts_with("_LIBCPP_HAS_") && std::ranges::find(valid_macros, macro) == valid_macros.end()) {
+      check_.diag(location, std::string("\'") + std::string{macro} + "' is always defined to 1 or 0.");
+    }
+  }
+
+  clang::tidy::ClangTidyCheck& check_;
+};
+} // namespace
+
+internal_ftm_use::internal_ftm_use(llvm::StringRef name, clang::tidy::ClangTidyContext* context)
+    : clang::tidy::ClangTidyCheck(name, context) {}
+
+void internal_ftm_use::registerPPCallbacks(const clang::SourceManager& source_manager,
+                                           clang::Preprocessor* preprocessor,
+                                           clang::Preprocessor* module_expander) {
+  preprocessor->addPPCallbacks(std::make_unique<internal_ftm_use_callbacks>(*this));
+}
+
+} // namespace libcpp
diff --git a/libcxx/test/tools/clang_tidy_checks/internal_ftm_use.hpp b/libcxx/test/tools/clang_tidy_checks/internal_ftm_use.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..90c5448f6b38e04933ef8205929d56ad7095d0b0
--- /dev/null
+++ b/libcxx/test/tools/clang_tidy_checks/internal_ftm_use.hpp
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang-tidy/ClangTidyCheck.h"
+
+namespace libcpp {
+class internal_ftm_use : public clang::tidy::ClangTidyCheck {
+public:
+  internal_ftm_use(llvm::StringRef, clang::tidy::ClangTidyContext*);
+  void registerPPCallbacks(const clang::SourceManager& source_manager,
+                           clang::Preprocessor* preprocessor,
+                           clang::Preprocessor* module_expander) override;
+};
+} // namespace libcpp
diff --git a/libcxx/test/tools/clang_tidy_checks/libcpp_module.cpp b/libcxx/test/tools/clang_tidy_checks/libcpp_module.cpp
index 16acf32ae55b38831f5a95f1594b4ca4936c182b..54beed5e30be58f912803c321bd50b38e692069c 100644
--- a/libcxx/test/tools/clang_tidy_checks/libcpp_module.cpp
+++ b/libcxx/test/tools/clang_tidy_checks/libcpp_module.cpp
@@ -12,6 +12,7 @@
 #include "abi_tag_on_virtual.hpp"
 #include "header_exportable_declarations.hpp"
 #include "hide_from_abi.hpp"
+#include "internal_ftm_use.hpp"
 #include "proper_version_checks.hpp"
 #include "qualify_declval.hpp"
 #include "robust_against_adl.hpp"
@@ -24,6 +25,7 @@ public:
     check_factories.registerCheck<libcpp::abi_tag_on_virtual>("libcpp-avoid-abi-tag-on-virtual");
     check_factories.registerCheck<libcpp::header_exportable_declarations>("libcpp-header-exportable-declarations");
     check_factories.registerCheck<libcpp::hide_from_abi>("libcpp-hide-from-abi");
+    check_factories.registerCheck<libcpp::internal_ftm_use>("libcpp-internal-ftms");
     check_factories.registerCheck<libcpp::proper_version_checks>("libcpp-cpp-version-check");
     check_factories.registerCheck<libcpp::robust_against_adl_check>("libcpp-robust-against-adl");
     check_factories.registerCheck<libcpp::uglify_attributes>("libcpp-uglify-attributes");
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index 0ce1def5f372240c94219ea9622d79c8243adbfe..0b72f8c4031031191dd098fb5c7cb66e702a4b8f 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -7,7 +7,7 @@
 #
 # ===----------------------------------------------------------------------===##
 
-set -ex
+set -e
 set -o pipefail
 unset LANG
 unset LC_ALL
@@ -96,12 +96,37 @@ if [ -z "${CMAKE}" ]; then
     fi
 fi
 
+function step() {
+  endstep
+  set +x
+  if [[ ! -z ${GITHUB_ACTIONS+x} ]]; then
+    echo "::group::$1"
+    export IN_GROUP=1
+  else
+    echo "--- $1"
+  fi
+  set -x
+}
+
+function endstep() {
+  set +x
+  if [[ ! -z ${GITHUB_ACTIONS+x} ]] && [[ ! -z ${IN_GROUP+x} ]]; then
+    echo "::endgroup::"
+    unset IN_GROUP
+  fi
+  set -x
+}
+
+function error() {
+    echo "::error::$1"
+}
+
 function clean() {
     rm -rf "${BUILD_DIR}"
 }
 
 function generate-cmake-base() {
-    echo "--- Generating CMake"
+    step "Generating CMake"
     ${CMAKE} \
           -S "${MONOREPO_ROOT}/runtimes" \
           -B "${BUILD_DIR}" \
@@ -138,29 +163,32 @@ function generate-cmake-android() {
 }
 
 function check-runtimes() {
-    echo "+++ Running the libc++ tests"
+    step "Building libc++ test dependencies"
+    ${NINJA} -vC "${BUILD_DIR}" cxx-test-depends
+
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
 
-    echo "+++ Running the libc++abi tests"
+    step "Running the libc++abi tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxxabi
 
-    echo "+++ Running the libunwind tests"
+    step "Running the libunwind tests"
     ${NINJA} -vC "${BUILD_DIR}" check-unwind
 }
 
 # TODO: The goal is to test this against all configurations. We should also move
 #       this to the Lit test suite instead of being a separate CMake target.
 function check-abi-list() {
-    echo "+++ Running the libc++ ABI list test"
+    step "Running the libc++ ABI list test"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx-abilist || (
-        echo "+++ Generating the libc++ ABI list after failed check"
+        error "Generating the libc++ ABI list after failed check"
         ${NINJA} -vC "${BUILD_DIR}" generate-cxx-abilist
         false
     )
 }
 
 function check-cxx-benchmarks() {
-    echo "--- Running the benchmarks"
+    step "Running the benchmarks"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx-benchmarks
 }
 
@@ -170,12 +198,13 @@ function test-armv7m-picolibc() {
     # To make it easier to get this builder up and running, build picolibc
     # from scratch. Anecdotally, the build-picolibc script takes about 16 seconds.
     # This could be optimised by building picolibc into the Docker container.
+    step "Building picolibc from source"
     ${MONOREPO_ROOT}/libcxx/utils/ci/build-picolibc.sh \
         --build-dir "${BUILD_DIR}" \
         --install-dir "${INSTALL_DIR}" \
         --target armv7m-none-eabi
 
-    echo "--- Generating CMake"
+    step "Generating CMake for compiler-rt"
     flags="--sysroot=${INSTALL_DIR}"
     ${CMAKE} \
         -S "${MONOREPO_ROOT}/compiler-rt" \
@@ -187,6 +216,8 @@ function test-armv7m-picolibc() {
         -DCMAKE_CXX_FLAGS="${flags}" \
         -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON \
         "${@}"
+
+    step "Generating CMake for libc++"
     generate-cmake \
         -DLIBCXX_TEST_CONFIG="armv7m-picolibc-libc++.cfg.in" \
         -DLIBCXXABI_TEST_CONFIG="armv7m-picolibc-libc++abi.cfg.in" \
@@ -195,6 +226,7 @@ function test-armv7m-picolibc() {
         -DCMAKE_CXX_FLAGS="${flags}" \
         "${@}"
 
+    step "Installing compiler-rt"
     ${NINJA} -vC "${BUILD_DIR}/compiler-rt" install
 
     # Prior to clang 19, armv7m-none-eabi normalised to armv7m-none-unknown-eabi.
@@ -208,9 +240,9 @@ function test-armv7m-picolibc() {
 }
 
 # Print the version of a few tools to aid diagnostics in some cases
+step "Diagnose tools in use"
 ${CMAKE} --version
 ${NINJA} --version
-
 if [ ! -z "${CXX}" ]; then ${CXX} --version; fi
 
 case "${BUILDER}" in
@@ -220,10 +252,9 @@ check-generated-output)
     clean
     generate-cmake
 
-    set +x # Printing all the commands below just creates extremely confusing output
-
     # Reject patches that forgot to re-run the generator scripts.
-    echo "+++ Making sure the generator scripts were run"
+    step "Making sure the generator scripts were run"
+    set +x # Printing all the commands below just creates extremely confusing output
     ${NINJA} -vC "${BUILD_DIR}" libcxx-generate-files
     git diff | tee ${BUILD_DIR}/generated_output.patch
     git ls-files -o --exclude-standard | tee ${BUILD_DIR}/generated_output.status
@@ -235,9 +266,8 @@ check-generated-output)
         false
     fi
 
-    # Reject patches that introduce non-ASCII characters or hard tabs.
-    # Depends on LC_COLLATE set at the top of this script.
-    set -x
+    # This depends on LC_COLLATE set at the top of this script.
+    step "Reject patches that introduce non-ASCII characters or hard tabs."
     ! grep -rn '[^ -~]' libcxx/include libcxx/src libcxx/test \
            --exclude '*.dat' \
            --exclude '*unicode*.cpp' \
@@ -345,7 +375,7 @@ generic-ubsan)
 bootstrapping-build)
     clean
 
-    echo "--- Generating CMake"
+    step "Generating CMake"
     ${CMAKE} \
           -S "${MONOREPO_ROOT}/llvm" \
           -B "${BUILD_DIR}" \
@@ -362,14 +392,14 @@ bootstrapping-build)
           -DLLVM_ENABLE_ASSERTIONS=ON \
           -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests"
 
-    echo "+++ Running the LLDB libc++ data formatter tests"
+    step "Running the LLDB libc++ data formatter tests"
     ${NINJA} -vC "${BUILD_DIR}" lldb-api-test-deps
     ${BUILD_DIR}/bin/llvm-lit -sv --param dotest-args='--category libc++' "${MONOREPO_ROOT}/lldb/test/API"
 
-    echo "--- Running the libc++ and libc++abi tests"
+    step "Running the libc++ and libc++abi tests"
     ${NINJA} -vC "${BUILD_DIR}" check-runtimes
 
-    echo "+++ Installing libc++ and libc++abi to a fake location"
+    step "Installing libc++ and libc++abi to a fake location"
     ${NINJA} -vC "${BUILD_DIR}" install-runtimes
 
     ccache -s
@@ -502,6 +532,7 @@ generic-optimized-speed)
 apple-configuration)
     clean
 
+    step "Installing libc++ with the Apple system configuration"
     arch="$(uname -m)"
     xcrun --sdk macosx                                              \
         ${MONOREPO_ROOT}/libcxx/utils/ci/apple-install-libcxx.sh    \
@@ -512,6 +543,7 @@ apple-configuration)
             --architectures "${arch}"                               \
             --version "999.99"
 
+    step "Running tests against Apple-configured libc++"
     # TODO: It would be better to run the tests against the fake-installed version of libc++ instead
     xcrun --sdk macosx ninja -vC "${BUILD_DIR}/${arch}" check-cxx check-cxxabi check-cxx-abilist
 ;;
@@ -524,6 +556,7 @@ apple-system-hardened)
     params+=";hardening_mode=fast"
 
     # In the Apple system configuration, we build libc++ and libunwind separately.
+    step "Installing libc++ and libc++abi in Apple-system configuration"
     ${CMAKE} \
         -S "${MONOREPO_ROOT}/runtimes" \
         -B "${BUILD_DIR}/cxx" \
@@ -539,6 +572,7 @@ apple-system-hardened)
         -DLIBCXX_TEST_PARAMS="${params}" \
         -DLIBCXXABI_TEST_PARAMS="${params}"
 
+    step "Installing libunwind in Apple-system configuration"
     ${CMAKE} \
         -S "${MONOREPO_ROOT}/runtimes" \
         -B "${BUILD_DIR}/unwind" \
@@ -551,13 +585,13 @@ apple-system-hardened)
         -DLIBUNWIND_TEST_PARAMS="${params}" \
         -DCMAKE_INSTALL_NAME_DIR="/usr/lib/system"
 
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}/cxx" check-cxx
 
-    echo "+++ Running the libc++abi tests"
+    step "Running the libc++abi tests"
     ${NINJA} -vC "${BUILD_DIR}/cxx" check-cxxabi
 
-    echo "+++ Running the libunwind tests"
+    step "Running the libunwind tests"
     ${NINJA} -vC "${BUILD_DIR}/unwind" check-unwind
 ;;
 apple-system)
@@ -568,6 +602,7 @@ apple-system)
     params="target_triple=${arch}-apple-macosx${version}"
 
     # In the Apple system configuration, we build libc++ and libunwind separately.
+    step "Installing libc++ and libc++abi in Apple-system configuration"
     ${CMAKE} \
         -S "${MONOREPO_ROOT}/runtimes" \
         -B "${BUILD_DIR}/cxx" \
@@ -583,6 +618,7 @@ apple-system)
         -DLIBCXX_TEST_PARAMS="${params}" \
         -DLIBCXXABI_TEST_PARAMS="${params}"
 
+    step "Installing libunwind in Apple-system configuration"
     ${CMAKE} \
         -S "${MONOREPO_ROOT}/runtimes" \
         -B "${BUILD_DIR}/unwind" \
@@ -595,13 +631,13 @@ apple-system)
         -DLIBUNWIND_TEST_PARAMS="${params}" \
         -DCMAKE_INSTALL_NAME_DIR="/usr/lib/system"
 
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}/cxx" check-cxx
 
-    echo "+++ Running the libc++abi tests"
+    step "Running the libc++abi tests"
     ${NINJA} -vC "${BUILD_DIR}/cxx" check-cxxabi
 
-    echo "+++ Running the libunwind tests"
+    step "Running the libunwind tests"
     ${NINJA} -vC "${BUILD_DIR}/unwind" check-unwind
 ;;
 benchmarks)
@@ -664,13 +700,13 @@ clang-cl-dll)
     # anyway), thus just disable the experimental library. Remove this
     # setting when cmake and the test driver does the right thing automatically.
     generate-cmake-libcxx-win -DLIBCXX_TEST_PARAMS="enable_experimental=False"
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
 ;;
 clang-cl-static)
     clean
     generate-cmake-libcxx-win -DLIBCXX_ENABLE_SHARED=OFF
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
 ;;
 clang-cl-no-vcruntime)
@@ -681,14 +717,14 @@ clang-cl-no-vcruntime)
     # exceptions enabled.
     generate-cmake-libcxx-win -DLIBCXX_TEST_PARAMS="enable_experimental=False" \
                               -DLIBCXX_TEST_CONFIG="llvm-libc++-shared-no-vcruntime-clangcl.cfg.in"
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
 ;;
 clang-cl-debug)
     clean
     generate-cmake-libcxx-win -DLIBCXX_TEST_PARAMS="enable_experimental=False" \
                               -DCMAKE_BUILD_TYPE=Debug
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
 ;;
 clang-cl-static-crt)
@@ -697,7 +733,7 @@ clang-cl-static-crt)
     # the static CRT, as opposed to "MultiThreadedDLL" which is the default).
     generate-cmake-libcxx-win -DLIBCXX_ENABLE_SHARED=OFF \
                               -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
 ;;
 mingw-dll)
@@ -743,6 +779,7 @@ mingw-incomplete-sysroot)
           -C "${MONOREPO_ROOT}/libcxx/cmake/caches/MinGW.cmake"
     # Only test that building succeeds; there's not much extra value in running
     # the tests here, as it would be equivalent to the mingw-dll config above.
+    step "Building the runtimes"
     ${NINJA} -vC "${BUILD_DIR}"
 ;;
 aix)
@@ -760,7 +797,7 @@ android-ndk-*)
     ANDROID_EMU_IMG="${BUILDER#android-ndk-}"
     . "${MONOREPO_ROOT}/libcxx/utils/ci/vendor/android/emulator-functions.sh"
     if ! validate_emu_img "${ANDROID_EMU_IMG}"; then
-        echo "error: android-ndk suffix must be a valid emulator image (${ANDROID_EMU_IMG})" >&2
+        error "android-ndk suffix must be a valid emulator image (${ANDROID_EMU_IMG})" >&2
         exit 1
     fi
     ARCH=$(arch_of_emu_img ${ANDROID_EMU_IMG})
@@ -792,9 +829,9 @@ android-ndk-*)
     # directories.
     adb shell mkdir -p /data/local/tmp/adb_run
     adb push "${BUILD_DIR}/lib/libc++_shared.so" /data/local/tmp/libc++/libc++_shared.so
-    echo "+++ Running the libc++ tests"
+    step "Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
-    echo "+++ Running the libc++abi tests"
+    step "Running the libc++abi tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxxabi
 ;;
 #################################################################
@@ -810,3 +847,5 @@ android-ndk-*)
     exit 1
 ;;
 esac
+
+endstep # Make sure we close any still-open output group
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index db14c1781dc35a0ba0a3e2f4e8439b2b9167d2cd..7ab1af93d17740642970754d0590e2958470d7b3 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -280,7 +280,7 @@ feature_test_macros = [
                 "string_view",
             ],
             "test_suite_guard": "defined(__cpp_char8_t)",
-            "libcxx_guard": "!defined(_LIBCPP_HAS_NO_CHAR8_T)",
+            "libcxx_guard": "_LIBCPP_HAS_CHAR8_T",
         },
         {
             "name": "__cpp_lib_chrono",
diff --git a/libcxx/utils/libcxx/header_information.py b/libcxx/utils/libcxx/header_information.py
index fd48b35a88b558cb78740d87076d81893c705202..6bebf3302ffae943c4a1d338c1379ab1f723df08 100644
--- a/libcxx/utils/libcxx/header_information.py
+++ b/libcxx/utils/libcxx/header_information.py
@@ -10,8 +10,8 @@ import os, pathlib
 
 header_restrictions = {
     # headers with #error directives
-    "atomic": "!defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)",
-    "stdatomic.h": "!defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)",
+    "atomic": "_LIBCPP_HAS_ATOMIC_HEADER",
+    "stdatomic.h": "_LIBCPP_HAS_ATOMIC_HEADER",
 
     # headers with #error directives
     "ios": "!defined(_LIBCPP_HAS_NO_LOCALIZATION)",
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index 43400c6e8d9af19ed432ba72bec13c618c4b0fd4..ac1ee69d5f11c93013d28c7d29b30f42ff801ef9 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -131,11 +131,6 @@ if (NOT LIBCXXABI_ENABLE_SHARED AND NOT LIBCXXABI_ENABLE_STATIC)
   message(FATAL_ERROR "libc++abi must be built as either a shared or static library.")
 endif()
 
-# TODO: Remove this, which shouldn't be necessary since we know we're being built
-#       side-by-side with libc++.
-set(LIBCXXABI_LIBCXX_INCLUDES "" CACHE PATH
-    "Specify path to libc++ includes.")
-
 set(LIBCXXABI_HERMETIC_STATIC_LIBRARY_DEFAULT OFF)
 if (WIN32)
   set(LIBCXXABI_HERMETIC_STATIC_LIBRARY_DEFAULT ON)
diff --git a/libcxxabi/src/fallback_malloc.cpp b/libcxxabi/src/fallback_malloc.cpp
index 76bd2e9bcd9f7452a8d7f30453e46d6dfa5c3424..75788fe9be8d9e7135ebce1adc58983acd43b932 100644
--- a/libcxxabi/src/fallback_malloc.cpp
+++ b/libcxxabi/src/fallback_malloc.cpp
@@ -259,7 +259,7 @@ void* __aligned_malloc_with_fallback(size_t size) {
 #if defined(_WIN32)
   if (void* dest = std::__libcpp_aligned_alloc(alignof(__aligned_type), size))
     return dest;
-#elif defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
+#elif !_LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
   if (void* dest = ::malloc(size))
     return dest;
 #else
@@ -286,7 +286,7 @@ void __aligned_free_with_fallback(void* ptr) {
   if (is_fallback_ptr(ptr))
     fallback_free(ptr);
   else {
-#if defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
+#if !_LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
     ::free(ptr);
 #else
     std::__libcpp_aligned_free(ptr);
diff --git a/libcxxabi/src/stdlib_new_delete.cpp b/libcxxabi/src/stdlib_new_delete.cpp
index bd576e6aeed744ccf89735688134e621e8d7f162..f386b28f0cfe64d7c94e2cb53caf6755e509cde8 100644
--- a/libcxxabi/src/stdlib_new_delete.cpp
+++ b/libcxxabi/src/stdlib_new_delete.cpp
@@ -24,12 +24,12 @@
 #  error The _LIBCPP_WEAK macro should be already defined by libc++
 #endif
 
-#if defined(_LIBCXXABI_NO_EXCEPTIONS) != defined(_LIBCPP_HAS_NO_EXCEPTIONS)
+#if defined(_LIBCXXABI_NO_EXCEPTIONS) != !_LIBCPP_HAS_EXCEPTIONS
 #  error libc++ and libc++abi seem to disagree on whether exceptions are enabled
 #endif
 
 inline void __throw_bad_alloc_shim() {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+#if _LIBCPP_HAS_EXCEPTIONS
   throw std::bad_alloc();
 #else
   __abort_message("bad_alloc was thrown in -fno-exceptions mode");
@@ -71,7 +71,7 @@ _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new(std
 }
 
 _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
-#ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#if !_LIBCPP_HAS_EXCEPTIONS
 #  if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new)),
@@ -99,7 +99,7 @@ _LIBCPP_MAKE_OVERRIDABLE_FUNCTION_DETECTABLE _LIBCPP_WEAK void* operator new[](s
 }
 
 _LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept {
-#ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#if !_LIBCPP_HAS_EXCEPTIONS
 #  if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t)>(&operator new[])),
@@ -134,7 +134,7 @@ _LIBCPP_WEAK void operator delete[](void* ptr, const std::nothrow_t&) noexcept {
 
 _LIBCPP_WEAK void operator delete[](void* ptr, size_t) noexcept { ::operator delete[](ptr); }
 
-#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
+#if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 
 static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignment) {
   if (size == 0)
@@ -165,7 +165,7 @@ operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
 }
 
 _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
-#  ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new)),
@@ -194,7 +194,7 @@ operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
 }
 
 _LIBCPP_WEAK void* operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
-#  ifdef _LIBCPP_HAS_NO_EXCEPTIONS
+#  if !_LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION
   _LIBCPP_ASSERT_SHIM(
       !std::__is_function_overridden(static_cast<void* (*)(std::size_t, std::align_val_t)>(&operator new[])),
@@ -240,5 +240,5 @@ _LIBCPP_WEAK void operator delete[](void* ptr, size_t, std::align_val_t alignmen
   ::operator delete[](ptr, alignment);
 }
 
-#endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#endif // _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
 // ------------------ END COPY ------------------
diff --git a/lld/ELF/AArch64ErrataFix.cpp b/lld/ELF/AArch64ErrataFix.cpp
index b1f6c424688b36660adba81d93660ec668b0b401..cd8fbf16f5b8397697e1fa8ce518c81037aead53 100644
--- a/lld/ELF/AArch64ErrataFix.cpp
+++ b/lld/ELF/AArch64ErrataFix.cpp
@@ -398,9 +398,9 @@ Patch843419Section::Patch843419Section(Ctx &ctx, InputSection *p, uint64_t off)
       patchee(p), patcheeOffset(off) {
   this->parent = p->getParent();
   patchSym = addSyntheticLocal(
-      saver().save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC,
-      0, getSize(), *this);
-  addSyntheticLocal(saver().save("$x"), STT_NOTYPE, 0, 0, *this);
+      ctx, saver().save("__CortexA53843419_" + utohexstr(getLDSTAddr())),
+      STT_FUNC, 0, getSize(), *this);
+  addSyntheticLocal(ctx, saver().save("$x"), STT_NOTYPE, 0, 0, *this);
 }
 
 uint64_t Patch843419Section::getLDSTAddr() const {
diff --git a/lld/ELF/ARMErrataFix.cpp b/lld/ELF/ARMErrataFix.cpp
index 6dc6432c40ea5cb76bc8084aead2b85bb4d310d6..630084afd509ce7d6323f2db0d3b43f3b8d3299c 100644
--- a/lld/ELF/ARMErrataFix.cpp
+++ b/lld/ELF/ARMErrataFix.cpp
@@ -141,9 +141,10 @@ Patch657417Section::Patch657417Section(Ctx &ctx, InputSection *p, uint64_t off,
       patchee(p), patcheeOffset(off), instr(instr), isARM(isARM) {
   parent = p->getParent();
   patchSym = addSyntheticLocal(
-      saver().save("__CortexA8657417_" + utohexstr(getBranchAddr())), STT_FUNC,
-      isARM ? 0 : 1, getSize(), *this);
-  addSyntheticLocal(saver().save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0, *this);
+      ctx, saver().save("__CortexA8657417_" + utohexstr(getBranchAddr())),
+      STT_FUNC, isARM ? 0 : 1, getSize(), *this);
+  addSyntheticLocal(ctx, saver().save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0,
+                    *this);
 }
 
 uint64_t Patch657417Section::getBranchAddr() const {
@@ -259,6 +260,7 @@ struct ScanResult {
 // branch so the minimum offset for a patch is 4.
 static ScanResult scanCortexA8Errata657417(InputSection *isec, uint64_t &off,
                                            uint64_t limit) {
+  Ctx &ctx = isec->getCtx();
   uint64_t isecAddr = isec->getVA(0);
   // Advance Off so that (isecAddr + off) modulo 0x1000 is at least 0xffa. We
   // need to check for a 32-bit instruction immediately before a 32-bit branch
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index f595504e621181eb723f5202eba9feef75a6df6e..260307ac4c3dcb7d6477d2060896afd73b88461e 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -255,23 +255,23 @@ RelType AArch64::getDynRel(RelType type) const {
 int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {
   switch (type) {
   case R_AARCH64_TLSDESC:
-    return read64(buf + 8);
+    return read64(ctx, buf + 8);
   case R_AARCH64_NONE:
   case R_AARCH64_GLOB_DAT:
   case R_AARCH64_JUMP_SLOT:
     return 0;
   case R_AARCH64_ABS16:
   case R_AARCH64_PREL16:
-    return SignExtend64<16>(read16(buf));
+    return SignExtend64<16>(read16(ctx, buf));
   case R_AARCH64_ABS32:
   case R_AARCH64_PREL32:
-    return SignExtend64<32>(read32(buf));
+    return SignExtend64<32>(read32(ctx, buf));
   case R_AARCH64_ABS64:
   case R_AARCH64_PREL64:
   case R_AARCH64_RELATIVE:
   case R_AARCH64_IRELATIVE:
   case R_AARCH64_TLS_TPREL64:
-    return read64(buf);
+    return read64(ctx, buf);
 
     // The following relocation types all point at instructions, and
     // relocate an immediate field in the instruction.
@@ -355,12 +355,12 @@ int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {
 }
 
 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const {
-  write64(buf, ctx.in.plt->getVA());
+  write64(ctx, buf, ctx.in.plt->getVA());
 }
 
 void AArch64::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
   if (ctx.arg.writeAddends)
-    write64(buf, s.getVA());
+    write64(ctx, buf, s.getVA());
 }
 
 void AArch64::writePltHeader(uint8_t *buf) const {
@@ -484,18 +484,18 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   switch (rel.type) {
   case R_AARCH64_ABS16:
   case R_AARCH64_PREL16:
-    checkIntUInt(loc, val, 16, rel);
-    write16(loc, val);
+    checkIntUInt(ctx, loc, val, 16, rel);
+    write16(ctx, loc, val);
     break;
   case R_AARCH64_ABS32:
   case R_AARCH64_PREL32:
-    checkIntUInt(loc, val, 32, rel);
-    write32(loc, val);
+    checkIntUInt(ctx, loc, val, 32, rel);
+    write32(ctx, loc, val);
     break;
   case R_AARCH64_PLT32:
   case R_AARCH64_GOTPCREL32:
-    checkInt(loc, val, 32, rel);
-    write32(loc, val);
+    checkInt(ctx, loc, val, 32, rel);
+    write32(ctx, loc, val);
     break;
   case R_AARCH64_ABS64:
     // AArch64 relocations to tagged symbols have extended semantics, as
@@ -508,12 +508,12 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     if (rel.sym && rel.sym->isTagged() &&
         (rel.addend < 0 ||
          rel.addend >= static_cast<int64_t>(rel.sym->getSize())))
-      write64(loc, -rel.addend);
+      write64(ctx, loc, -rel.addend);
     else
-      write64(loc, val);
+      write64(ctx, loc, val);
     break;
   case R_AARCH64_PREL64:
-    write64(loc, val);
+    write64(ctx, loc, val);
     break;
   case R_AARCH64_AUTH_ABS64:
     // If val is wider than 32 bits, the relocation must have been moved from
@@ -526,7 +526,7 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     //   finalizeAddressDependentContent(). Writing the value is harmless
     //   because dynamic linking ignores it.
     if (isInt<32>(val))
-      write32(loc, val);
+      write32(ctx, loc, val);
     break;
   case R_AARCH64_ADD_ABS_LO12_NC:
     write32Imm12(loc, val);
@@ -535,13 +535,13 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_ADR_PREL_PG_HI21:
   case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
   case R_AARCH64_TLSDESC_ADR_PAGE21:
-    checkInt(loc, val, 33, rel);
+    checkInt(ctx, loc, val, 33, rel);
     [[fallthrough]];
   case R_AARCH64_ADR_PREL_PG_HI21_NC:
     write32AArch64Addr(loc, val >> 12);
     break;
   case R_AARCH64_ADR_PREL_LO21:
-    checkInt(loc, val, 21, rel);
+    checkInt(ctx, loc, val, 21, rel);
     write32AArch64Addr(loc, val);
     break;
   case R_AARCH64_JUMP26:
@@ -555,14 +555,14 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     write32le(loc, 0x14000000);
     [[fallthrough]];
   case R_AARCH64_CALL26:
-    checkInt(loc, val, 28, rel);
+    checkInt(ctx, loc, val, 28, rel);
     writeMaskedBits32le(loc, (val & 0x0FFFFFFC) >> 2, 0x0FFFFFFC >> 2);
     break;
   case R_AARCH64_CONDBR19:
   case R_AARCH64_LD_PREL_LO19:
   case R_AARCH64_GOT_LD_PREL19:
-    checkAlignment(loc, val, 4, rel);
-    checkInt(loc, val, 21, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
+    checkInt(ctx, loc, val, 21, rel);
     writeMaskedBits32le(loc, (val & 0x1FFFFC) << 3, 0x1FFFFC << 3);
     break;
   case R_AARCH64_LDST8_ABS_LO12_NC:
@@ -571,12 +571,12 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     break;
   case R_AARCH64_LDST16_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     write32Imm12(loc, getBits(val, 1, 11));
     break;
   case R_AARCH64_LDST32_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
-    checkAlignment(loc, val, 4, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
     write32Imm12(loc, getBits(val, 2, 11));
     break;
   case R_AARCH64_LDST64_ABS_LO12_NC:
@@ -584,32 +584,32 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
   case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
   case R_AARCH64_TLSDESC_LD64_LO12:
-    checkAlignment(loc, val, 8, rel);
+    checkAlignment(ctx, loc, val, 8, rel);
     write32Imm12(loc, getBits(val, 3, 11));
     break;
   case R_AARCH64_LDST128_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
-    checkAlignment(loc, val, 16, rel);
+    checkAlignment(ctx, loc, val, 16, rel);
     write32Imm12(loc, getBits(val, 4, 11));
     break;
   case R_AARCH64_LD64_GOTPAGE_LO15:
-    checkAlignment(loc, val, 8, rel);
+    checkAlignment(ctx, loc, val, 8, rel);
     write32Imm12(loc, getBits(val, 3, 14));
     break;
   case R_AARCH64_MOVW_UABS_G0:
-    checkUInt(loc, val, 16, rel);
+    checkUInt(ctx, loc, val, 16, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_UABS_G0_NC:
     writeMaskedBits32le(loc, (val & 0xFFFF) << 5, 0xFFFF << 5);
     break;
   case R_AARCH64_MOVW_UABS_G1:
-    checkUInt(loc, val, 32, rel);
+    checkUInt(ctx, loc, val, 32, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_UABS_G1_NC:
     writeMaskedBits32le(loc, (val & 0xFFFF0000) >> 11, 0xFFFF0000 >> 11);
     break;
   case R_AARCH64_MOVW_UABS_G2:
-    checkUInt(loc, val, 48, rel);
+    checkUInt(ctx, loc, val, 48, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_UABS_G2_NC:
     writeMaskedBits32le(loc, (val & 0xFFFF00000000) >> 27,
@@ -622,7 +622,7 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_MOVW_PREL_G0:
   case R_AARCH64_MOVW_SABS_G0:
   case R_AARCH64_TLSLE_MOVW_TPREL_G0:
-    checkInt(loc, val, 17, rel);
+    checkInt(ctx, loc, val, 17, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_PREL_G0_NC:
   case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
@@ -631,7 +631,7 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_MOVW_PREL_G1:
   case R_AARCH64_MOVW_SABS_G1:
   case R_AARCH64_TLSLE_MOVW_TPREL_G1:
-    checkInt(loc, val, 33, rel);
+    checkInt(ctx, loc, val, 33, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_PREL_G1_NC:
   case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
@@ -640,7 +640,7 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_MOVW_PREL_G2:
   case R_AARCH64_MOVW_SABS_G2:
   case R_AARCH64_TLSLE_MOVW_TPREL_G2:
-    checkInt(loc, val, 49, rel);
+    checkInt(ctx, loc, val, 49, rel);
     [[fallthrough]];
   case R_AARCH64_MOVW_PREL_G2_NC:
     writeSMovWImm(loc, val >> 32);
@@ -649,11 +649,11 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     writeSMovWImm(loc, val >> 48);
     break;
   case R_AARCH64_TSTBR14:
-    checkInt(loc, val, 16, rel);
+    checkInt(ctx, loc, val, 16, rel);
     writeMaskedBits32le(loc, (val & 0xFFFC) << 3, 0xFFFC << 3);
     break;
   case R_AARCH64_TLSLE_ADD_TPREL_HI12:
-    checkUInt(loc, val, 24, rel);
+    checkUInt(ctx, loc, val, 24, rel);
     write32Imm12(loc, val >> 12);
     break;
   case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
@@ -662,7 +662,7 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     break;
   case R_AARCH64_TLSDESC:
     // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word.
-    write64(loc + 8, val);
+    write64(ctx, loc + 8, val);
     break;
   default:
     llvm_unreachable("unknown relocation");
@@ -682,7 +682,7 @@ void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
   //   movk    x0, #0x10
   //   nop
   //   nop
-  checkUInt(loc, val, 32, rel);
+  checkUInt(ctx, loc, val, 32, rel);
 
   switch (rel.type) {
   case R_AARCH64_TLSDESC_ADD_LO12:
@@ -734,7 +734,7 @@ void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
 
 void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
                              uint64_t val) const {
-  checkUInt(loc, val, 32, rel);
+  checkUInt(ctx, loc, val, 32, rel);
 
   if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) {
     // Generate MOVZ.
@@ -1150,7 +1150,7 @@ addTaggedSymbolReferences(InputSectionBase &sec,
 // symbols should also be built with tagging. But, to handle these cases, we
 // demote the symbol to be untagged.
 void elf::createTaggedSymbols(Ctx &ctx) {
-  assert(hasMemtag());
+  assert(hasMemtag(ctx));
 
   // First, collect all symbols that are marked as tagged, and count how many
   // times they're marked as tagged.
diff --git a/lld/ELF/Arch/AMDGPU.cpp b/lld/ELF/Arch/AMDGPU.cpp
index 130da19d0e210d3ea6288d56870d2ad60cbb0198..b4bb029feb72abd4cb4e258499f2aa9c5bd385b3 100644
--- a/lld/ELF/Arch/AMDGPU.cpp
+++ b/lld/ELF/Arch/AMDGPU.cpp
@@ -167,7 +167,7 @@ void AMDGPU::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     break;
   case R_AMDGPU_REL16: {
     int64_t simm = (static_cast<int64_t>(val) - 4) / 4;
-    checkInt(loc, simm, 16, rel);
+    checkInt(ctx, loc, simm, 16, rel);
     write16le(loc, simm);
     break;
   }
@@ -211,7 +211,7 @@ int64_t AMDGPU::getImplicitAddend(const uint8_t *buf, RelType type) const {
     return 0;
   case R_AMDGPU_ABS64:
   case R_AMDGPU_RELATIVE64:
-    return read64(buf);
+    return read64(ctx, buf);
   default:
     internalLinkerError(getErrorLoc(ctx, buf),
                         "cannot read addend for relocation " + toString(type));
diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index d964994a4b3cffdb1a391a3c0f95a5b023c7e8f7..9bb3604ce61cc5d9dd4ed40ad0eceb2698e39099 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -208,28 +208,28 @@ RelType ARM::getDynRel(RelType type) const {
 }
 
 void ARM::writeGotPlt(uint8_t *buf, const Symbol &) const {
-  write32(buf, ctx.in.plt->getVA());
+  write32(ctx, buf, ctx.in.plt->getVA());
 }
 
 void ARM::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
   // An ARM entry is the address of the ifunc resolver function.
-  write32(buf, s.getVA());
+  write32(ctx, buf, s.getVA());
 }
 
 // Long form PLT Header that does not have any restrictions on the displacement
 // of the .plt from the .got.plt.
 static void writePltHeaderLong(Ctx &ctx, uint8_t *buf) {
-  write32(buf + 0, 0xe52de004);   //     str lr, [sp,#-4]!
-  write32(buf + 4, 0xe59fe004);   //     ldr lr, L2
-  write32(buf + 8, 0xe08fe00e);   // L1: add lr, pc, lr
-  write32(buf + 12, 0xe5bef008);  //     ldr pc, [lr, #8]
-  write32(buf + 16, 0x00000000);  // L2: .word   &(.got.plt) - L1 - 8
-  write32(buf + 20, 0xd4d4d4d4);  //     Pad to 32-byte boundary
-  write32(buf + 24, 0xd4d4d4d4);  //     Pad to 32-byte boundary
-  write32(buf + 28, 0xd4d4d4d4);
+  write32(ctx, buf + 0, 0xe52de004);  //     str lr, [sp,#-4]!
+  write32(ctx, buf + 4, 0xe59fe004);  //     ldr lr, L2
+  write32(ctx, buf + 8, 0xe08fe00e);  // L1: add lr, pc, lr
+  write32(ctx, buf + 12, 0xe5bef008); //     ldr pc, [lr, #8]
+  write32(ctx, buf + 16, 0x00000000); // L2: .word   &(.got.plt) - L1 - 8
+  write32(ctx, buf + 20, 0xd4d4d4d4); //     Pad to 32-byte boundary
+  write32(ctx, buf + 24, 0xd4d4d4d4); //     Pad to 32-byte boundary
+  write32(ctx, buf + 28, 0xd4d4d4d4);
   uint64_t gotPlt = ctx.in.gotPlt->getVA();
   uint64_t l1 = ctx.in.plt->getVA() + 8;
-  write32(buf + 16, gotPlt - l1 - 8);
+  write32(ctx, buf + 16, gotPlt - l1 - 8);
 }
 
 // True if we should use Thumb PLTs, which currently require Thumb2, and are
@@ -255,15 +255,15 @@ void ARM::writePltHeader(uint8_t *buf) const {
     //
     uint64_t offset = ctx.in.gotPlt->getVA() - ctx.in.plt->getVA() - 16;
     assert(llvm::isUInt<32>(offset) && "This should always fit into a 32-bit offset");
-    write16(buf + 0, 0xb500);
+    write16(ctx, buf + 0, 0xb500);
     // Split into two halves to support endianness correctly.
-    write16(buf + 2, 0xf8df);
-    write16(buf + 4, 0xe008);
-    write16(buf + 6, 0x44fe);
+    write16(ctx, buf + 2, 0xf8df);
+    write16(ctx, buf + 4, 0xe008);
+    write16(ctx, buf + 6, 0x44fe);
     // Split into two halves to support endianness correctly.
-    write16(buf + 8, 0xf85e);
-    write16(buf + 10, 0xff08);
-    write32(buf + 12, offset);
+    write16(ctx, buf + 8, 0xf85e);
+    write16(ctx, buf + 10, 0xff08);
+    write32(ctx, buf + 12, offset);
 
     memcpy(buf + 16, trapInstr.data(), 4);  // Pad to 32-byte boundary
     memcpy(buf + 20, trapInstr.data(), 4);
@@ -287,10 +287,10 @@ void ARM::writePltHeader(uint8_t *buf) const {
       writePltHeaderLong(ctx, buf);
       return;
     }
-    write32(buf + 0, pltData[0]);
-    write32(buf + 4, pltData[1] | ((offset >> 20) & 0xff));
-    write32(buf + 8, pltData[2] | ((offset >> 12) & 0xff));
-    write32(buf + 12, pltData[3] | (offset & 0xfff));
+    write32(ctx, buf + 0, pltData[0]);
+    write32(ctx, buf + 4, pltData[1] | ((offset >> 20) & 0xff));
+    write32(ctx, buf + 8, pltData[2] | ((offset >> 12) & 0xff));
+    write32(ctx, buf + 12, pltData[3] | (offset & 0xfff));
     memcpy(buf + 16, trapInstr.data(), 4); // Pad to 32-byte boundary
     memcpy(buf + 20, trapInstr.data(), 4);
     memcpy(buf + 24, trapInstr.data(), 4);
@@ -300,24 +300,24 @@ void ARM::writePltHeader(uint8_t *buf) const {
 
 void ARM::addPltHeaderSymbols(InputSection &isec) const {
   if (useThumbPLTs(ctx)) {
-    addSyntheticLocal("$t", STT_NOTYPE, 0, 0, isec);
-    addSyntheticLocal("$d", STT_NOTYPE, 12, 0, isec);
+    addSyntheticLocal(ctx, "$t", STT_NOTYPE, 0, 0, isec);
+    addSyntheticLocal(ctx, "$d", STT_NOTYPE, 12, 0, isec);
   } else {
-    addSyntheticLocal("$a", STT_NOTYPE, 0, 0, isec);
-    addSyntheticLocal("$d", STT_NOTYPE, 16, 0, isec);
+    addSyntheticLocal(ctx, "$a", STT_NOTYPE, 0, 0, isec);
+    addSyntheticLocal(ctx, "$d", STT_NOTYPE, 16, 0, isec);
   }
 }
 
 // Long form PLT entries that do not have any restrictions on the displacement
 // of the .plt from the .got.plt.
-static void writePltLong(uint8_t *buf, uint64_t gotPltEntryAddr,
+static void writePltLong(Ctx &ctx, uint8_t *buf, uint64_t gotPltEntryAddr,
                          uint64_t pltEntryAddr) {
-  write32(buf + 0, 0xe59fc004);   //     ldr ip, L2
-  write32(buf + 4, 0xe08cc00f);   // L1: add ip, ip, pc
-  write32(buf + 8, 0xe59cf000);   //     ldr pc, [ip]
-  write32(buf + 12, 0x00000000);  // L2: .word   Offset(&(.got.plt) - L1 - 8
+  write32(ctx, buf + 0, 0xe59fc004);  //     ldr ip, L2
+  write32(ctx, buf + 4, 0xe08cc00f);  // L1: add ip, ip, pc
+  write32(ctx, buf + 8, 0xe59cf000);  //     ldr pc, [ip]
+  write32(ctx, buf + 12, 0x00000000); // L2: .word   Offset(&(.got.plt) - L1 - 8
   uint64_t l1 = pltEntryAddr + 4;
-  write32(buf + 12, gotPltEntryAddr - l1 - 8);
+  write32(ctx, buf + 12, gotPltEntryAddr - l1 - 8);
 }
 
 // The default PLT entries require the .got.plt to be within 128 Mb of the
@@ -339,12 +339,12 @@ void ARM::writePlt(uint8_t *buf, const Symbol &sym,
     };
     if (!llvm::isUInt<27>(offset)) {
       // We cannot encode the Offset, use the long form.
-      writePltLong(buf, sym.getGotPltVA(ctx), pltEntryAddr);
+      writePltLong(ctx, buf, sym.getGotPltVA(ctx), pltEntryAddr);
       return;
     }
-    write32(buf + 0, pltData[0] | ((offset >> 20) & 0xff));
-    write32(buf + 4, pltData[1] | ((offset >> 12) & 0xff));
-    write32(buf + 8, pltData[2] | (offset & 0xfff));
+    write32(ctx, buf + 0, pltData[0] | ((offset >> 20) & 0xff));
+    write32(ctx, buf + 4, pltData[1] | ((offset >> 12) & 0xff));
+    write32(ctx, buf + 8, pltData[2] | (offset & 0xfff));
     memcpy(buf + 12, trapInstr.data(), 4); // Pad to 16-byte boundary
   } else {
     uint64_t offset = sym.getGotPltVA(ctx) - pltEntryAddr - 12;
@@ -361,26 +361,26 @@ void ARM::writePlt(uint8_t *buf, const Symbol &sym,
     // where ip = r12 = 0xc
 
     // movw ip, #<lower 16 bits>
-    write16(buf + 2, 0x0c00); // use `ip`
+    write16(ctx, buf + 2, 0x0c00); // use `ip`
     relocateNoSym(buf, R_ARM_THM_MOVW_ABS_NC, offset);
 
     // movt ip, #<upper 16 bits>
-    write16(buf + 6, 0x0c00); // use `ip`
+    write16(ctx, buf + 6, 0x0c00); // use `ip`
     relocateNoSym(buf + 4, R_ARM_THM_MOVT_ABS, offset);
 
-    write16(buf + 8, 0x44fc);       // add ip, pc
-    write16(buf + 10, 0xf8dc);      // ldr.w   pc, [ip] (bottom half)
-    write16(buf + 12, 0xf000);      // ldr.w   pc, [ip] (upper half)
-    write16(buf + 14, 0xe7fc);      // Branch to previous instruction
+    write16(ctx, buf + 8, 0x44fc);  // add ip, pc
+    write16(ctx, buf + 10, 0xf8dc); // ldr.w   pc, [ip] (bottom half)
+    write16(ctx, buf + 12, 0xf000); // ldr.w   pc, [ip] (upper half)
+    write16(ctx, buf + 14, 0xe7fc); // Branch to previous instruction
   }
 }
 
 void ARM::addPltSymbols(InputSection &isec, uint64_t off) const {
   if (useThumbPLTs(ctx)) {
-    addSyntheticLocal("$t", STT_NOTYPE, off, 0, isec);
+    addSyntheticLocal(ctx, "$t", STT_NOTYPE, off, 0, isec);
   } else {
-    addSyntheticLocal("$a", STT_NOTYPE, off, 0, isec);
-    addSyntheticLocal("$d", STT_NOTYPE, off + 12, 0, isec);
+    addSyntheticLocal(ctx, "$a", STT_NOTYPE, off, 0, isec);
+    addSyntheticLocal(ctx, "$d", STT_NOTYPE, off + 12, 0, isec);
   }
 }
 
@@ -558,11 +558,12 @@ void ARM::encodeAluGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
   if (check && imm > 0xff)
     error(getErrorLoc(ctx, loc) + "unencodeable immediate " + Twine(val).str() +
           " for relocation " + toString(rel.type));
-  write32(loc, (read32(loc) & 0xff3ff000) | opcode | rot | (imm & 0xff));
+  write32(ctx, loc,
+          (read32(ctx, loc) & 0xff3ff000) | opcode | rot | (imm & 0xff));
 }
 
-static void encodeLdrGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
-                           int group) {
+static void encodeLdrGroup(Ctx &ctx, uint8_t *loc, const Relocation &rel,
+                           uint64_t val, int group) {
   // R_ARM_LDR_PC_Gn is S + A - P, we have ((S + A) | T) - P, if S is a
   // function then addr is 0 (modulo 2) and Pa is 0 (modulo 4) so we can clear
   // bottom bit to recover S + A - P.
@@ -575,12 +576,12 @@ static void encodeLdrGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
     val = -val;
   }
   uint32_t imm = getRemAndLZForGroup(group, val).first;
-  checkUInt(loc, imm, 12, rel);
-  write32(loc, (read32(loc) & 0xff7ff000) | opcode | imm);
+  checkUInt(ctx, loc, imm, 12, rel);
+  write32(ctx, loc, (read32(ctx, loc) & 0xff7ff000) | opcode | imm);
 }
 
-static void encodeLdrsGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
-                            int group) {
+static void encodeLdrsGroup(Ctx &ctx, uint8_t *loc, const Relocation &rel,
+                            uint64_t val, int group) {
   // R_ARM_LDRS_PC_Gn is S + A - P, we have ((S + A) | T) - P, if S is a
   // function then addr is 0 (modulo 2) and Pa is 0 (modulo 4) so we can clear
   // bottom bit to recover S + A - P.
@@ -593,9 +594,10 @@ static void encodeLdrsGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
     val = -val;
   }
   uint32_t imm = getRemAndLZForGroup(group, val).first;
-  checkUInt(loc, imm, 8, rel);
-  write32(loc, (read32(loc) & 0xff7ff0f0) | opcode | ((imm & 0xf0) << 4) |
-                     (imm & 0xf));
+  checkUInt(ctx, loc, imm, 8, rel);
+  write32(ctx, loc,
+          (read32(ctx, loc) & 0xff7ff0f0) | opcode | ((imm & 0xf0) << 4) |
+              (imm & 0xf));
 }
 
 void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
@@ -617,11 +619,11 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_ARM_TLS_LE32:
   case R_ARM_TLS_TPOFF32:
   case R_ARM_TLS_DTPOFF32:
-    write32(loc, val);
+    write32(ctx, loc, val);
     break;
   case R_ARM_PREL31:
-    checkInt(loc, val, 31, rel);
-    write32(loc, (read32(loc) & 0x80000000) | (val & ~0x80000000));
+    checkInt(ctx, loc, val, 31, rel);
+    write32(ctx, loc, (read32(ctx, loc) & 0x80000000) | (val & ~0x80000000));
     break;
   case R_ARM_CALL: {
     // R_ARM_CALL is used for BL and BLX instructions, for symbols of type
@@ -630,53 +632,55 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     // not of type STT_FUNC then we must preserve the original instruction.
     assert(rel.sym); // R_ARM_CALL is always reached via relocate().
     bool bit0Thumb = val & 1;
-    bool isBlx = (read32(loc) & 0xfe000000) == 0xfa000000;
+    bool isBlx = (read32(ctx, loc) & 0xfe000000) == 0xfa000000;
     // lld 10.0 and before always used bit0Thumb when deciding to write a BLX
     // even when type not STT_FUNC.
     if (!rel.sym->isFunc() && isBlx != bit0Thumb)
       stateChangeWarning(ctx, loc, rel.type, *rel.sym);
     if (rel.sym->isFunc() ? bit0Thumb : isBlx) {
       // The BLX encoding is 0xfa:H:imm24 where Val = imm24:H:'1'
-      checkInt(loc, val, 26, rel);
-      write32(loc, 0xfa000000 |                    // opcode
-                         ((val & 2) << 23) |         // H
-                         ((val >> 2) & 0x00ffffff)); // imm24
+      checkInt(ctx, loc, val, 26, rel);
+      write32(ctx, loc,
+              0xfa000000 |                    // opcode
+                  ((val & 2) << 23) |         // H
+                  ((val >> 2) & 0x00ffffff)); // imm24
       break;
     }
     // BLX (always unconditional) instruction to an ARM Target, select an
     // unconditional BL.
-    write32(loc, 0xeb000000 | (read32(loc) & 0x00ffffff));
+    write32(ctx, loc, 0xeb000000 | (read32(ctx, loc) & 0x00ffffff));
     // fall through as BL encoding is shared with B
   }
     [[fallthrough]];
   case R_ARM_JUMP24:
   case R_ARM_PC24:
   case R_ARM_PLT32:
-    checkInt(loc, val, 26, rel);
-    write32(loc, (read32(loc) & ~0x00ffffff) | ((val >> 2) & 0x00ffffff));
+    checkInt(ctx, loc, val, 26, rel);
+    write32(ctx, loc,
+            (read32(ctx, loc) & ~0x00ffffff) | ((val >> 2) & 0x00ffffff));
     break;
   case R_ARM_THM_JUMP8:
     // We do a 9 bit check because val is right-shifted by 1 bit.
-    checkInt(loc, val, 9, rel);
-    write16(loc, (read32(loc) & 0xff00) | ((val >> 1) & 0x00ff));
+    checkInt(ctx, loc, val, 9, rel);
+    write16(ctx, loc, (read32(ctx, loc) & 0xff00) | ((val >> 1) & 0x00ff));
     break;
   case R_ARM_THM_JUMP11:
     // We do a 12 bit check because val is right-shifted by 1 bit.
-    checkInt(loc, val, 12, rel);
-    write16(loc, (read32(loc) & 0xf800) | ((val >> 1) & 0x07ff));
+    checkInt(ctx, loc, val, 12, rel);
+    write16(ctx, loc, (read32(ctx, loc) & 0xf800) | ((val >> 1) & 0x07ff));
     break;
   case R_ARM_THM_JUMP19:
     // Encoding T3: Val = S:J2:J1:imm6:imm11:0
-    checkInt(loc, val, 21, rel);
-    write16(loc,
-              (read16(loc) & 0xfbc0) |   // opcode cond
-                  ((val >> 10) & 0x0400) | // S
-                  ((val >> 12) & 0x003f)); // imm6
-    write16(loc + 2,
-              0x8000 |                    // opcode
-                  ((val >> 8) & 0x0800) | // J2
-                  ((val >> 5) & 0x2000) | // J1
-                  ((val >> 1) & 0x07ff)); // imm11
+    checkInt(ctx, loc, val, 21, rel);
+    write16(ctx, loc,
+            (read16(ctx, loc) & 0xfbc0) | // opcode cond
+                ((val >> 10) & 0x0400) |  // S
+                ((val >> 12) & 0x003f));  // imm6
+    write16(ctx, loc + 2,
+            0x8000 |                    // opcode
+                ((val >> 8) & 0x0800) | // J2
+                ((val >> 5) & 0x2000) | // J1
+                ((val >> 1) & 0x07ff)); // imm11
     break;
   case R_ARM_THM_CALL: {
     // R_ARM_THM_CALL is used for BL and BLX instructions, for symbols of type
@@ -687,7 +691,7 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     assert(rel.sym); // R_ARM_THM_CALL is always reached via relocate().
     bool bit0Thumb = val & 1;
     bool useThumb = bit0Thumb || useThumbPLTs(ctx);
-    bool isBlx = (read16(loc + 2) & 0x1000) == 0;
+    bool isBlx = (read16(ctx, loc + 2) & 0x1000) == 0;
     // lld 10.0 and before always used bit0Thumb when deciding to write a BLX
     // even when type not STT_FUNC.
     if (!rel.sym->isFunc() && !rel.sym->isInPlt(ctx) && isBlx == useThumb)
@@ -697,21 +701,21 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
       // the BLX instruction may only be two byte aligned. This must be done
       // before overflow check.
       val = alignTo(val, 4);
-      write16(loc + 2, read16(loc + 2) & ~0x1000);
+      write16(ctx, loc + 2, read16(ctx, loc + 2) & ~0x1000);
     } else {
-      write16(loc + 2, (read16(loc + 2) & ~0x1000) | 1 << 12);
+      write16(ctx, loc + 2, (read16(ctx, loc + 2) & ~0x1000) | 1 << 12);
     }
     if (!ctx.arg.armJ1J2BranchEncoding) {
       // Older Arm architectures do not support R_ARM_THM_JUMP24 and have
       // different encoding rules and range due to J1 and J2 always being 1.
-      checkInt(loc, val, 23, rel);
-      write16(loc,
-                0xf000 |                     // opcode
-                    ((val >> 12) & 0x07ff)); // imm11
-      write16(loc + 2,
-                (read16(loc + 2) & 0xd000) | // opcode
-                    0x2800 |                   // J1 == J2 == 1
-                    ((val >> 1) & 0x07ff));    // imm11
+      checkInt(ctx, loc, val, 23, rel);
+      write16(ctx, loc,
+              0xf000 |                     // opcode
+                  ((val >> 12) & 0x07ff)); // imm11
+      write16(ctx, loc + 2,
+              (read16(ctx, loc + 2) & 0xd000) | // opcode
+                  0x2800 |                      // J1 == J2 == 1
+                  ((val >> 1) & 0x07ff));       // imm11
       break;
     }
   }
@@ -719,68 +723,70 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     [[fallthrough]];
   case R_ARM_THM_JUMP24:
     // Encoding B  T4, BL T1, BLX T2: Val = S:I1:I2:imm10:imm11:0
-    checkInt(loc, val, 25, rel);
-    write16(loc,
-              0xf000 |                     // opcode
-                  ((val >> 14) & 0x0400) | // S
-                  ((val >> 12) & 0x03ff)); // imm10
-    write16(loc + 2,
-              (read16(loc + 2) & 0xd000) |                  // opcode
-                  (((~(val >> 10)) ^ (val >> 11)) & 0x2000) | // J1
-                  (((~(val >> 11)) ^ (val >> 13)) & 0x0800) | // J2
-                  ((val >> 1) & 0x07ff));                     // imm11
+    checkInt(ctx, loc, val, 25, rel);
+    write16(ctx, loc,
+            0xf000 |                     // opcode
+                ((val >> 14) & 0x0400) | // S
+                ((val >> 12) & 0x03ff)); // imm10
+    write16(ctx, loc + 2,
+            (read16(ctx, loc + 2) & 0xd000) |               // opcode
+                (((~(val >> 10)) ^ (val >> 11)) & 0x2000) | // J1
+                (((~(val >> 11)) ^ (val >> 13)) & 0x0800) | // J2
+                ((val >> 1) & 0x07ff));                     // imm11
     break;
   case R_ARM_MOVW_ABS_NC:
   case R_ARM_MOVW_PREL_NC:
   case R_ARM_MOVW_BREL_NC:
-    write32(loc, (read32(loc) & ~0x000f0fff) | ((val & 0xf000) << 4) |
-                       (val & 0x0fff));
+    write32(ctx, loc,
+            (read32(ctx, loc) & ~0x000f0fff) | ((val & 0xf000) << 4) |
+                (val & 0x0fff));
     break;
   case R_ARM_MOVT_ABS:
   case R_ARM_MOVT_PREL:
   case R_ARM_MOVT_BREL:
-    write32(loc, (read32(loc) & ~0x000f0fff) |
-                       (((val >> 16) & 0xf000) << 4) | ((val >> 16) & 0xfff));
+    write32(ctx, loc,
+            (read32(ctx, loc) & ~0x000f0fff) | (((val >> 16) & 0xf000) << 4) |
+                ((val >> 16) & 0xfff));
     break;
   case R_ARM_THM_MOVT_ABS:
   case R_ARM_THM_MOVT_PREL:
   case R_ARM_THM_MOVT_BREL:
     // Encoding T1: A = imm4:i:imm3:imm8
 
-    write16(loc,
+    write16(ctx, loc,
             0xf2c0 |                     // opcode
                 ((val >> 17) & 0x0400) | // i
                 ((val >> 28) & 0x000f)); // imm4
 
-    write16(loc + 2,
-              (read16(loc + 2) & 0x8f00) | // opcode
-                  ((val >> 12) & 0x7000) |   // imm3
-                  ((val >> 16) & 0x00ff));   // imm8
+    write16(ctx, loc + 2,
+            (read16(ctx, loc + 2) & 0x8f00) | // opcode
+                ((val >> 12) & 0x7000) |      // imm3
+                ((val >> 16) & 0x00ff));      // imm8
     break;
   case R_ARM_THM_MOVW_ABS_NC:
   case R_ARM_THM_MOVW_PREL_NC:
   case R_ARM_THM_MOVW_BREL_NC:
     // Encoding T3: A = imm4:i:imm3:imm8
-    write16(loc,
-              0xf240 |                     // opcode
-                  ((val >> 1) & 0x0400) |  // i
-                  ((val >> 12) & 0x000f)); // imm4
-    write16(loc + 2,
-              (read16(loc + 2) & 0x8f00) | // opcode
-                  ((val << 4) & 0x7000) |    // imm3
-                  (val & 0x00ff));           // imm8
+    write16(ctx, loc,
+            0xf240 |                     // opcode
+                ((val >> 1) & 0x0400) |  // i
+                ((val >> 12) & 0x000f)); // imm4
+    write16(ctx, loc + 2,
+            (read16(ctx, loc + 2) & 0x8f00) | // opcode
+                ((val << 4) & 0x7000) |       // imm3
+                (val & 0x00ff));              // imm8
     break;
   case R_ARM_THM_ALU_ABS_G3:
-    write16(loc, (read16(loc) &~ 0x00ff) | ((val >> 24) & 0x00ff));
+    write16(ctx, loc, (read16(ctx, loc) & ~0x00ff) | ((val >> 24) & 0x00ff));
     break;
   case R_ARM_THM_ALU_ABS_G2_NC:
-    write16(loc, (read16(loc) &~ 0x00ff) | ((val >> 16) & 0x00ff));
+    write16(ctx, loc, (read16(ctx, loc) & ~0x00ff) | ((val >> 16) & 0x00ff));
     break;
   case R_ARM_THM_ALU_ABS_G1_NC:
-    write16(loc, (read16(loc) &~ 0x00ff) | ((val >> 8) & 0x00ff));
+    write16(ctx, loc, (read16(ctx, loc) & ~0x00ff) | ((val >> 8) & 0x00ff));
     break;
   case R_ARM_THM_ALU_ABS_G0_NC:
-    write16(loc, (read16(loc) &~ 0x00ff) | (val & 0x00ff));
+    write16(ctx, loc, (read16(ctx, loc) & ~0x00ff) | (val & 0x00ff));
     break;
   case R_ARM_ALU_PC_G0:
     encodeAluGroup(loc, rel, val, 0, true);
@@ -798,22 +804,22 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     encodeAluGroup(loc, rel, val, 2, true);
     break;
   case R_ARM_LDR_PC_G0:
-    encodeLdrGroup(loc, rel, val, 0);
+    encodeLdrGroup(ctx, loc, rel, val, 0);
     break;
   case R_ARM_LDR_PC_G1:
-    encodeLdrGroup(loc, rel, val, 1);
+    encodeLdrGroup(ctx, loc, rel, val, 1);
     break;
   case R_ARM_LDR_PC_G2:
-    encodeLdrGroup(loc, rel, val, 2);
+    encodeLdrGroup(ctx, loc, rel, val, 2);
     break;
   case R_ARM_LDRS_PC_G0:
-    encodeLdrsGroup(loc, rel, val, 0);
+    encodeLdrsGroup(ctx, loc, rel, val, 0);
     break;
   case R_ARM_LDRS_PC_G1:
-    encodeLdrsGroup(loc, rel, val, 1);
+    encodeLdrsGroup(ctx, loc, rel, val, 1);
     break;
   case R_ARM_LDRS_PC_G2:
-    encodeLdrsGroup(loc, rel, val, 2);
+    encodeLdrsGroup(ctx, loc, rel, val, 2);
     break;
   case R_ARM_THM_ALU_PREL_11_0: {
     // ADR encoding T2 (sub), T3 (add) i:imm3:imm8
@@ -823,10 +829,11 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
       imm = -imm;
       sub = 0x00a0;
     }
-    checkUInt(loc, imm, 12, rel);
-    write16(loc, (read16(loc) & 0xfb0f) | sub | (imm & 0x800) >> 1);
-    write16(loc + 2,
-              (read16(loc + 2) & 0x8f00) | (imm & 0x700) << 4 | (imm & 0xff));
+    checkUInt(ctx, loc, imm, 12, rel);
+    write16(ctx, loc, (read16(ctx, loc) & 0xfb0f) | sub | (imm & 0x800) >> 1);
+    write16(ctx, loc + 2,
+            (read16(ctx, loc + 2) & 0x8f00) | (imm & 0x700) << 4 |
+                (imm & 0xff));
     break;
   }
   case R_ARM_THM_PC8:
@@ -836,9 +843,9 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     // bottom bit to recover S + A - Pa.
     if (rel.sym->isFunc())
       val &= ~0x1;
-    checkUInt(loc, val, 10, rel);
-    checkAlignment(loc, val, 4, rel);
-    write16(loc, (read16(loc) & 0xff00) | (val & 0x3fc) >> 2);
+    checkUInt(ctx, loc, val, 10, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
+    write16(ctx, loc, (read16(ctx, loc) & 0xff00) | (val & 0x3fc) >> 2);
     break;
   case R_ARM_THM_PC12: {
     // LDR (literal) encoding T2, add = (U == '1') imm12
@@ -854,9 +861,9 @@ void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
       imm12 = -imm12;
       u = 0;
     }
-    checkUInt(loc, imm12, 12, rel);
-    write16(loc, read16(loc) | u);
-    write16(loc + 2, (read16(loc + 2) & 0xf000) | imm12);
+    checkUInt(ctx, loc, imm12, 12, rel);
+    write16(ctx, loc, read16(ctx, loc) | u);
+    write16(ctx, loc + 2, (read16(ctx, loc + 2) & 0xf000) | imm12);
     break;
   }
   default:
@@ -890,22 +897,22 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_ARM_TLS_LE32:
   case R_ARM_TLS_LDO32:
   case R_ARM_TLS_TPOFF32:
-    return SignExtend64<32>(read32(buf));
+    return SignExtend64<32>(read32(ctx, buf));
   case R_ARM_PREL31:
-    return SignExtend64<31>(read32(buf));
+    return SignExtend64<31>(read32(ctx, buf));
   case R_ARM_CALL:
   case R_ARM_JUMP24:
   case R_ARM_PC24:
   case R_ARM_PLT32:
-    return SignExtend64<26>(read32(buf) << 2);
+    return SignExtend64<26>(read32(ctx, buf) << 2);
   case R_ARM_THM_JUMP8:
-    return SignExtend64<9>(read16(buf) << 1);
+    return SignExtend64<9>(read16(ctx, buf) << 1);
   case R_ARM_THM_JUMP11:
-    return SignExtend64<12>(read16(buf) << 1);
+    return SignExtend64<12>(read16(ctx, buf) << 1);
   case R_ARM_THM_JUMP19: {
     // Encoding T3: A = S:J2:J1:imm10:imm6:0
-    uint16_t hi = read16(buf);
-    uint16_t lo = read16(buf + 2);
+    uint16_t hi = read16(ctx, buf);
+    uint16_t lo = read16(ctx, buf + 2);
     return SignExtend64<20>(((hi & 0x0400) << 10) | // S
                             ((lo & 0x0800) << 8) |  // J2
                             ((lo & 0x2000) << 5) |  // J1
@@ -916,8 +923,8 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
     if (!ctx.arg.armJ1J2BranchEncoding) {
       // Older Arm architectures do not support R_ARM_THM_JUMP24 and have
       // different encoding rules and range due to J1 and J2 always being 1.
-      uint16_t hi = read16(buf);
-      uint16_t lo = read16(buf + 2);
+      uint16_t hi = read16(ctx, buf);
+      uint16_t lo = read16(ctx, buf + 2);
       return SignExtend64<22>(((hi & 0x7ff) << 12) | // imm11
                               ((lo & 0x7ff) << 1));  // imm11:0
       break;
@@ -926,8 +933,8 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_ARM_THM_JUMP24: {
     // Encoding B T4, BL T1, BLX T2: A = S:I1:I2:imm10:imm11:0
     // I1 = NOT(J1 EOR S), I2 = NOT(J2 EOR S)
-    uint16_t hi = read16(buf);
-    uint16_t lo = read16(buf + 2);
+    uint16_t hi = read16(ctx, buf);
+    uint16_t lo = read16(ctx, buf + 2);
     return SignExtend64<24>(((hi & 0x0400) << 14) |                    // S
                             (~((lo ^ (hi << 3)) << 10) & 0x00800000) | // I1
                             (~((lo ^ (hi << 1)) << 11) & 0x00400000) | // I2
@@ -942,7 +949,7 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_ARM_MOVT_PREL:
   case R_ARM_MOVW_BREL_NC:
   case R_ARM_MOVT_BREL: {
-    uint64_t val = read32(buf) & 0x000f0fff;
+    uint64_t val = read32(ctx, buf) & 0x000f0fff;
     return SignExtend64<16>(((val & 0x000f0000) >> 4) | (val & 0x00fff));
   }
   case R_ARM_THM_MOVW_ABS_NC:
@@ -952,8 +959,8 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_ARM_THM_MOVW_BREL_NC:
   case R_ARM_THM_MOVT_BREL: {
     // Encoding T3: A = imm4:i:imm3:imm8
-    uint16_t hi = read16(buf);
-    uint16_t lo = read16(buf + 2);
+    uint16_t hi = read16(ctx, buf);
+    uint16_t lo = read16(ctx, buf + 2);
     return SignExtend64<16>(((hi & 0x000f) << 12) | // imm4
                             ((hi & 0x0400) << 1) |  // i
                             ((lo & 0x7000) >> 4) |  // imm3
@@ -963,7 +970,7 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_ARM_THM_ALU_ABS_G1_NC:
   case R_ARM_THM_ALU_ABS_G2_NC:
   case R_ARM_THM_ALU_ABS_G3:
-    return read16(buf) & 0xff;
+    return read16(ctx, buf) & 0xff;
   case R_ARM_ALU_PC_G0:
   case R_ARM_ALU_PC_G0_NC:
   case R_ARM_ALU_PC_G1:
@@ -973,7 +980,7 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
     // right rotation and 8-bit constant. After the rotation the value
     // is zero-extended. When bit 23 is set the instruction is an add, when
     // bit 22 is set it is a sub.
-    uint32_t instr = read32(buf);
+    uint32_t instr = read32(ctx, buf);
     uint32_t val = rotr32(instr & 0xff, ((instr & 0xf00) >> 8) * 2);
     return (instr & 0x00400000) ? -val : val;
   }
@@ -982,15 +989,15 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_ARM_LDR_PC_G2: {
     // ADR (literal) add = bit23, sub = bit22
     // LDR (literal) u = bit23 unsigned imm12
-    bool u = read32(buf) & 0x00800000;
-    uint32_t imm12 = read32(buf) & 0xfff;
+    bool u = read32(ctx, buf) & 0x00800000;
+    uint32_t imm12 = read32(ctx, buf) & 0xfff;
     return u ? imm12 : -imm12;
   }
   case R_ARM_LDRS_PC_G0:
   case R_ARM_LDRS_PC_G1:
   case R_ARM_LDRS_PC_G2: {
     // LDRD/LDRH/LDRSB/LDRSH (literal) u = bit23 unsigned imm8
-    uint32_t opcode = read32(buf);
+    uint32_t opcode = read32(ctx, buf);
     bool u = opcode & 0x00800000;
     uint32_t imm4l = opcode & 0xf;
     uint32_t imm4h = (opcode & 0xf00) >> 4;
@@ -1000,8 +1007,8 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
     // Thumb2 ADR, which is an alias for a sub or add instruction with an
     // unsigned immediate.
     // ADR encoding T2 (sub), T3 (add) i:imm3:imm8
-    uint16_t hi = read16(buf);
-    uint16_t lo = read16(buf + 2);
+    uint16_t hi = read16(ctx, buf);
+    uint16_t lo = read16(ctx, buf + 2);
     uint64_t imm = (hi & 0x0400) << 1 | // i
                    (lo & 0x7000) >> 4 | // imm3
                    (lo & 0x00ff);       // imm8
@@ -1013,11 +1020,11 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
     // From ELF for the ARM Architecture the initial signed addend is formed
     // from an unsigned field using expression (((imm8:00 + 4) & 0x3ff) – 4)
     // this trick permits the PC bias of -4 to be encoded using imm8 = 0xff
-    return ((((read16(buf) & 0xff) << 2) + 4) & 0x3ff) - 4;
+    return ((((read16(ctx, buf) & 0xff) << 2) + 4) & 0x3ff) - 4;
   case R_ARM_THM_PC12: {
     // LDR (literal) encoding T2, add = (U == '1') imm12
-    bool u = read16(buf) & 0x0080;
-    uint64_t imm12 = read16(buf + 2) & 0x0fff;
+    bool u = read16(ctx, buf) & 0x0080;
+    uint64_t imm12 = read16(ctx, buf + 2) & 0x0fff;
     return u ? imm12 : -imm12;
   }
   case R_ARM_NONE:
@@ -1089,11 +1096,11 @@ static void toLittleEndianInstructions(uint8_t *buf, uint64_t start,
   CodeState curState = static_cast<CodeState>(width);
   if (curState == CodeState::Arm)
     for (uint64_t i = start; i < end; i += width)
-      write32le(buf + i, read32(buf + i));
+      write32le(buf + i, read32be(buf + i));
 
   if (curState == CodeState::Thumb)
     for (uint64_t i = start; i < end; i += width)
-      write16le(buf + i, read16(buf + i));
+      write16le(buf + i, read16be(buf + i));
 }
 
 // Arm BE8 big endian format requires instructions to be little endian, with
@@ -1103,10 +1110,11 @@ static void toLittleEndianInstructions(uint8_t *buf, uint64_t start,
 // [$t, non $t) and convert these to little endian a word or half word at a
 // time respectively.
 void elf::convertArmInstructionstoBE8(InputSection *sec, uint8_t *buf) {
-  if (!sectionMap.contains(sec))
+  auto it = sectionMap.find(sec);
+  if (it == sectionMap.end())
     return;
 
-  SmallVector<const Defined *, 0> &mapSyms = sectionMap[sec];
+  SmallVector<const Defined *, 0> &mapSyms = it->second;
 
   if (mapSyms.empty())
     return;
@@ -1386,10 +1394,10 @@ void ArmCmseSGSection::addSGVeneer(Symbol *acleSeSym, Symbol *sym) {
 void ArmCmseSGSection::writeTo(uint8_t *buf) {
   for (ArmCmseSGVeneer *s : sgVeneers) {
     uint8_t *p = buf + s->offset;
-    write16(p + 0, 0xe97f); // SG
-    write16(p + 2, 0xe97f);
-    write16(p + 4, 0xf000); // B.W S
-    write16(p + 6, 0xb000);
+    write16(ctx, p + 0, 0xe97f); // SG
+    write16(ctx, p + 2, 0xe97f);
+    write16(ctx, p + 4, 0xf000); // B.W S
+    write16(ctx, p + 6, 0xb000);
     ctx.target->relocateNoSym(p + 4, R_ARM_THM_JUMP24,
                               s->acleSeSym->getVA() -
                                   (getVA() + s->offset + s->size));
@@ -1397,7 +1405,7 @@ void ArmCmseSGSection::writeTo(uint8_t *buf) {
 }
 
 void ArmCmseSGSection::addMappingSymbol() {
-  addSyntheticLocal("$t", STT_NOTYPE, /*off=*/0, /*size=*/0, *this);
+  addSyntheticLocal(ctx, "$t", STT_NOTYPE, /*off=*/0, /*size=*/0, *this);
 }
 
 size_t ArmCmseSGSection::getSize() const {
@@ -1452,9 +1460,11 @@ template <typename ELFT> void elf::writeARMCmseImportLib(Ctx &ctx) {
       make<SymbolTableSection<ELFT>>(ctx, *strtab);
 
   SmallVector<std::pair<OutputSection *, SyntheticSection *>, 0> osIsPairs;
-  osIsPairs.emplace_back(make<OutputSection>(strtab->name, 0, 0), strtab);
-  osIsPairs.emplace_back(make<OutputSection>(impSymTab->name, 0, 0), impSymTab);
-  osIsPairs.emplace_back(make<OutputSection>(shstrtab->name, 0, 0), shstrtab);
+  osIsPairs.emplace_back(make<OutputSection>(ctx, strtab->name, 0, 0), strtab);
+  osIsPairs.emplace_back(make<OutputSection>(ctx, impSymTab->name, 0, 0),
+                         impSymTab);
+  osIsPairs.emplace_back(make<OutputSection>(ctx, shstrtab->name, 0, 0),
+                         shstrtab);
 
   std::sort(ctx.symtab->cmseSymMap.begin(), ctx.symtab->cmseSymMap.end(),
             [](const auto &a, const auto &b) -> bool {
@@ -1464,7 +1474,7 @@ template <typename ELFT> void elf::writeARMCmseImportLib(Ctx &ctx) {
   for (auto &p : ctx.symtab->cmseSymMap) {
     Defined *d = cast<Defined>(p.second.sym);
     impSymTab->addSymbol(makeDefined(
-        ctx.internalFile, d->getName(), d->computeBinding(),
+        ctx.internalFile, d->getName(), d->computeBinding(ctx),
         /*stOther=*/0, STT_FUNC, d->getVA(), d->getSize(), nullptr));
   }
 
@@ -1473,7 +1483,7 @@ template <typename ELFT> void elf::writeARMCmseImportLib(Ctx &ctx) {
   for (auto &[osec, isec] : osIsPairs) {
     osec->sectionIndex = ++idx;
     osec->recordSection(isec);
-    osec->finalizeInputSections(ctx);
+    osec->finalizeInputSections();
     osec->shName = shstrtab->addString(osec->name);
     osec->size = isec->getSize();
     isec->finalizeContents();
diff --git a/lld/ELF/Arch/AVR.cpp b/lld/ELF/Arch/AVR.cpp
index 53c10b89e5db7e2bf5b83c5cf5f5a038b1120674..4dc605c47059c1ad53f2e44ec169e28cb053d866 100644
--- a/lld/ELF/Arch/AVR.cpp
+++ b/lld/ELF/Arch/AVR.cpp
@@ -119,19 +119,19 @@ bool AVR::needsThunk(RelExpr expr, RelType type, const InputFile *file,
 void AVR::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   switch (rel.type) {
   case R_AVR_8:
-    checkUInt(loc, val, 8, rel);
+    checkUInt(ctx, loc, val, 8, rel);
     *loc = val;
     break;
   case R_AVR_8_LO8:
-    checkUInt(loc, val, 32, rel);
+    checkUInt(ctx, loc, val, 32, rel);
     *loc = val & 0xff;
     break;
   case R_AVR_8_HI8:
-    checkUInt(loc, val, 32, rel);
+    checkUInt(ctx, loc, val, 32, rel);
     *loc = (val >> 8) & 0xff;
     break;
   case R_AVR_8_HLO8:
-    checkUInt(loc, val, 32, rel);
+    checkUInt(ctx, loc, val, 32, rel);
     *loc = (val >> 16) & 0xff;
     break;
   case R_AVR_16:
@@ -141,17 +141,17 @@ void AVR::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     write16le(loc, val & 0xffff);
     break;
   case R_AVR_16_PM:
-    checkAlignment(loc, val, 2, rel);
-    checkUInt(loc, val >> 1, 16, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
+    checkUInt(ctx, loc, val >> 1, 16, rel);
     write16le(loc, val >> 1);
     break;
   case R_AVR_32:
-    checkUInt(loc, val, 32, rel);
+    checkUInt(ctx, loc, val, 32, rel);
     write32le(loc, val);
     break;
 
   case R_AVR_LDI:
-    checkUInt(loc, val, 8, rel);
+    checkUInt(ctx, loc, val, 8, rel);
     writeLDI(loc, val & 0xff);
     break;
 
@@ -181,39 +181,39 @@ void AVR::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     break;
 
   case R_AVR_LO8_LDI_GS:
-    checkUInt(loc, val, 17, rel);
+    checkUInt(ctx, loc, val, 17, rel);
     [[fallthrough]];
   case R_AVR_LO8_LDI_PM:
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     writeLDI(loc, (val >> 1) & 0xff);
     break;
   case R_AVR_HI8_LDI_GS:
-    checkUInt(loc, val, 17, rel);
+    checkUInt(ctx, loc, val, 17, rel);
     [[fallthrough]];
   case R_AVR_HI8_LDI_PM:
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     writeLDI(loc, (val >> 9) & 0xff);
     break;
   case R_AVR_HH8_LDI_PM:
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     writeLDI(loc, (val >> 17) & 0xff);
     break;
 
   case R_AVR_LO8_LDI_PM_NEG:
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     writeLDI(loc, (-val >> 1) & 0xff);
     break;
   case R_AVR_HI8_LDI_PM_NEG:
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     writeLDI(loc, (-val >> 9) & 0xff);
     break;
   case R_AVR_HH8_LDI_PM_NEG:
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     writeLDI(loc, (-val >> 17) & 0xff);
     break;
 
   case R_AVR_LDS_STS_16: {
-    checkUInt(loc, val, 7, rel);
+    checkUInt(ctx, loc, val, 7, rel);
     const uint16_t hi = val >> 4;
     const uint16_t lo = val & 0xf;
     write16le(loc, (read16le(loc) & 0xf8f0) | ((hi << 8) | lo));
@@ -221,41 +221,41 @@ void AVR::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   }
 
   case R_AVR_PORT5:
-    checkUInt(loc, val, 5, rel);
+    checkUInt(ctx, loc, val, 5, rel);
     write16le(loc, (read16le(loc) & 0xff07) | (val << 3));
     break;
   case R_AVR_PORT6:
-    checkUInt(loc, val, 6, rel);
+    checkUInt(ctx, loc, val, 6, rel);
     write16le(loc, (read16le(loc) & 0xf9f0) | (val & 0x30) << 5 | (val & 0x0f));
     break;
 
   // Since every jump destination is word aligned we gain an extra bit
   case R_AVR_7_PCREL: {
-    checkInt(loc, val - 2, 8, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val - 2, 8, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     const uint16_t target = (val - 2) >> 1;
     write16le(loc, (read16le(loc) & 0xfc07) | ((target & 0x7f) << 3));
     break;
   }
   case R_AVR_13_PCREL: {
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     const uint16_t target = (val - 2) >> 1;
     write16le(loc, (read16le(loc) & 0xf000) | (target & 0xfff));
     break;
   }
 
   case R_AVR_6:
-    checkInt(loc, val, 6, rel);
+    checkInt(ctx, loc, val, 6, rel);
     write16le(loc, (read16le(loc) & 0xd3f8) | (val & 0x20) << 8 |
                        (val & 0x18) << 7 | (val & 0x07));
     break;
   case R_AVR_6_ADIW:
-    checkInt(loc, val, 6, rel);
+    checkInt(ctx, loc, val, 6, rel);
     write16le(loc, (read16le(loc) & 0xff30) | (val & 0x30) << 2 | (val & 0x0F));
     break;
 
   case R_AVR_CALL: {
-    checkAlignment(loc, val, 2, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     uint16_t hi = val >> 17;
     uint16_t lo = val >> 1;
     write16le(loc, read16le(loc) | ((hi >> 1) << 4) | (hi & 1));
diff --git a/lld/ELF/Arch/Hexagon.cpp b/lld/ELF/Arch/Hexagon.cpp
index 5d2477cc800651a731034452729823c77be65a11..80fcb3b747d1eaae42a97b73d4315bf4959490e2 100644
--- a/lld/ELF/Arch/Hexagon.cpp
+++ b/lld/ELF/Arch/Hexagon.cpp
@@ -181,11 +181,13 @@ static const InstructionMask r6[] = {
     {0xd7000000, 0x006020e0}, {0xd8000000, 0x006020e0},
     {0xdb000000, 0x006020e0}, {0xdf000000, 0x006020e0}};
 
+constexpr uint32_t instParsePacketEnd = 0x0000c000;
+
 static bool isDuplex(uint32_t insn) {
   // Duplex forms have a fixed mask and parse bits 15:14 are always
   // zero.  Non-duplex insns will always have at least one bit set in the
   // parse field.
-  return (0xC000 & insn) == 0;
+  return (instParsePacketEnd & insn) == 0;
 }
 
 static uint32_t findMaskR6(uint32_t insn) {
@@ -216,6 +218,12 @@ static uint32_t findMaskR11(uint32_t insn) {
 }
 
 static uint32_t findMaskR16(uint32_t insn) {
+  if (isDuplex(insn))
+    return 0x03f00000;
+
+  // Clear the end-packet-parse bits:
+  insn = insn & ~instParsePacketEnd;
+
   if ((0xff000000 & insn) == 0x48000000)
     return 0x061f20ff;
   if ((0xff000000 & insn) == 0x49000000)
@@ -225,8 +233,14 @@ static uint32_t findMaskR16(uint32_t insn) {
   if ((0xff000000 & insn) == 0xb0000000)
     return 0x0fe03fe0;
 
-  if (isDuplex(insn))
-    return 0x03f00000;
+  if ((0xff802000 & insn) == 0x74000000)
+    return 0x00001fe0;
+  if ((0xff802000 & insn) == 0x74002000)
+    return 0x00001fe0;
+  if ((0xff802000 & insn) == 0x74800000)
+    return 0x00001fe0;
+  if ((0xff802000 & insn) == 0x74802000)
+    return 0x00001fe0;
 
   for (InstructionMask i : r6)
     if ((0xff000000 & insn) == i.cmpMask)
@@ -295,18 +309,18 @@ void Hexagon::relocate(uint8_t *loc, const Relocation &rel,
     or32le(loc, applyMask(0x0fff3fff, val >> 6));
     break;
   case R_HEX_B9_PCREL:
-    checkInt(loc, val, 11, rel);
+    checkInt(ctx, loc, val, 11, rel);
     or32le(loc, applyMask(0x003000fe, val >> 2));
     break;
   case R_HEX_B9_PCREL_X:
     or32le(loc, applyMask(0x003000fe, val & 0x3f));
     break;
   case R_HEX_B13_PCREL:
-    checkInt(loc, val, 15, rel);
+    checkInt(ctx, loc, val, 15, rel);
     or32le(loc, applyMask(0x00202ffe, val >> 2));
     break;
   case R_HEX_B15_PCREL:
-    checkInt(loc, val, 17, rel);
+    checkInt(ctx, loc, val, 17, rel);
     or32le(loc, applyMask(0x00df20fe, val >> 2));
     break;
   case R_HEX_B15_PCREL_X:
@@ -315,7 +329,7 @@ void Hexagon::relocate(uint8_t *loc, const Relocation &rel,
   case R_HEX_B22_PCREL:
   case R_HEX_GD_PLT_B22_PCREL:
   case R_HEX_PLT_B22_PCREL:
-    checkInt(loc, val, 22, rel);
+    checkInt(ctx, loc, val, 22, rel);
     or32le(loc, applyMask(0x1ff3ffe, val >> 2));
     break;
   case R_HEX_B22_PCREL_X:
@@ -396,7 +410,7 @@ int64_t Hexagon::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_HEX_DTPMOD_32:
   case R_HEX_DTPREL_32:
   case R_HEX_TPREL_32:
-    return SignExtend64<32>(read32(buf));
+    return SignExtend64<32>(read32(ctx, buf));
   default:
     internalLinkerError(getErrorLoc(ctx, buf),
                         "cannot read addend for relocation " + toString(type));
diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index f16f8f0c8d5ce4a26e9e278814b41e85b6e996a4..5923cda2298b4eecf1d2d44856cb679f0542f292 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -552,7 +552,7 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
                          uint64_t val) const {
   switch (rel.type) {
   case R_LARCH_32_PCREL:
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     [[fallthrough]];
   case R_LARCH_32:
   case R_LARCH_TLS_DTPREL32:
@@ -569,26 +569,26 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
   case R_LARCH_TLS_LD_PCREL20_S2:
   case R_LARCH_TLS_GD_PCREL20_S2:
   case R_LARCH_TLS_DESC_PCREL20_S2:
-    checkInt(loc, val, 22, rel);
-    checkAlignment(loc, val, 4, rel);
+    checkInt(ctx, loc, val, 22, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
     write32le(loc, setJ20(read32le(loc), val >> 2));
     return;
 
   case R_LARCH_B16:
-    checkInt(loc, val, 18, rel);
-    checkAlignment(loc, val, 4, rel);
+    checkInt(ctx, loc, val, 18, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
     write32le(loc, setK16(read32le(loc), val >> 2));
     return;
 
   case R_LARCH_B21:
-    checkInt(loc, val, 23, rel);
-    checkAlignment(loc, val, 4, rel);
+    checkInt(ctx, loc, val, 23, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
     write32le(loc, setD5k16(read32le(loc), val >> 2));
     return;
 
   case R_LARCH_B26:
-    checkInt(loc, val, 28, rel);
-    checkAlignment(loc, val, 4, rel);
+    checkInt(ctx, loc, val, 28, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
     write32le(loc, setD10k16(read32le(loc), val >> 2));
     return;
 
@@ -600,7 +600,7 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
     if (((int64_t)val + 0x20000) != llvm::SignExtend64(val + 0x20000, 38))
       reportRangeError(ctx, loc, rel, Twine(val), llvm::minIntN(38) - 0x20000,
                        llvm::maxIntN(38) - 0x20000);
-    checkAlignment(loc, val, 4, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
     // Since jirl performs sign extension on the offset immediate, adds (1<<17)
     // to original val to get the correct hi20.
     uint32_t hi20 = extractBits(val + (1 << 17), 37, 18);
@@ -620,7 +620,7 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
     // In this case, process like an R_LARCH_B16, but without overflow checking
     // and only taking the value's lowest 12 bits.
     if (isJirl(read32le(loc))) {
-      checkAlignment(loc, val, 4, rel);
+      checkAlignment(ctx, loc, val, 4, rel);
       val = SignExtend64<12>(val);
       write32le(loc, setK16(read32le(loc), val >> 2));
       return;
@@ -744,7 +744,7 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
   }
 }
 
-static bool relax(InputSection &sec) {
+static bool relax(Ctx &ctx, InputSection &sec) {
   const uint64_t secAddr = sec.getVA();
   const MutableArrayRef<Relocation> relocs = sec.relocs();
   auto &aux = *sec.relaxAux;
@@ -833,7 +833,7 @@ bool LoongArch::relaxOnce(int pass) const {
     if (!(osec->flags & SHF_EXECINSTR))
       continue;
     for (InputSection *sec : getInputSections(*osec, storage))
-      changed |= relax(*sec);
+      changed |= relax(ctx, *sec);
   }
   return changed;
 }
diff --git a/lld/ELF/Arch/MSP430.cpp b/lld/ELF/Arch/MSP430.cpp
index b3aab52e179c976b72a179830536b8c8bf3292da..5d48518c53d8df9dbcd440d9955225c8ffb6abae 100644
--- a/lld/ELF/Arch/MSP430.cpp
+++ b/lld/ELF/Arch/MSP430.cpp
@@ -62,23 +62,23 @@ RelExpr MSP430::getRelExpr(RelType type, const Symbol &s,
 void MSP430::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   switch (rel.type) {
   case R_MSP430_8:
-    checkIntUInt(loc, val, 8, rel);
+    checkIntUInt(ctx, loc, val, 8, rel);
     *loc = val;
     break;
   case R_MSP430_16:
   case R_MSP430_16_PCREL:
   case R_MSP430_16_BYTE:
   case R_MSP430_16_PCREL_BYTE:
-    checkIntUInt(loc, val, 16, rel);
+    checkIntUInt(ctx, loc, val, 16, rel);
     write16le(loc, val);
     break;
   case R_MSP430_32:
-    checkIntUInt(loc, val, 32, rel);
+    checkIntUInt(ctx, loc, val, 32, rel);
     write32le(loc, val);
     break;
   case R_MSP430_10_PCREL: {
     int16_t offset = ((int16_t)val >> 1) - 1;
-    checkInt(loc, offset, 10, rel);
+    checkInt(ctx, loc, offset, 10, rel);
     write16le(loc, (read16le(loc) & 0xFC00) | (offset & 0x3FF));
     break;
   }
diff --git a/lld/ELF/Arch/Mips.cpp b/lld/ELF/Arch/Mips.cpp
index 6313ac8ca4fb9a999e3b5b6d0df040740abffd3e..1d3000793ca2689f434578c2509a7da26f086a12 100644
--- a/lld/ELF/Arch/Mips.cpp
+++ b/lld/ELF/Arch/Mips.cpp
@@ -52,7 +52,7 @@ template <class ELFT> MIPS<ELFT>::MIPS(Ctx &ctx) : TargetInfo(ctx) {
   needsThunks = true;
 
   // Set `sigrie 1` as a trap instruction.
-  write32(trapInstr.data(), 0x04170001);
+  write32(ctx, trapInstr.data(), 0x04170001);
 
   if (ELFT::Is64Bits) {
     relativeRel = (R_MIPS_64 << 8) | R_MIPS_REL32;
@@ -208,51 +208,52 @@ void MIPS<ELFT>::writeGotPlt(uint8_t *buf, const Symbol &) const {
   uint64_t va = ctx.in.plt->getVA();
   if (isMicroMips(ctx))
     va |= 1;
-  write32(buf, va);
+  write32(ctx, buf, va);
 }
 
-template <endianness E> static uint32_t readShuffle(const uint8_t *loc) {
+template <endianness E>
+static uint32_t readShuffle(Ctx &ctx, const uint8_t *loc) {
   // The major opcode of a microMIPS instruction needs to appear
   // in the first 16-bit word (lowest address) for efficient hardware
   // decode so that it knows if the instruction is 16-bit or 32-bit
   // as early as possible. To do so, little-endian binaries keep 16-bit
   // words in a big-endian order. That is why we have to swap these
   // words to get a correct value.
-  uint32_t v = read32(loc);
+  uint32_t v = read32(ctx, loc);
   if (E == llvm::endianness::little)
     return (v << 16) | (v >> 16);
   return v;
 }
 
-static void writeValue(uint8_t *loc, uint64_t v, uint8_t bitsSize,
+static void writeValue(Ctx &ctx, uint8_t *loc, uint64_t v, uint8_t bitsSize,
                        uint8_t shift) {
-  uint32_t instr = read32(loc);
+  uint32_t instr = read32(ctx, loc);
   uint32_t mask = 0xffffffff >> (32 - bitsSize);
   uint32_t data = (instr & ~mask) | ((v >> shift) & mask);
-  write32(loc, data);
+  write32(ctx, loc, data);
 }
 
 template <endianness E>
-static void writeShuffleValue(uint8_t *loc, uint64_t v, uint8_t bitsSize,
-                              uint8_t shift) {
+static void writeShuffle(Ctx &ctx, uint8_t *loc, uint64_t v, uint8_t bitsSize,
+                         uint8_t shift) {
   // See comments in readShuffle for purpose of this code.
   uint16_t *words = (uint16_t *)loc;
   if (E == llvm::endianness::little)
     std::swap(words[0], words[1]);
 
-  writeValue(loc, v, bitsSize, shift);
+  writeValue(ctx, loc, v, bitsSize, shift);
 
   if (E == llvm::endianness::little)
     std::swap(words[0], words[1]);
 }
 
 template <endianness E>
-static void writeMicroRelocation16(uint8_t *loc, uint64_t v, uint8_t bitsSize,
-                                   uint8_t shift) {
-  uint16_t instr = read16(loc);
+static void writeMicroRelocation16(Ctx &ctx, uint8_t *loc, uint64_t v,
+                                   uint8_t bitsSize, uint8_t shift) {
+  uint16_t instr = read16(ctx, loc);
   uint16_t mask = 0xffff >> (16 - bitsSize);
   uint16_t data = (instr & ~mask) | ((v >> shift) & mask);
-  write16(loc, data);
+  write16(ctx, loc, data);
 }
 
 template <class ELFT> void MIPS<ELFT>::writePltHeader(uint8_t *buf) const {
@@ -262,58 +263,59 @@ template <class ELFT> void MIPS<ELFT>::writePltHeader(uint8_t *buf) const {
     // Overwrite trap instructions written by Writer::writeTrapInstr.
     memset(buf, 0, pltHeaderSize);
 
-    write16(buf, isMipsR6(ctx) ? 0x7860 : 0x7980); // addiupc v1, (GOTPLT) - .
-    write16(buf + 4, 0xff23);    // lw      $25, 0($3)
-    write16(buf + 8, 0x0535);    // subu16  $2,  $2, $3
-    write16(buf + 10, 0x2525);   // srl16   $2,  $2, 2
-    write16(buf + 12, 0x3302);   // addiu   $24, $2, -2
-    write16(buf + 14, 0xfffe);
-    write16(buf + 16, 0x0dff);   // move    $15, $31
+    write16(ctx, buf,
+            isMipsR6(ctx) ? 0x7860 : 0x7980); // addiupc v1, (GOTPLT) - .
+    write16(ctx, buf + 4, 0xff23);            // lw      $25, 0($3)
+    write16(ctx, buf + 8, 0x0535);            // subu16  $2,  $2, $3
+    write16(ctx, buf + 10, 0x2525);           // srl16   $2,  $2, 2
+    write16(ctx, buf + 12, 0x3302);           // addiu   $24, $2, -2
+    write16(ctx, buf + 14, 0xfffe);
+    write16(ctx, buf + 16, 0x0dff); // move    $15, $31
     if (isMipsR6(ctx)) {
-      write16(buf + 18, 0x0f83); // move    $28, $3
-      write16(buf + 20, 0x472b); // jalrc   $25
-      write16(buf + 22, 0x0c00); // nop
+      write16(ctx, buf + 18, 0x0f83); // move    $28, $3
+      write16(ctx, buf + 20, 0x472b); // jalrc   $25
+      write16(ctx, buf + 22, 0x0c00); // nop
       relocateNoSym(buf, R_MICROMIPS_PC19_S2, gotPlt - plt);
     } else {
-      write16(buf + 18, 0x45f9); // jalrc   $25
-      write16(buf + 20, 0x0f83); // move    $28, $3
-      write16(buf + 22, 0x0c00); // nop
+      write16(ctx, buf + 18, 0x45f9); // jalrc   $25
+      write16(ctx, buf + 20, 0x0f83); // move    $28, $3
+      write16(ctx, buf + 22, 0x0c00); // nop
       relocateNoSym(buf, R_MICROMIPS_PC23_S2, gotPlt - plt);
     }
     return;
   }
 
   if (ctx.arg.mipsN32Abi) {
-    write32(buf, 0x3c0e0000);      // lui   $14, %hi(&GOTPLT[0])
-    write32(buf + 4, 0x8dd90000);  // lw    $25, %lo(&GOTPLT[0])($14)
-    write32(buf + 8, 0x25ce0000);  // addiu $14, $14, %lo(&GOTPLT[0])
-    write32(buf + 12, 0x030ec023); // subu  $24, $24, $14
-    write32(buf + 16, 0x03e07825); // move  $15, $31
-    write32(buf + 20, 0x0018c082); // srl   $24, $24, 2
+    write32(ctx, buf, 0x3c0e0000);      // lui   $14, %hi(&GOTPLT[0])
+    write32(ctx, buf + 4, 0x8dd90000);  // lw    $25, %lo(&GOTPLT[0])($14)
+    write32(ctx, buf + 8, 0x25ce0000);  // addiu $14, $14, %lo(&GOTPLT[0])
+    write32(ctx, buf + 12, 0x030ec023); // subu  $24, $24, $14
+    write32(ctx, buf + 16, 0x03e07825); // move  $15, $31
+    write32(ctx, buf + 20, 0x0018c082); // srl   $24, $24, 2
   } else if (ELFT::Is64Bits) {
-    write32(buf, 0x3c0e0000);      // lui   $14, %hi(&GOTPLT[0])
-    write32(buf + 4, 0xddd90000);  // ld    $25, %lo(&GOTPLT[0])($14)
-    write32(buf + 8, 0x25ce0000);  // addiu $14, $14, %lo(&GOTPLT[0])
-    write32(buf + 12, 0x030ec023); // subu  $24, $24, $14
-    write32(buf + 16, 0x03e07825); // move  $15, $31
-    write32(buf + 20, 0x0018c0c2); // srl   $24, $24, 3
+    write32(ctx, buf, 0x3c0e0000);      // lui   $14, %hi(&GOTPLT[0])
+    write32(ctx, buf + 4, 0xddd90000);  // ld    $25, %lo(&GOTPLT[0])($14)
+    write32(ctx, buf + 8, 0x25ce0000);  // addiu $14, $14, %lo(&GOTPLT[0])
+    write32(ctx, buf + 12, 0x030ec023); // subu  $24, $24, $14
+    write32(ctx, buf + 16, 0x03e07825); // move  $15, $31
+    write32(ctx, buf + 20, 0x0018c0c2); // srl   $24, $24, 3
   } else {
-    write32(buf, 0x3c1c0000);      // lui   $28, %hi(&GOTPLT[0])
-    write32(buf + 4, 0x8f990000);  // lw    $25, %lo(&GOTPLT[0])($28)
-    write32(buf + 8, 0x279c0000);  // addiu $28, $28, %lo(&GOTPLT[0])
-    write32(buf + 12, 0x031cc023); // subu  $24, $24, $28
-    write32(buf + 16, 0x03e07825); // move  $15, $31
-    write32(buf + 20, 0x0018c082); // srl   $24, $24, 2
+    write32(ctx, buf, 0x3c1c0000);      // lui   $28, %hi(&GOTPLT[0])
+    write32(ctx, buf + 4, 0x8f990000);  // lw    $25, %lo(&GOTPLT[0])($28)
+    write32(ctx, buf + 8, 0x279c0000);  // addiu $28, $28, %lo(&GOTPLT[0])
+    write32(ctx, buf + 12, 0x031cc023); // subu  $24, $24, $28
+    write32(ctx, buf + 16, 0x03e07825); // move  $15, $31
+    write32(ctx, buf + 20, 0x0018c082); // srl   $24, $24, 2
   }
 
   uint32_t jalrInst = ctx.arg.zHazardplt ? 0x0320fc09 : 0x0320f809;
-  write32(buf + 24, jalrInst); // jalr.hb $25 or jalr $25
-  write32(buf + 28, 0x2718fffe); // subu  $24, $24, 2
+  write32(ctx, buf + 24, jalrInst);   // jalr.hb $25 or jalr $25
+  write32(ctx, buf + 28, 0x2718fffe); // subu  $24, $24, 2
 
   uint64_t gotPlt = ctx.in.gotPlt->getVA();
-  writeValue(buf, gotPlt + 0x8000, 16, 16);
-  writeValue(buf + 4, gotPlt, 16, 0);
-  writeValue(buf + 8, gotPlt, 16, 0);
+  writeValue(ctx, buf, gotPlt + 0x8000, 16, 16);
+  writeValue(ctx, buf + 4, gotPlt, 16, 0);
+  writeValue(ctx, buf + 8, gotPlt, 16, 0);
 }
 
 template <class ELFT>
@@ -325,16 +327,16 @@ void MIPS<ELFT>::writePlt(uint8_t *buf, const Symbol &sym,
     memset(buf, 0, pltEntrySize);
 
     if (isMipsR6(ctx)) {
-      write16(buf, 0x7840);      // addiupc $2, (GOTPLT) - .
-      write16(buf + 4, 0xff22);  // lw $25, 0($2)
-      write16(buf + 8, 0x0f02);  // move $24, $2
-      write16(buf + 10, 0x4723); // jrc $25 / jr16 $25
+      write16(ctx, buf, 0x7840);      // addiupc $2, (GOTPLT) - .
+      write16(ctx, buf + 4, 0xff22);  // lw $25, 0($2)
+      write16(ctx, buf + 8, 0x0f02);  // move $24, $2
+      write16(ctx, buf + 10, 0x4723); // jrc $25 / jr16 $25
       relocateNoSym(buf, R_MICROMIPS_PC19_S2, gotPltEntryAddr - pltEntryAddr);
     } else {
-      write16(buf, 0x7900);      // addiupc $2, (GOTPLT) - .
-      write16(buf + 4, 0xff22);  // lw $25, 0($2)
-      write16(buf + 8, 0x4599);  // jrc $25 / jr16 $25
-      write16(buf + 10, 0x0f02); // move $24, $2
+      write16(ctx, buf, 0x7900);      // addiupc $2, (GOTPLT) - .
+      write16(ctx, buf + 4, 0xff22);  // lw $25, 0($2)
+      write16(ctx, buf + 8, 0x4599);  // jrc $25 / jr16 $25
+      write16(ctx, buf + 10, 0x0f02); // move $24, $2
       relocateNoSym(buf, R_MICROMIPS_PC23_S2, gotPltEntryAddr - pltEntryAddr);
     }
     return;
@@ -346,13 +348,13 @@ void MIPS<ELFT>::writePlt(uint8_t *buf, const Symbol &sym,
                         : (ctx.arg.zHazardplt ? 0x03200408 : 0x03200008);
   uint32_t addInst = ELFT::Is64Bits ? 0x65f80000 : 0x25f80000;
 
-  write32(buf, 0x3c0f0000);     // lui   $15, %hi(.got.plt entry)
-  write32(buf + 4, loadInst);   // l[wd] $25, %lo(.got.plt entry)($15)
-  write32(buf + 8, jrInst);     // jr  $25 / jr.hb $25
-  write32(buf + 12, addInst);   // [d]addiu $24, $15, %lo(.got.plt entry)
-  writeValue(buf, gotPltEntryAddr + 0x8000, 16, 16);
-  writeValue(buf + 4, gotPltEntryAddr, 16, 0);
-  writeValue(buf + 12, gotPltEntryAddr, 16, 0);
+  write32(ctx, buf, 0x3c0f0000);   // lui   $15, %hi(.got.plt entry)
+  write32(ctx, buf + 4, loadInst); // l[wd] $25, %lo(.got.plt entry)($15)
+  write32(ctx, buf + 8, jrInst);   // jr  $25 / jr.hb $25
+  write32(ctx, buf + 12, addInst); // [d]addiu $24, $15, %lo(.got.plt entry)
+  writeValue(ctx, buf, gotPltEntryAddr + 0x8000, 16, 16);
+  writeValue(ctx, buf + 4, gotPltEntryAddr, 16, 0);
+  writeValue(ctx, buf + 12, gotPltEntryAddr, 16, 0);
 }
 
 template <class ELFT>
@@ -389,18 +391,18 @@ int64_t MIPS<ELFT>::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_MIPS_TLS_DTPREL32:
   case R_MIPS_TLS_DTPMOD32:
   case R_MIPS_TLS_TPREL32:
-    return SignExtend64<32>(read32(buf));
+    return SignExtend64<32>(read32(ctx, buf));
   case R_MIPS_26:
     // FIXME (simon): If the relocation target symbol is not a PLT entry
     // we should use another expression for calculation:
     // ((A << 2) | (P & 0xf0000000)) >> 2
-    return SignExtend64<28>(read32(buf) << 2);
+    return SignExtend64<28>(read32(ctx, buf) << 2);
   case R_MIPS_CALL_HI16:
   case R_MIPS_GOT16:
   case R_MIPS_GOT_HI16:
   case R_MIPS_HI16:
   case R_MIPS_PCHI16:
-    return SignExtend64<16>(read32(buf)) << 16;
+    return SignExtend64<16>(read32(ctx, buf)) << 16;
   case R_MIPS_CALL16:
   case R_MIPS_CALL_LO16:
   case R_MIPS_GOT_LO16:
@@ -414,10 +416,10 @@ int64_t MIPS<ELFT>::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_MIPS_TLS_LDM:
   case R_MIPS_TLS_TPREL_HI16:
   case R_MIPS_TLS_TPREL_LO16:
-    return SignExtend64<16>(read32(buf));
+    return SignExtend64<16>(read32(ctx, buf));
   case R_MICROMIPS_GOT16:
   case R_MICROMIPS_HI16:
-    return SignExtend64<16>(readShuffle<e>(buf)) << 16;
+    return SignExtend64<16>(readShuffle<e>(ctx, buf)) << 16;
   case R_MICROMIPS_CALL16:
   case R_MICROMIPS_GPREL16:
   case R_MICROMIPS_LO16:
@@ -428,45 +430,45 @@ int64_t MIPS<ELFT>::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_MICROMIPS_TLS_LDM:
   case R_MICROMIPS_TLS_TPREL_HI16:
   case R_MICROMIPS_TLS_TPREL_LO16:
-    return SignExtend64<16>(readShuffle<e>(buf));
+    return SignExtend64<16>(readShuffle<e>(ctx, buf));
   case R_MICROMIPS_GPREL7_S2:
-    return SignExtend64<9>(readShuffle<e>(buf) << 2);
+    return SignExtend64<9>(readShuffle<e>(ctx, buf) << 2);
   case R_MIPS_PC16:
-    return SignExtend64<18>(read32(buf) << 2);
+    return SignExtend64<18>(read32(ctx, buf) << 2);
   case R_MIPS_PC19_S2:
-    return SignExtend64<21>(read32(buf) << 2);
+    return SignExtend64<21>(read32(ctx, buf) << 2);
   case R_MIPS_PC21_S2:
-    return SignExtend64<23>(read32(buf) << 2);
+    return SignExtend64<23>(read32(ctx, buf) << 2);
   case R_MIPS_PC26_S2:
-    return SignExtend64<28>(read32(buf) << 2);
+    return SignExtend64<28>(read32(ctx, buf) << 2);
   case R_MIPS_PC32:
-    return SignExtend64<32>(read32(buf));
+    return SignExtend64<32>(read32(ctx, buf));
   case R_MICROMIPS_26_S1:
-    return SignExtend64<27>(readShuffle<e>(buf) << 1);
+    return SignExtend64<27>(readShuffle<e>(ctx, buf) << 1);
   case R_MICROMIPS_PC7_S1:
-    return SignExtend64<8>(read16(buf) << 1);
+    return SignExtend64<8>(read16(ctx, buf) << 1);
   case R_MICROMIPS_PC10_S1:
-    return SignExtend64<11>(read16(buf) << 1);
+    return SignExtend64<11>(read16(ctx, buf) << 1);
   case R_MICROMIPS_PC16_S1:
-    return SignExtend64<17>(readShuffle<e>(buf) << 1);
+    return SignExtend64<17>(readShuffle<e>(ctx, buf) << 1);
   case R_MICROMIPS_PC18_S3:
-    return SignExtend64<21>(readShuffle<e>(buf) << 3);
+    return SignExtend64<21>(readShuffle<e>(ctx, buf) << 3);
   case R_MICROMIPS_PC19_S2:
-    return SignExtend64<21>(readShuffle<e>(buf) << 2);
+    return SignExtend64<21>(readShuffle<e>(ctx, buf) << 2);
   case R_MICROMIPS_PC21_S1:
-    return SignExtend64<22>(readShuffle<e>(buf) << 1);
+    return SignExtend64<22>(readShuffle<e>(ctx, buf) << 1);
   case R_MICROMIPS_PC23_S2:
-    return SignExtend64<25>(readShuffle<e>(buf) << 2);
+    return SignExtend64<25>(readShuffle<e>(ctx, buf) << 2);
   case R_MICROMIPS_PC26_S1:
-    return SignExtend64<27>(readShuffle<e>(buf) << 1);
+    return SignExtend64<27>(readShuffle<e>(ctx, buf) << 1);
   case R_MIPS_64:
   case R_MIPS_TLS_DTPMOD64:
   case R_MIPS_TLS_DTPREL64:
   case R_MIPS_TLS_TPREL64:
   case (R_MIPS_64 << 8) | R_MIPS_REL32:
-    return read64(buf);
+    return read64(ctx, buf);
   case R_MIPS_COPY:
-    return ctx.arg.is64 ? read64(buf) : read32(buf);
+    return ctx.arg.is64 ? read64(ctx, buf) : read32(ctx, buf);
   case R_MIPS_NONE:
   case R_MIPS_JUMP_SLOT:
   case R_MIPS_JALR:
@@ -517,7 +519,8 @@ static bool isMicroBranchReloc(RelType type) {
 }
 
 template <class ELFT>
-static uint64_t fixupCrossModeJump(uint8_t *loc, RelType type, uint64_t val) {
+static uint64_t fixupCrossModeJump(Ctx &ctx, uint8_t *loc, RelType type,
+                                   uint64_t val) {
   // Here we need to detect jump/branch from regular MIPS code
   // to a microMIPS target and vice versa. In that cases jump
   // instructions need to be replaced by their "cross-mode"
@@ -531,18 +534,18 @@ static uint64_t fixupCrossModeJump(uint8_t *loc, RelType type, uint64_t val) {
 
   switch (type) {
   case R_MIPS_26: {
-    uint32_t inst = read32(loc) >> 26;
+    uint32_t inst = read32(ctx, loc) >> 26;
     if (inst == 0x3 || inst == 0x1d) { // JAL or JALX
-      writeValue(loc, 0x1d << 26, 32, 0);
+      writeValue(ctx, loc, 0x1d << 26, 32, 0);
       return val;
     }
     break;
   }
   case R_MICROMIPS_26_S1: {
-    uint32_t inst = readShuffle<e>(loc) >> 26;
+    uint32_t inst = readShuffle<e>(ctx, loc) >> 26;
     if (inst == 0x3d || inst == 0x3c) { // JAL32 or JALX32
       val >>= 1;
-      writeShuffleValue<e>(loc, 0x3c << 26, 32, 0);
+      writeShuffle<e>(ctx, loc, 0x3c << 26, 32, 0);
       return val;
     }
     break;
@@ -575,7 +578,7 @@ void MIPS<ELFT>::relocate(uint8_t *loc, const Relocation &rel,
     std::tie(type, val) = calculateMipsRelChain(ctx, loc, type, val);
 
   // Detect cross-mode jump/branch and fix instruction.
-  val = fixupCrossModeJump<ELFT>(loc, type, val);
+  val = fixupCrossModeJump<ELFT>(ctx, loc, type, val);
 
   // Thread pointer and DRP offsets from the start of TLS data area.
   // https://www.linux-mips.org/wiki/NPTL
@@ -591,33 +594,33 @@ void MIPS<ELFT>::relocate(uint8_t *loc, const Relocation &rel,
   case R_MIPS_GPREL32:
   case R_MIPS_TLS_DTPREL32:
   case R_MIPS_TLS_TPREL32:
-    write32(loc, val);
+    write32(ctx, loc, val);
     break;
   case R_MIPS_64:
   case R_MIPS_TLS_DTPREL64:
   case R_MIPS_TLS_TPREL64:
-    write64(loc, val);
+    write64(ctx, loc, val);
     break;
   case R_MIPS_26:
-    writeValue(loc, val, 26, 2);
+    writeValue(ctx, loc, val, 26, 2);
     break;
   case R_MIPS_GOT16:
     // The R_MIPS_GOT16 relocation's value in "relocatable" linking mode
     // is updated addend (not a GOT index). In that case write high 16 bits
     // to store a correct addend value.
     if (ctx.arg.relocatable) {
-      writeValue(loc, val + 0x8000, 16, 16);
+      writeValue(ctx, loc, val + 0x8000, 16, 16);
     } else {
-      checkInt(loc, val, 16, rel);
-      writeValue(loc, val, 16, 0);
+      checkInt(ctx, loc, val, 16, rel);
+      writeValue(ctx, loc, val, 16, 0);
     }
     break;
   case R_MICROMIPS_GOT16:
     if (ctx.arg.relocatable) {
-      writeShuffleValue<e>(loc, val + 0x8000, 16, 16);
+      writeShuffle<e>(ctx, loc, val + 0x8000, 16, 16);
     } else {
-      checkInt(loc, val, 16, rel);
-      writeShuffleValue<e>(loc, val, 16, 0);
+      checkInt(ctx, loc, val, 16, rel);
+      writeShuffle<e>(ctx, loc, val, 16, 0);
     }
     break;
   case R_MIPS_CALL16:
@@ -627,7 +630,7 @@ void MIPS<ELFT>::relocate(uint8_t *loc, const Relocation &rel,
   case R_MIPS_TLS_GD:
   case R_MIPS_TLS_GOTTPREL:
   case R_MIPS_TLS_LDM:
-    checkInt(loc, val, 16, rel);
+    checkInt(ctx, loc, val, 16, rel);
     [[fallthrough]];
   case R_MIPS_CALL_LO16:
   case R_MIPS_GOT_LO16:
@@ -636,13 +639,13 @@ void MIPS<ELFT>::relocate(uint8_t *loc, const Relocation &rel,
   case R_MIPS_PCLO16:
   case R_MIPS_TLS_DTPREL_LO16:
   case R_MIPS_TLS_TPREL_LO16:
-    writeValue(loc, val, 16, 0);
+    writeValue(ctx, loc, val, 16, 0);
     break;
   case R_MICROMIPS_GPREL16:
   case R_MICROMIPS_TLS_GD:
   case R_MICROMIPS_TLS_LDM:
-    checkInt(loc, val, 16, rel);
-    writeShuffleValue<e>(loc, val, 16, 0);
+    checkInt(ctx, loc, val, 16, rel);
+    writeShuffle<e>(ctx, loc, val, 16, 0);
     break;
   case R_MICROMIPS_CALL16:
   case R_MICROMIPS_CALL_LO16:
@@ -650,11 +653,11 @@ void MIPS<ELFT>::relocate(uint8_t *loc, const Relocation &rel,
   case R_MICROMIPS_TLS_DTPREL_LO16:
   case R_MICROMIPS_TLS_GOTTPREL:
   case R_MICROMIPS_TLS_TPREL_LO16:
-    writeShuffleValue<e>(loc, val, 16, 0);
+    writeShuffle<e>(ctx, loc, val, 16, 0);
     break;
   case R_MICROMIPS_GPREL7_S2:
-    checkInt(loc, val, 7, rel);
-    writeShuffleValue<e>(loc, val, 7, 2);
+    checkInt(ctx, loc, val, 7, rel);
+    writeShuffle<e>(ctx, loc, val, 7, 2);
     break;
   case R_MIPS_CALL_HI16:
   case R_MIPS_GOT_HI16:
@@ -662,32 +665,32 @@ void MIPS<ELFT>::relocate(uint8_t *loc, const Relocation &rel,
   case R_MIPS_PCHI16:
   case R_MIPS_TLS_DTPREL_HI16:
   case R_MIPS_TLS_TPREL_HI16:
-    writeValue(loc, val + 0x8000, 16, 16);
+    writeValue(ctx, loc, val + 0x8000, 16, 16);
     break;
   case R_MICROMIPS_CALL_HI16:
   case R_MICROMIPS_GOT_HI16:
   case R_MICROMIPS_HI16:
   case R_MICROMIPS_TLS_DTPREL_HI16:
   case R_MICROMIPS_TLS_TPREL_HI16:
-    writeShuffleValue<e>(loc, val + 0x8000, 16, 16);
+    writeShuffle<e>(ctx, loc, val + 0x8000, 16, 16);
     break;
   case R_MIPS_HIGHER:
-    writeValue(loc, val + 0x80008000, 16, 32);
+    writeValue(ctx, loc, val + 0x80008000, 16, 32);
     break;
   case R_MIPS_HIGHEST:
-    writeValue(loc, val + 0x800080008000, 16, 48);
+    writeValue(ctx, loc, val + 0x800080008000, 16, 48);
     break;
   case R_MIPS_JALR:
     val -= 4;
     // Replace jalr/jr instructions by bal/b if the target
     // offset fits into the 18-bit range.
     if (isInt<18>(val)) {
-      switch (read32(loc)) {
+      switch (read32(ctx, loc)) {
       case 0x0320f809:  // jalr $25 => bal sym
-        write32(loc, 0x04110000 | ((val >> 2) & 0xffff));
+        write32(ctx, loc, 0x04110000 | ((val >> 2) & 0xffff));
         break;
       case 0x03200008:  // jr $25 => b sym
-        write32(loc, 0x10000000 | ((val >> 2) & 0xffff));
+        write32(ctx, loc, 0x10000000 | ((val >> 2) & 0xffff));
         break;
       }
     }
@@ -696,60 +699,60 @@ void MIPS<ELFT>::relocate(uint8_t *loc, const Relocation &rel,
     // Ignore this optimization relocation for now
     break;
   case R_MIPS_PC16:
-    checkAlignment(loc, val, 4, rel);
-    checkInt(loc, val, 18, rel);
-    writeValue(loc, val, 16, 2);
+    checkAlignment(ctx, loc, val, 4, rel);
+    checkInt(ctx, loc, val, 18, rel);
+    writeValue(ctx, loc, val, 16, 2);
     break;
   case R_MIPS_PC19_S2:
-    checkAlignment(loc, val, 4, rel);
-    checkInt(loc, val, 21, rel);
-    writeValue(loc, val, 19, 2);
+    checkAlignment(ctx, loc, val, 4, rel);
+    checkInt(ctx, loc, val, 21, rel);
+    writeValue(ctx, loc, val, 19, 2);
     break;
   case R_MIPS_PC21_S2:
-    checkAlignment(loc, val, 4, rel);
-    checkInt(loc, val, 23, rel);
-    writeValue(loc, val, 21, 2);
+    checkAlignment(ctx, loc, val, 4, rel);
+    checkInt(ctx, loc, val, 23, rel);
+    writeValue(ctx, loc, val, 21, 2);
     break;
   case R_MIPS_PC26_S2:
-    checkAlignment(loc, val, 4, rel);
-    checkInt(loc, val, 28, rel);
-    writeValue(loc, val, 26, 2);
+    checkAlignment(ctx, loc, val, 4, rel);
+    checkInt(ctx, loc, val, 28, rel);
+    writeValue(ctx, loc, val, 26, 2);
     break;
   case R_MIPS_PC32:
-    writeValue(loc, val, 32, 0);
+    writeValue(ctx, loc, val, 32, 0);
     break;
   case R_MICROMIPS_26_S1:
   case R_MICROMIPS_PC26_S1:
-    checkInt(loc, val, 27, rel);
-    writeShuffleValue<e>(loc, val, 26, 1);
+    checkInt(ctx, loc, val, 27, rel);
+    writeShuffle<e>(ctx, loc, val, 26, 1);
     break;
   case R_MICROMIPS_PC7_S1:
-    checkInt(loc, val, 8, rel);
-    writeMicroRelocation16<e>(loc, val, 7, 1);
+    checkInt(ctx, loc, val, 8, rel);
+    writeMicroRelocation16<e>(ctx, loc, val, 7, 1);
     break;
   case R_MICROMIPS_PC10_S1:
-    checkInt(loc, val, 11, rel);
-    writeMicroRelocation16<e>(loc, val, 10, 1);
+    checkInt(ctx, loc, val, 11, rel);
+    writeMicroRelocation16<e>(ctx, loc, val, 10, 1);
     break;
   case R_MICROMIPS_PC16_S1:
-    checkInt(loc, val, 17, rel);
-    writeShuffleValue<e>(loc, val, 16, 1);
+    checkInt(ctx, loc, val, 17, rel);
+    writeShuffle<e>(ctx, loc, val, 16, 1);
     break;
   case R_MICROMIPS_PC18_S3:
-    checkInt(loc, val, 21, rel);
-    writeShuffleValue<e>(loc, val, 18, 3);
+    checkInt(ctx, loc, val, 21, rel);
+    writeShuffle<e>(ctx, loc, val, 18, 3);
     break;
   case R_MICROMIPS_PC19_S2:
-    checkInt(loc, val, 21, rel);
-    writeShuffleValue<e>(loc, val, 19, 2);
+    checkInt(ctx, loc, val, 21, rel);
+    writeShuffle<e>(ctx, loc, val, 19, 2);
     break;
   case R_MICROMIPS_PC21_S1:
-    checkInt(loc, val, 22, rel);
-    writeShuffleValue<e>(loc, val, 21, 1);
+    checkInt(ctx, loc, val, 22, rel);
+    writeShuffle<e>(ctx, loc, val, 21, 1);
     break;
   case R_MICROMIPS_PC23_S2:
-    checkInt(loc, val, 25, rel);
-    writeShuffleValue<e>(loc, val, 23, 2);
+    checkInt(ctx, loc, val, 25, rel);
+    writeShuffle<e>(ctx, loc, val, 23, 2);
     break;
   default:
     llvm_unreachable("unknown relocation");
diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp
index fa1b2a6c39d494479fce9d544f2306904e28e612..3af4101fff606f643c5b34a83837b4787af4f0ec 100644
--- a/lld/ELF/Arch/PPC.cpp
+++ b/lld/ELF/Arch/PPC.cpp
@@ -65,11 +65,11 @@ static uint16_t lo(uint32_t v) { return v; }
 static uint16_t ha(uint32_t v) { return (v + 0x8000) >> 16; }
 
 static uint32_t readFromHalf16(Ctx &ctx, const uint8_t *loc) {
-  return read32(ctx.arg.isLE ? loc : loc - 2);
+  return read32(ctx, ctx.arg.isLE ? loc : loc - 2);
 }
 
 static void writeFromHalf16(Ctx &ctx, uint8_t *loc, uint32_t insn) {
-  write32(ctx.arg.isLE ? loc : loc - 2, insn);
+  write32(ctx, ctx.arg.isLE ? loc : loc - 2, insn);
 }
 
 void elf::writePPC32GlinkSection(Ctx &ctx, uint8_t *buf, size_t numEntries) {
@@ -96,7 +96,7 @@ void elf::writePPC32GlinkSection(Ctx &ctx, uint8_t *buf, size_t numEntries) {
 
   // Write N `b PLTresolve` first.
   for (size_t i = 0; i != numEntries; ++i)
-    write32(buf + 4 * i, 0x48000000 | 4 * (numEntries - i));
+    write32(ctx, buf + 4 * i, 0x48000000 | 4 * (numEntries - i));
   buf += 4 * numEntries;
 
   // Then write PLTresolve(), which has two forms: PIC and non-PIC. PLTresolve()
@@ -107,49 +107,53 @@ void elf::writePPC32GlinkSection(Ctx &ctx, uint8_t *buf, size_t numEntries) {
   if (ctx.arg.isPic) {
     uint32_t afterBcl = 4 * ctx.in.plt->getNumEntries() + 12;
     uint32_t gotBcl = got + 4 - (glink + afterBcl);
-    write32(buf + 0, 0x3d6b0000 | ha(afterBcl));  // addis r11,r11,1f-glink@ha
-    write32(buf + 4, 0x7c0802a6);                 // mflr r0
-    write32(buf + 8, 0x429f0005);                 // bcl 20,30,.+4
-    write32(buf + 12, 0x396b0000 | lo(afterBcl)); // 1: addi r11,r11,1b-glink@l
-    write32(buf + 16, 0x7d8802a6);                // mflr r12
-    write32(buf + 20, 0x7c0803a6);                // mtlr r0
-    write32(buf + 24, 0x7d6c5850);                // sub r11,r11,r12
-    write32(buf + 28, 0x3d8c0000 | ha(gotBcl));   // addis 12,12,GOT+4-1b@ha
+    write32(ctx, buf + 0,
+            0x3d6b0000 | ha(afterBcl)); // addis r11,r11,1f-glink@ha
+    write32(ctx, buf + 4, 0x7c0802a6);  // mflr r0
+    write32(ctx, buf + 8, 0x429f0005);  // bcl 20,30,.+4
+    write32(ctx, buf + 12,
+            0x396b0000 | lo(afterBcl)); // 1: addi r11,r11,1b-glink@l
+    write32(ctx, buf + 16, 0x7d8802a6); // mflr r12
+    write32(ctx, buf + 20, 0x7c0803a6); // mtlr r0
+    write32(ctx, buf + 24, 0x7d6c5850); // sub r11,r11,r12
+    write32(ctx, buf + 28, 0x3d8c0000 | ha(gotBcl)); // addis 12,12,GOT+4-1b@ha
     if (ha(gotBcl) == ha(gotBcl + 4)) {
-      write32(buf + 32, 0x800c0000 | lo(gotBcl)); // lwz r0,r12,GOT+4-1b@l(r12)
-      write32(buf + 36,
-              0x818c0000 | lo(gotBcl + 4));       // lwz r12,r12,GOT+8-1b@l(r12)
+      write32(ctx, buf + 32,
+              0x800c0000 | lo(gotBcl)); // lwz r0,r12,GOT+4-1b@l(r12)
+      write32(ctx, buf + 36,
+              0x818c0000 | lo(gotBcl + 4)); // lwz r12,r12,GOT+8-1b@l(r12)
     } else {
-      write32(buf + 32, 0x840c0000 | lo(gotBcl)); // lwzu r0,r12,GOT+4-1b@l(r12)
-      write32(buf + 36, 0x818c0000 | 4);          // lwz r12,r12,4(r12)
+      write32(ctx, buf + 32,
+              0x840c0000 | lo(gotBcl));       // lwzu r0,r12,GOT+4-1b@l(r12)
+      write32(ctx, buf + 36, 0x818c0000 | 4); // lwz r12,r12,4(r12)
     }
-    write32(buf + 40, 0x7c0903a6);                // mtctr 0
-    write32(buf + 44, 0x7c0b5a14);                // add r0,11,11
-    write32(buf + 48, 0x7d605a14);                // add r11,0,11
-    write32(buf + 52, 0x4e800420);                // bctr
+    write32(ctx, buf + 40, 0x7c0903a6); // mtctr 0
+    write32(ctx, buf + 44, 0x7c0b5a14); // add r0,11,11
+    write32(ctx, buf + 48, 0x7d605a14); // add r11,0,11
+    write32(ctx, buf + 52, 0x4e800420); // bctr
     buf += 56;
   } else {
-    write32(buf + 0, 0x3d800000 | ha(got + 4));   // lis     r12,GOT+4@ha
-    write32(buf + 4, 0x3d6b0000 | ha(-glink));    // addis   r11,r11,-glink@ha
+    write32(ctx, buf + 0, 0x3d800000 | ha(got + 4)); // lis     r12,GOT+4@ha
+    write32(ctx, buf + 4, 0x3d6b0000 | ha(-glink)); // addis   r11,r11,-glink@ha
     if (ha(got + 4) == ha(got + 8))
-      write32(buf + 8, 0x800c0000 | lo(got + 4)); // lwz r0,GOT+4@l(r12)
+      write32(ctx, buf + 8, 0x800c0000 | lo(got + 4)); // lwz r0,GOT+4@l(r12)
     else
-      write32(buf + 8, 0x840c0000 | lo(got + 4)); // lwzu r0,GOT+4@l(r12)
-    write32(buf + 12, 0x396b0000 | lo(-glink));   // addi    r11,r11,-glink@l
-    write32(buf + 16, 0x7c0903a6);                // mtctr   r0
-    write32(buf + 20, 0x7c0b5a14);                // add     r0,r11,r11
+      write32(ctx, buf + 8, 0x840c0000 | lo(got + 4)); // lwzu r0,GOT+4@l(r12)
+    write32(ctx, buf + 12, 0x396b0000 | lo(-glink)); // addi    r11,r11,-glink@l
+    write32(ctx, buf + 16, 0x7c0903a6);              // mtctr   r0
+    write32(ctx, buf + 20, 0x7c0b5a14);              // add     r0,r11,r11
     if (ha(got + 4) == ha(got + 8))
-      write32(buf + 24, 0x818c0000 | lo(got + 8)); // lwz r12,GOT+8@l(r12)
+      write32(ctx, buf + 24, 0x818c0000 | lo(got + 8)); // lwz r12,GOT+8@l(r12)
     else
-      write32(buf + 24, 0x818c0000 | 4);          // lwz r12,4(r12)
-    write32(buf + 28, 0x7d605a14);                // add     r11,r0,r11
-    write32(buf + 32, 0x4e800420);                // bctr
+      write32(ctx, buf + 24, 0x818c0000 | 4); // lwz r12,4(r12)
+    write32(ctx, buf + 28, 0x7d605a14);       // add     r11,r0,r11
+    write32(ctx, buf + 32, 0x4e800420);       // bctr
     buf += 36;
   }
 
   // Pad with nop. They should not be executed.
   for (; buf < end; buf += 4)
-    write32(buf, 0x60000000);
+    write32(ctx, buf, 0x60000000);
 }
 
 PPC::PPC(Ctx &ctx) : TargetInfo(ctx) {
@@ -174,7 +178,7 @@ PPC::PPC(Ctx &ctx) : TargetInfo(ctx) {
   defaultMaxPageSize = 65536;
   defaultImageBase = 0x10000000;
 
-  write32(trapInstr.data(), 0x7fe00008);
+  write32(ctx, trapInstr.data(), 0x7fe00008);
 }
 
 void PPC::writeIplt(uint8_t *buf, const Symbol &sym,
@@ -188,12 +192,12 @@ void PPC::writeGotHeader(uint8_t *buf) const {
   // _GLOBAL_OFFSET_TABLE_[0] = _DYNAMIC
   // glibc stores _dl_runtime_resolve in _GLOBAL_OFFSET_TABLE_[1],
   // link_map in _GLOBAL_OFFSET_TABLE_[2].
-  write32(buf, ctx.mainPart->dynamic->getVA());
+  write32(ctx, buf, ctx.mainPart->dynamic->getVA());
 }
 
 void PPC::writeGotPlt(uint8_t *buf, const Symbol &s) const {
   // Address of the symbol resolver stub in .glink .
-  write32(buf,
+  write32(ctx, buf,
           ctx.in.plt->getVA() + ctx.in.plt->headerSize + 4 * s.getPltIdx(ctx));
 }
 
@@ -290,7 +294,7 @@ int64_t PPC::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_PPC_DTPMOD32:
   case R_PPC_DTPREL32:
   case R_PPC_TPREL32:
-    return SignExtend64<32>(read32(buf));
+    return SignExtend64<32>(read32(ctx, buf));
   default:
     internalLinkerError(getErrorLoc(ctx, buf),
                         "cannot read addend for relocation " + toString(type));
@@ -321,16 +325,16 @@ void PPC::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   std::tie(newType, val) = fromDTPREL(rel.type, val);
   switch (newType) {
   case R_PPC_ADDR16:
-    checkIntUInt(loc, val, 16, rel);
-    write16(loc, val);
+    checkIntUInt(ctx, loc, val, 16, rel);
+    write16(ctx, loc, val);
     break;
   case R_PPC_GOT16:
   case R_PPC_GOT_TLSGD16:
   case R_PPC_GOT_TLSLD16:
   case R_PPC_GOT_TPREL16:
   case R_PPC_TPREL16:
-    checkInt(loc, val, 16, rel);
-    write16(loc, val);
+    checkInt(ctx, loc, val, 16, rel);
+    write16(ctx, loc, val);
     break;
   case R_PPC_ADDR16_HA:
   case R_PPC_DTPREL16_HA:
@@ -339,7 +343,7 @@ void PPC::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_PPC_GOT_TPREL16_HA:
   case R_PPC_REL16_HA:
   case R_PPC_TPREL16_HA:
-    write16(loc, ha(val));
+    write16(ctx, loc, ha(val));
     break;
   case R_PPC_ADDR16_HI:
   case R_PPC_DTPREL16_HI:
@@ -348,7 +352,7 @@ void PPC::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_PPC_GOT_TPREL16_HI:
   case R_PPC_REL16_HI:
   case R_PPC_TPREL16_HI:
-    write16(loc, val >> 16);
+    write16(ctx, loc, val >> 16);
     break;
   case R_PPC_ADDR16_LO:
   case R_PPC_DTPREL16_LO:
@@ -357,17 +361,17 @@ void PPC::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_PPC_GOT_TPREL16_LO:
   case R_PPC_REL16_LO:
   case R_PPC_TPREL16_LO:
-    write16(loc, val);
+    write16(ctx, loc, val);
     break;
   case R_PPC_ADDR32:
   case R_PPC_REL32:
-    write32(loc, val);
+    write32(ctx, loc, val);
     break;
   case R_PPC_REL14: {
     uint32_t mask = 0x0000FFFC;
-    checkInt(loc, val, 16, rel);
-    checkAlignment(loc, val, 4, rel);
-    write32(loc, (read32(loc) & ~mask) | (val & mask));
+    checkInt(ctx, loc, val, 16, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
+    write32(ctx, loc, (read32(ctx, loc) & ~mask) | (val & mask));
     break;
   }
   case R_PPC_ADDR24:
@@ -375,9 +379,9 @@ void PPC::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_PPC_LOCAL24PC:
   case R_PPC_PLTREL24: {
     uint32_t mask = 0x03FFFFFC;
-    checkInt(loc, val, 26, rel);
-    checkAlignment(loc, val, 4, rel);
-    write32(loc, (read32(loc) & ~mask) | (val & mask));
+    checkInt(ctx, loc, val, 26, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
+    write32(ctx, loc, (read32(ctx, loc) & ~mask) | (val & mask));
     break;
   }
   default:
@@ -419,7 +423,7 @@ void PPC::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
   }
   case R_PPC_TLSGD:
     // bl __tls_get_addr(x@tldgd) --> add r3, r3, r2
-    write32(loc, 0x7c631214);
+    write32(ctx, loc, 0x7c631214);
     break;
   default:
     llvm_unreachable("unsupported relocation for TLS GD to IE relaxation");
@@ -435,7 +439,7 @@ void PPC::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
     break;
   case R_PPC_TLSGD:
     // bl __tls_get_addr(x@tldgd) --> add r3, r3, x@tprel@l
-    write32(loc, 0x38630000 | lo(val));
+    write32(ctx, loc, 0x38630000 | lo(val));
     break;
   default:
     llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
@@ -453,7 +457,7 @@ void PPC::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel,
     // r3+x@dtprel computes r3+x-0x8000, while we want it to compute r3+x@tprel
     // = r3+x-0x7000, so add 4096 to r3.
     // bl __tls_get_addr(x@tlsld) --> addi r3, r3, 4096
-    write32(loc, 0x38631000);
+    write32(ctx, loc, 0x38631000);
     break;
   case R_PPC_DTPREL16:
   case R_PPC_DTPREL16_HA:
@@ -476,18 +480,18 @@ void PPC::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
     break;
   }
   case R_PPC_TLS: {
-    uint32_t insn = read32(loc);
+    uint32_t insn = read32(ctx, loc);
     if (insn >> 26 != 31)
       error("unrecognized instruction for IE to LE R_PPC_TLS");
     // addi rT, rT, x@tls --> addi rT, rT, x@tprel@l
-    unsigned secondaryOp = (read32(loc) & 0x000007fe) >> 1;
+    unsigned secondaryOp = (read32(ctx, loc) & 0x000007fe) >> 1;
     uint32_t dFormOp = getPPCDFormOp(secondaryOp);
     if (dFormOp == 0) { // Expecting a DS-Form instruction.
       dFormOp = getPPCDSFormOp(secondaryOp);
       if (dFormOp == 0)
         error("unrecognized instruction for IE to LE R_PPC_TLS");
     }
-    write32(loc, (dFormOp | (insn & 0x03ff0000) | lo(val)));
+    write32(ctx, loc, (dFormOp | (insn & 0x03ff0000) | lo(val)));
     break;
   }
   default:
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index 056d495ff1f7dbb8226bf45ef14554fb234bdbd6..d937492fe440d7301866d9caf4c51c79bdf2c3f9 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -246,7 +246,7 @@ unsigned elf::getPPC64GlobalEntryToLocalEntryOffset(uint8_t stOther) {
 
 void elf::writePrefixedInst(Ctx &ctx, uint8_t *loc, uint64_t insn) {
   insn = ctx.arg.isLE ? insn << 32 | insn >> 32 : insn;
-  write64(loc, insn);
+  write64(ctx, loc, insn);
 }
 
 static bool addOptional(Ctx &ctx, StringRef name, uint64_t value,
@@ -254,9 +254,9 @@ static bool addOptional(Ctx &ctx, StringRef name, uint64_t value,
   Symbol *sym = ctx.symtab->find(name);
   if (!sym || sym->isDefined())
     return false;
-  sym->resolve(Defined{ctx.internalFile, StringRef(), STB_GLOBAL, STV_HIDDEN,
-                       STT_FUNC, value,
-                       /*size=*/0, /*section=*/nullptr});
+  sym->resolve(ctx, Defined{ctx.internalFile, StringRef(), STB_GLOBAL,
+                            STV_HIDDEN, STT_FUNC, value,
+                            /*size=*/0, /*section=*/nullptr});
   defined.push_back(cast<Defined>(sym));
   return true;
 }
@@ -275,10 +275,10 @@ static void writeSequence(Ctx &ctx, MutableArrayRef<uint32_t> buf,
     format("%s%d", prefix, r).snprint(name, sizeof(name));
     if (addOptional(ctx, name, 4 * (r - from), defined) && defined.size() == 1)
       first = r - from;
-    write32(ptr++, firstInsn + 0x200008 * (r - from));
+    write32(ctx, ptr++, firstInsn + 0x200008 * (r - from));
   }
   for (uint32_t insn : tail)
-    write32(ptr++, insn);
+    write32(ctx, ptr++, insn);
   assert(ptr == &*buf.end());
 
   if (defined.empty())
@@ -566,15 +566,15 @@ static int64_t getTotalDisp(uint64_t prefixedInsn, uint32_t accessInsn) {
 // little-endian it is pointing to the start of the word. These 2 helpers are to
 // simplify reading and writing in that context.
 static void writeFromHalf16(Ctx &ctx, uint8_t *loc, uint32_t insn) {
-  write32(ctx.arg.isLE ? loc : loc - 2, insn);
+  write32(ctx, ctx.arg.isLE ? loc : loc - 2, insn);
 }
 
 static uint32_t readFromHalf16(Ctx &ctx, const uint8_t *loc) {
-  return read32(ctx.arg.isLE ? loc : loc - 2);
+  return read32(ctx, ctx.arg.isLE ? loc : loc - 2);
 }
 
 static uint64_t readPrefixedInst(Ctx &ctx, const uint8_t *loc) {
-  uint64_t fullInstr = read64(loc);
+  uint64_t fullInstr = read64(ctx, loc);
   return ctx.arg.isLE ? (fullInstr << 32 | fullInstr >> 32) : fullInstr;
 }
 
@@ -613,7 +613,7 @@ PPC64::PPC64(Ctx &ctx) : TargetInfo(ctx) {
   // use 0x10000000 as the starting address.
   defaultImageBase = 0x10000000;
 
-  write32(trapInstr.data(), 0x7fe00008);
+  write32(ctx, trapInstr.data(), 0x7fe00008);
 }
 
 int PPC64::getTlsGdRelaxSkip(RelType type) const {
@@ -684,7 +684,7 @@ void PPC64::relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     // be relaxed. The eligibility for the relaxation needs to be determined
     // on that relocation since this one does not relocate a symbol.
     uint64_t insn = readPrefixedInst(ctx, loc);
-    uint32_t accessInsn = read32(loc + rel.addend);
+    uint32_t accessInsn = read32(ctx, loc + rel.addend);
     uint64_t pcRelInsn = getPCRelativeForm(accessInsn);
 
     // This error is not necessary for correctness but is emitted for now
@@ -705,7 +705,7 @@ void PPC64::relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     writePrefixedInst(ctx, loc,
                       pcRelInsn | ((totalDisp & 0x3ffff0000) << 16) |
                           (totalDisp & 0xffff));
-    write32(loc + rel.addend, NOP); // nop accessInsn.
+    write32(ctx, loc + rel.addend, NOP); // nop accessInsn.
     break;
   }
   default:
@@ -757,15 +757,15 @@ void PPC64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
     //            addi r3, r3, x@tprel@l
     const uintptr_t locAsInt = reinterpret_cast<uintptr_t>(loc);
     if (locAsInt % 4 == 0) {
-      write32(loc, NOP);            // nop
-      write32(loc + 4, 0x38630000); // addi r3, r3
+      write32(ctx, loc, NOP);            // nop
+      write32(ctx, loc + 4, 0x38630000); // addi r3, r3
       // Since we are relocating a half16 type relocation and Loc + 4 points to
       // the start of an instruction we need to advance the buffer by an extra
       // 2 bytes on BE.
       relocateNoSym(loc + 4 + (ctx.arg.ekind == ELF64BEKind ? 2 : 0),
                     R_PPC64_TPREL16_LO, val);
     } else if (locAsInt % 4 == 1) {
-      write32(loc - 1, NOP);
+      write32(ctx, loc - 1, NOP);
     } else {
       errorOrWarn("R_PPC64_TLSGD has unexpected byte alignment");
     }
@@ -818,10 +818,10 @@ void PPC64::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel,
     //            addi r3, r3, 4096
     const uintptr_t locAsInt = reinterpret_cast<uintptr_t>(loc);
     if (locAsInt % 4 == 0) {
-      write32(loc, NOP);
-      write32(loc + 4, 0x38631000); // addi r3, r3, 4096
+      write32(ctx, loc, NOP);
+      write32(ctx, loc + 4, 0x38631000); // addi r3, r3, 4096
     } else if (locAsInt % 4 == 1) {
-      write32(loc - 1, NOP);
+      write32(ctx, loc - 1, NOP);
     } else {
       errorOrWarn("R_PPC64_TLSLD has unexpected byte alignment");
     }
@@ -912,12 +912,12 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
   unsigned offset = (ctx.arg.ekind == ELF64BEKind) ? 2 : 0;
   switch (rel.type) {
   case R_PPC64_GOT_TPREL16_HA:
-    write32(loc - offset, NOP);
+    write32(ctx, loc - offset, NOP);
     break;
   case R_PPC64_GOT_TPREL16_LO_DS:
   case R_PPC64_GOT_TPREL16_DS: {
-    uint32_t regNo = read32(loc - offset) & 0x03E00000; // bits 6-10
-    write32(loc - offset, 0x3C0D0000 | regNo);          // addis RegNo, r13
+    uint32_t regNo = read32(ctx, loc - offset) & 0x03e00000; // bits 6-10
+    write32(ctx, loc - offset, 0x3c0d0000 | regNo);          // addis RegNo, r13
     relocateNoSym(loc, R_PPC64_TPREL16_HA, val);
     break;
   }
@@ -931,10 +931,10 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
   case R_PPC64_TLS: {
     const uintptr_t locAsInt = reinterpret_cast<uintptr_t>(loc);
     if (locAsInt % 4 == 0) {
-      uint32_t primaryOp = getPrimaryOpCode(read32(loc));
+      uint32_t primaryOp = getPrimaryOpCode(read32(ctx, loc));
       if (primaryOp != 31)
         error("unrecognized instruction for IE to LE R_PPC64_TLS");
-      uint32_t secondaryOp = (read32(loc) & 0x000007FE) >> 1; // bits 21-30
+      uint32_t secondaryOp = (read32(ctx, loc) & 0x000007fe) >> 1; // bits 21-30
       uint32_t dFormOp = getPPCDFormOp(secondaryOp);
       uint32_t finalReloc;
       if (dFormOp == 0) { // Expecting a DS-Form instruction.
@@ -944,13 +944,13 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
         finalReloc = R_PPC64_TPREL16_LO_DS;
       } else
         finalReloc = R_PPC64_TPREL16_LO;
-      write32(loc, dFormOp | (read32(loc) & 0x03ff0000));
+      write32(ctx, loc, dFormOp | (read32(ctx, loc) & 0x03ff0000));
       relocateNoSym(loc + offset, finalReloc, val);
     } else if (locAsInt % 4 == 1) {
       // If the offset is not 4 byte aligned then we have a PCRel type reloc.
       // This version of the relocation is offset by one byte from the
       // instruction it references.
-      uint32_t tlsInstr = read32(loc - 1);
+      uint32_t tlsInstr = read32(ctx, loc - 1);
       uint32_t primaryOp = getPrimaryOpCode(tlsInstr);
       if (primaryOp != 31)
         errorOrWarn("unrecognized instruction for IE to LE R_PPC64_TLS");
@@ -963,10 +963,11 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
         uint32_t rt = (tlsInstr & 0x03E00000) >> 21; // bits 6-10
         uint32_t ra = (tlsInstr & 0x001F0000) >> 16; // bits 11-15
         if (ra == rt) {
-          write32(loc - 1, NOP);
+          write32(ctx, loc - 1, NOP);
         } else {
           // mr rt, ra
-          write32(loc - 1, 0x7C000378 | (rt << 16) | (ra << 21) | (ra << 11));
+          write32(ctx, loc - 1,
+                  0x7C000378 | (rt << 16) | (ra << 21) | (ra << 11));
         }
       } else {
         uint32_t dFormOp = getPPCDFormOp(secondaryOp);
@@ -975,7 +976,7 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
           if (dFormOp == 0)
             errorOrWarn("unrecognized instruction for IE to LE R_PPC64_TLS");
         }
-        write32(loc - 1, (dFormOp | (tlsInstr & 0x03ff0000)));
+        write32(ctx, loc - 1, (dFormOp | (tlsInstr & 0x03ff0000)));
       }
     } else {
       errorOrWarn("R_PPC64_TLS must be either 4 byte aligned or one byte "
@@ -1116,7 +1117,7 @@ int64_t PPC64::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_PPC64_JMP_SLOT:
     return 0;
   case R_PPC64_REL32:
-    return SignExtend64<32>(read32(buf));
+    return SignExtend64<32>(read32(ctx, buf));
   case R_PPC64_ADDR64:
   case R_PPC64_REL64:
   case R_PPC64_RELATIVE:
@@ -1124,7 +1125,7 @@ int64_t PPC64::getImplicitAddend(const uint8_t *buf, RelType type) const {
   case R_PPC64_DTPMOD64:
   case R_PPC64_DTPREL64:
   case R_PPC64_TPREL64:
-    return read64(buf);
+    return read64(ctx, buf);
   default:
     internalLinkerError(getErrorLoc(ctx, buf),
                         "cannot read addend for relocation " + toString(type));
@@ -1133,42 +1134,43 @@ int64_t PPC64::getImplicitAddend(const uint8_t *buf, RelType type) const {
 }
 
 void PPC64::writeGotHeader(uint8_t *buf) const {
-  write64(buf, getPPC64TocBase(ctx));
+  write64(ctx, buf, getPPC64TocBase(ctx));
 }
 
 void PPC64::writePltHeader(uint8_t *buf) const {
   // The generic resolver stub goes first.
-  write32(buf +  0, 0x7c0802a6); // mflr r0
-  write32(buf +  4, 0x429f0005); // bcl  20,4*cr7+so,8 <_glink+0x8>
-  write32(buf +  8, 0x7d6802a6); // mflr r11
-  write32(buf + 12, 0x7c0803a6); // mtlr r0
-  write32(buf + 16, 0x7d8b6050); // subf r12, r11, r12
-  write32(buf + 20, 0x380cffcc); // subi r0,r12,52
-  write32(buf + 24, 0x7800f082); // srdi r0,r0,62,2
-  write32(buf + 28, 0xe98b002c); // ld   r12,44(r11)
-  write32(buf + 32, 0x7d6c5a14); // add  r11,r12,r11
-  write32(buf + 36, 0xe98b0000); // ld   r12,0(r11)
-  write32(buf + 40, 0xe96b0008); // ld   r11,8(r11)
-  write32(buf + 44, 0x7d8903a6); // mtctr   r12
-  write32(buf + 48, 0x4e800420); // bctr
+  write32(ctx, buf + 0, 0x7c0802a6);  // mflr r0
+  write32(ctx, buf + 4, 0x429f0005);  // bcl  20,4*cr7+so,8 <_glink+0x8>
+  write32(ctx, buf + 8, 0x7d6802a6);  // mflr r11
+  write32(ctx, buf + 12, 0x7c0803a6); // mtlr r0
+  write32(ctx, buf + 16, 0x7d8b6050); // subf r12, r11, r12
+  write32(ctx, buf + 20, 0x380cffcc); // subi r0,r12,52
+  write32(ctx, buf + 24, 0x7800f082); // srdi r0,r0,62,2
+  write32(ctx, buf + 28, 0xe98b002c); // ld   r12,44(r11)
+  write32(ctx, buf + 32, 0x7d6c5a14); // add  r11,r12,r11
+  write32(ctx, buf + 36, 0xe98b0000); // ld   r12,0(r11)
+  write32(ctx, buf + 40, 0xe96b0008); // ld   r11,8(r11)
+  write32(ctx, buf + 44, 0x7d8903a6); // mtctr   r12
+  write32(ctx, buf + 48, 0x4e800420); // bctr
 
   // The 'bcl' instruction will set the link register to the address of the
   // following instruction ('mflr r11'). Here we store the offset from that
   // instruction  to the first entry in the GotPlt section.
   int64_t gotPltOffset = ctx.in.gotPlt->getVA() - (ctx.in.plt->getVA() + 8);
-  write64(buf + 52, gotPltOffset);
+  write64(ctx, buf + 52, gotPltOffset);
 }
 
 void PPC64::writePlt(uint8_t *buf, const Symbol &sym,
                      uint64_t /*pltEntryAddr*/) const {
   int32_t offset = pltHeaderSize + sym.getPltIdx(ctx) * pltEntrySize;
   // bl __glink_PLTresolve
-  write32(buf, 0x48000000 | ((-offset) & 0x03FFFFFc));
+  write32(ctx, buf, 0x48000000 | ((-offset) & 0x03fffffc));
 }
 
 void PPC64::writeIplt(uint8_t *buf, const Symbol &sym,
                       uint64_t /*pltEntryAddr*/) const {
-  writePPC64LoadAndBranch(buf, sym.getGotPltVA(ctx) - getPPC64TocBase(ctx));
+  writePPC64LoadAndBranch(ctx, buf,
+                          sym.getGotPltVA(ctx) - getPPC64TocBase(ctx));
 }
 
 static std::pair<RelType, uint64_t> toAddr16Rel(RelType type, uint64_t val) {
@@ -1265,28 +1267,28 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
 
   switch (type) {
   case R_PPC64_ADDR14: {
-    checkAlignment(loc, val, 4, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
     // Preserve the AA/LK bits in the branch instruction
     uint8_t aalk = loc[3];
-    write16(loc + 2, (aalk & 3) | (val & 0xfffc));
+    write16(ctx, loc + 2, (aalk & 3) | (val & 0xfffc));
     break;
   }
   case R_PPC64_ADDR16:
-    checkIntUInt(loc, val, 16, rel);
-    write16(loc, val);
+    checkIntUInt(ctx, loc, val, 16, rel);
+    write16(ctx, loc, val);
     break;
   case R_PPC64_ADDR32:
-    checkIntUInt(loc, val, 32, rel);
-    write32(loc, val);
+    checkIntUInt(ctx, loc, val, 32, rel);
+    write32(ctx, loc, val);
     break;
   case R_PPC64_ADDR16_DS:
   case R_PPC64_TPREL16_DS: {
-    checkInt(loc, val, 16, rel);
+    checkInt(ctx, loc, val, 16, rel);
     // DQ-form instructions use bits 28-31 as part of the instruction encoding
     // DS-form instructions only use bits 30-31.
     uint16_t mask = isDQFormInstruction(readFromHalf16(ctx, loc)) ? 0xf : 0x3;
-    checkAlignment(loc, lo(val), mask + 1, rel);
-    write16(loc, (read16(loc) & mask) | lo(val));
+    checkAlignment(ctx, loc, lo(val), mask + 1, rel);
+    write16(ctx, loc, (read16(ctx, loc) & mask) | lo(val));
   } break;
   case R_PPC64_ADDR16_HA:
   case R_PPC64_REL16_HA:
@@ -1294,34 +1296,34 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     if (ctx.arg.tocOptimize && shouldTocOptimize && ha(val) == 0)
       writeFromHalf16(ctx, loc, NOP);
     else {
-      checkInt(loc, val + 0x8000, 32, rel);
-      write16(loc, ha(val));
+      checkInt(ctx, loc, val + 0x8000, 32, rel);
+      write16(ctx, loc, ha(val));
     }
     break;
   case R_PPC64_ADDR16_HI:
   case R_PPC64_REL16_HI:
   case R_PPC64_TPREL16_HI:
-    checkInt(loc, val, 32, rel);
-    write16(loc, hi(val));
+    checkInt(ctx, loc, val, 32, rel);
+    write16(ctx, loc, hi(val));
     break;
   case R_PPC64_ADDR16_HIGH:
-    write16(loc, hi(val));
+    write16(ctx, loc, hi(val));
     break;
   case R_PPC64_ADDR16_HIGHER:
   case R_PPC64_TPREL16_HIGHER:
-    write16(loc, higher(val));
+    write16(ctx, loc, higher(val));
     break;
   case R_PPC64_ADDR16_HIGHERA:
   case R_PPC64_TPREL16_HIGHERA:
-    write16(loc, highera(val));
+    write16(ctx, loc, highera(val));
     break;
   case R_PPC64_ADDR16_HIGHEST:
   case R_PPC64_TPREL16_HIGHEST:
-    write16(loc, highest(val));
+    write16(ctx, loc, highest(val));
     break;
   case R_PPC64_ADDR16_HIGHESTA:
   case R_PPC64_TPREL16_HIGHESTA:
-    write16(loc, highesta(val));
+    write16(ctx, loc, highesta(val));
     break;
   case R_PPC64_ADDR16_LO:
   case R_PPC64_REL16_LO:
@@ -1336,7 +1338,7 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
               "can't toc-optimize an update instruction: 0x" + utohexstr(insn));
       writeFromHalf16(ctx, loc, (insn & 0xffe00000) | 0x00020000 | lo(val));
     } else {
-      write16(loc, lo(val));
+      write16(ctx, loc, lo(val));
     }
     break;
   case R_PPC64_ADDR16_LO_DS:
@@ -1345,7 +1347,7 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     // DS-form instructions only use bits 30-31.
     uint32_t insn = readFromHalf16(ctx, loc);
     uint16_t mask = isDQFormInstruction(insn) ? 0xf : 0x3;
-    checkAlignment(loc, lo(val), mask + 1, rel);
+    checkAlignment(ctx, loc, lo(val), mask + 1, rel);
     if (ctx.arg.tocOptimize && shouldTocOptimize && ha(val) == 0) {
       // When the high-adjusted part of a toc relocation evaluates to 0, it is
       // changed into a nop. The lo part then needs to be updated to use the toc
@@ -1357,39 +1359,39 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
       insn &= 0xffe00000 | mask;
       writeFromHalf16(ctx, loc, insn | 0x00020000 | lo(val));
     } else {
-      write16(loc, (read16(loc) & mask) | lo(val));
+      write16(ctx, loc, (read16(ctx, loc) & mask) | lo(val));
     }
   } break;
   case R_PPC64_TPREL16:
-    checkInt(loc, val, 16, rel);
-    write16(loc, val);
+    checkInt(ctx, loc, val, 16, rel);
+    write16(ctx, loc, val);
     break;
   case R_PPC64_REL32:
-    checkInt(loc, val, 32, rel);
-    write32(loc, val);
+    checkInt(ctx, loc, val, 32, rel);
+    write32(ctx, loc, val);
     break;
   case R_PPC64_ADDR64:
   case R_PPC64_REL64:
   case R_PPC64_TOC:
-    write64(loc, val);
+    write64(ctx, loc, val);
     break;
   case R_PPC64_REL14: {
     uint32_t mask = 0x0000FFFC;
-    checkInt(loc, val, 16, rel);
-    checkAlignment(loc, val, 4, rel);
-    write32(loc, (read32(loc) & ~mask) | (val & mask));
+    checkInt(ctx, loc, val, 16, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
+    write32(ctx, loc, (read32(ctx, loc) & ~mask) | (val & mask));
     break;
   }
   case R_PPC64_REL24:
   case R_PPC64_REL24_NOTOC: {
     uint32_t mask = 0x03FFFFFC;
-    checkInt(loc, val, 26, rel);
-    checkAlignment(loc, val, 4, rel);
-    write32(loc, (read32(loc) & ~mask) | (val & mask));
+    checkInt(ctx, loc, val, 26, rel);
+    checkAlignment(ctx, loc, val, 4, rel);
+    write32(ctx, loc, (read32(ctx, loc) & ~mask) | (val & mask));
     break;
   }
   case R_PPC64_DTPREL64:
-    write64(loc, val - dynamicThreadPointerOffset);
+    write64(ctx, loc, val - dynamicThreadPointerOffset);
     break;
   case R_PPC64_DTPREL34:
     // The Dynamic Thread Vector actually points 0x8000 bytes past the start
@@ -1406,7 +1408,7 @@ void PPC64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     const uint64_t si0Mask = 0x00000003ffff0000;
     const uint64_t si1Mask = 0x000000000000ffff;
     const uint64_t fullMask = 0x0003ffff0000ffff;
-    checkInt(loc, val, 34, rel);
+    checkInt(ctx, loc, val, 34, rel);
 
     uint64_t instr = readPrefixedInst(ctx, loc) & ~fullMask;
     writePrefixedInst(ctx, loc,
@@ -1544,11 +1546,11 @@ void PPC64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
     //            add r3, r3, r13
     const uintptr_t locAsInt = reinterpret_cast<uintptr_t>(loc);
     if (locAsInt % 4 == 0) {
-      write32(loc, NOP);            // bl __tls_get_addr(sym@tlsgd) --> nop
-      write32(loc + 4, 0x7c636A14); // nop --> add r3, r3, r13
+      write32(ctx, loc, NOP);            // bl __tls_get_addr(sym@tlsgd) --> nop
+      write32(ctx, loc + 4, 0x7c636a14); // nop --> add r3, r3, r13
     } else if (locAsInt % 4 == 1) {
       // bl __tls_get_addr(sym@tlsgd) --> add r3, r3, r13
-      write32(loc - 1, 0x7c636a14);
+      write32(ctx, loc - 1, 0x7c636a14);
     } else {
       errorOrWarn("R_PPC64_TLSGD has unexpected byte alignment");
     }
@@ -1598,7 +1600,7 @@ void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
       // If this is a call to __tls_get_addr, it may be part of a TLS
       // sequence that has been relaxed and turned into a nop. In this
       // case, we don't want to handle it as a call.
-      if (read32(loc) == 0x60000000) // nop
+      if (read32(ctx, loc) == 0x60000000) // nop
         break;
 
       // Patch a nop (0x60000000) to a ld.
@@ -1608,7 +1610,7 @@ void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
         // wrong in the common case where the function is not preempted at
         // runtime. Just ignore.
         if ((rel.offset + 8 > sec.content().size() ||
-             read32(loc + 4) != 0x60000000) &&
+             read32(ctx, loc + 4) != 0x60000000) &&
             rel.sym->file != sec.file) {
           // Use substr(6) to remove the "__plt_" prefix.
           errorOrWarn(getErrorLoc(ctx, loc) + "call to " +
@@ -1616,7 +1618,7 @@ void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
                       " lacks nop, can't restore toc");
           break;
         }
-        write32(loc + 4, 0xe8410018); // ld %r2, 24(%r1)
+        write32(ctx, loc + 4, 0xe8410018); // ld %r2, 24(%r1)
       }
       relocate(loc, rel, val);
       break;
@@ -1682,14 +1684,14 @@ bool PPC64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end,
     return false;
 
   // First instruction must be `ld r0, -0x7000-64(r13)`
-  if (read32(loc) != 0xe80d8fc0)
+  if (read32(ctx, loc) != 0xe80d8fc0)
     return false;
 
   int16_t hiImm = 0;
   int16_t loImm = 0;
   // First instruction can be either an addis if the frame size is larger then
   // 32K, or an addi if the size is less then 32K.
-  int32_t firstInstr = read32(loc + 4);
+  int32_t firstInstr = read32(ctx, loc + 4);
   if (getPrimaryOpCode(firstInstr) == 15) {
     hiImm = firstInstr & 0xFFFF;
   } else if (getPrimaryOpCode(firstInstr) == 14) {
@@ -1700,7 +1702,7 @@ bool PPC64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end,
 
   // Second instruction is either an addi or a nop. If the first instruction was
   // an addi then LoImm is set and the second instruction must be a nop.
-  uint32_t secondInstr = read32(loc + 8);
+  uint32_t secondInstr = read32(ctx, loc + 8);
   if (!loImm && getPrimaryOpCode(secondInstr) == 14) {
     loImm = secondInstr & 0xFFFF;
   } else if (secondInstr != NOP) {
@@ -1734,14 +1736,14 @@ bool PPC64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end,
   loImm = adjustedStackFrameSize & 0xFFFF;
   hiImm = (adjustedStackFrameSize + 0x8000) >> 16;
   if (hiImm) {
-    write32(loc + 4, 0x3D810000 | (uint16_t)hiImm);
+    write32(ctx, loc + 4, 0x3d810000 | (uint16_t)hiImm);
     // If the low immediate is zero the second instruction will be a nop.
     secondInstr = loImm ? 0x398C0000 | (uint16_t)loImm : NOP;
-    write32(loc + 8, secondInstr);
+    write32(ctx, loc + 8, secondInstr);
   } else {
     // addi r12, r1, imm
-    write32(loc + 4, (0x39810000) | (uint16_t)loImm);
-    write32(loc + 8, NOP);
+    write32(ctx, loc + 4, (0x39810000) | (uint16_t)loImm);
+    write32(ctx, loc + 8, NOP);
   }
 
   return true;
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index d65467f10378beaf4956ca2d1db3ec2aec1adf9b..1ae016e4de01eed34029e24ec4d5f753a05586b3 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -343,8 +343,8 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     return;
 
   case R_RISCV_RVC_BRANCH: {
-    checkInt(loc, val, 9, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 9, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     uint16_t insn = read16le(loc) & 0xE383;
     uint16_t imm8 = extractBits(val, 8, 8) << 12;
     uint16_t imm4_3 = extractBits(val, 4, 3) << 10;
@@ -358,8 +358,8 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   }
 
   case R_RISCV_RVC_JUMP: {
-    checkInt(loc, val, 12, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 12, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     uint16_t insn = read16le(loc) & 0xE003;
     uint16_t imm11 = extractBits(val, 11, 11) << 12;
     uint16_t imm4 = extractBits(val, 4, 4) << 11;
@@ -377,7 +377,7 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
 
   case R_RISCV_RVC_LUI: {
     int64_t imm = SignExtend64(val + 0x800, bits) >> 12;
-    checkInt(loc, imm, 6, rel);
+    checkInt(ctx, loc, imm, 6, rel);
     if (imm == 0) { // `c.lui rd, 0` is illegal, convert to `c.li rd, 0`
       write16le(loc, (read16le(loc) & 0x0F83) | 0x4000);
     } else {
@@ -389,8 +389,8 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   }
 
   case R_RISCV_JAL: {
-    checkInt(loc, val, 21, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 21, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
 
     uint32_t insn = read32le(loc) & 0xFFF;
     uint32_t imm20 = extractBits(val, 20, 20) << 31;
@@ -404,8 +404,8 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   }
 
   case R_RISCV_BRANCH: {
-    checkInt(loc, val, 13, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 13, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
 
     uint32_t insn = read32le(loc) & 0x1FFF07F;
     uint32_t imm12 = extractBits(val, 12, 12) << 31;
@@ -422,7 +422,7 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_RISCV_CALL:
   case R_RISCV_CALL_PLT: {
     int64_t hi = SignExtend64(val + 0x800, bits) >> 12;
-    checkInt(loc, hi, 20, rel);
+    checkInt(ctx, loc, hi, 20, rel);
     if (isInt<20>(hi)) {
       relocateNoSym(loc, R_RISCV_PCREL_HI20, val);
       relocateNoSym(loc + 4, R_RISCV_PCREL_LO12_I, val);
@@ -438,7 +438,7 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_RISCV_TPREL_HI20:
   case R_RISCV_HI20: {
     uint64_t hi = val + 0x800;
-    checkInt(loc, SignExtend64(hi, bits) >> 12, 20, rel);
+    checkInt(ctx, loc, SignExtend64(hi, bits) >> 12, 20, rel);
     write32le(loc, (read32le(loc) & 0xFFF) | (hi & 0xFFFFF000));
     return;
   }
@@ -467,7 +467,7 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case INTERNAL_R_RISCV_GPREL_S: {
     Defined *gp = ctx.sym.riscvGlobalPointer;
     int64_t displace = SignExtend64(val - gp->getVA(), bits);
-    checkInt(loc, displace, 12, rel);
+    checkInt(ctx, loc, displace, 12, rel);
     uint32_t insn = (read32le(loc) & ~(31 << 15)) | (X_GP << 15);
     if (rel.type == INTERNAL_R_RISCV_GPREL_I)
       insn = setLO12_I(insn, displace);
@@ -517,7 +517,7 @@ void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_RISCV_32_PCREL:
   case R_RISCV_PLT32:
   case R_RISCV_GOT32_PCREL:
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     write32le(loc, val);
     return;
 
@@ -1282,14 +1282,14 @@ void RISCVAttributesSection::writeTo(uint8_t *buf) {
   const size_t size = getSize();
   uint8_t *const end = buf + size;
   *buf = ELFAttrs::Format_Version;
-  write32(buf + 1, size - 1);
+  write32(ctx, buf + 1, size - 1);
   buf += 5;
 
   memcpy(buf, vendor.data(), vendor.size());
   buf += vendor.size() + 1;
 
   *buf = ELFAttrs::File;
-  write32(buf + 1, end - buf);
+  write32(ctx, buf + 1, end - buf);
   buf += 5;
 
   for (auto &attr : intAttr) {
diff --git a/lld/ELF/Arch/SPARCV9.cpp b/lld/ELF/Arch/SPARCV9.cpp
index bc9724c3ba342a5b1ecd10e811fb4c04040b8299..ef71014e12fa97f3b878080924ecf5718d6f134b 100644
--- a/lld/ELF/Arch/SPARCV9.cpp
+++ b/lld/ELF/Arch/SPARCV9.cpp
@@ -90,23 +90,23 @@ void SPARCV9::relocate(uint8_t *loc, const Relocation &rel,
   case R_SPARC_32:
   case R_SPARC_UA32:
     // V-word32
-    checkUInt(loc, val, 32, rel);
+    checkUInt(ctx, loc, val, 32, rel);
     write32be(loc, val);
     break;
   case R_SPARC_DISP32:
     // V-disp32
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     write32be(loc, val);
     break;
   case R_SPARC_WDISP30:
   case R_SPARC_WPLT30:
     // V-disp30
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     write32be(loc, (read32be(loc) & ~0x3fffffff) | ((val >> 2) & 0x3fffffff));
     break;
   case R_SPARC_22:
     // V-imm22
-    checkUInt(loc, val, 22, rel);
+    checkUInt(ctx, loc, val, 22, rel);
     write32be(loc, (read32be(loc) & ~0x003fffff) | (val & 0x003fffff));
     break;
   case R_SPARC_GOT22:
@@ -117,12 +117,12 @@ void SPARCV9::relocate(uint8_t *loc, const Relocation &rel,
     break;
   case R_SPARC_HI22:
     // V-imm22
-    checkUInt(loc, val >> 10, 22, rel);
+    checkUInt(ctx, loc, val >> 10, 22, rel);
     write32be(loc, (read32be(loc) & ~0x003fffff) | ((val >> 10) & 0x003fffff));
     break;
   case R_SPARC_WDISP19:
     // V-disp19
-    checkInt(loc, val, 21, rel);
+    checkInt(ctx, loc, val, 21, rel);
     write32be(loc, (read32be(loc) & ~0x0007ffff) | ((val >> 2) & 0x0007ffff));
     break;
   case R_SPARC_GOT10:
@@ -141,7 +141,7 @@ void SPARCV9::relocate(uint8_t *loc, const Relocation &rel,
     break;
   case R_SPARC_HH22:
     // V-imm22
-    checkUInt(loc, val >> 42, 22, rel);
+    checkUInt(ctx, loc, val >> 42, 22, rel);
     write32be(loc, (read32be(loc) & ~0x003fffff) | ((val >> 42) & 0x003fffff));
     break;
   case R_SPARC_HM10:
@@ -150,7 +150,7 @@ void SPARCV9::relocate(uint8_t *loc, const Relocation &rel,
     break;
   case R_SPARC_H44:
     // V-imm22
-    checkUInt(loc, val >> 22, 22, rel);
+    checkUInt(ctx, loc, val >> 22, 22, rel);
     write32be(loc, (read32be(loc) & ~0x003fffff) | ((val >> 22) & 0x003fffff));
     break;
   case R_SPARC_M44:
diff --git a/lld/ELF/Arch/SystemZ.cpp b/lld/ELF/Arch/SystemZ.cpp
index 1bc3b60af5c44d763776593613bc84a18a063a04..584379638ad981fde9c5ae8bb7006f647f03fb85 100644
--- a/lld/ELF/Arch/SystemZ.cpp
+++ b/lld/ELF/Arch/SystemZ.cpp
@@ -493,20 +493,20 @@ void SystemZ::relocate(uint8_t *loc, const Relocation &rel,
   }
   switch (rel.type) {
   case R_390_8:
-    checkIntUInt(loc, val, 8, rel);
+    checkIntUInt(ctx, loc, val, 8, rel);
     *loc = val;
     break;
   case R_390_12:
   case R_390_GOT12:
   case R_390_GOTPLT12:
   case R_390_TLS_GOTIE12:
-    checkUInt(loc, val, 12, rel);
+    checkUInt(ctx, loc, val, 12, rel);
     write16be(loc, (read16be(loc) & 0xF000) | val);
     break;
   case R_390_PC12DBL:
   case R_390_PLT12DBL:
-    checkInt(loc, val, 13, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 13, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     write16be(loc, (read16be(loc) & 0xF000) | ((val >> 1) & 0x0FFF));
     break;
   case R_390_16:
@@ -514,31 +514,31 @@ void SystemZ::relocate(uint8_t *loc, const Relocation &rel,
   case R_390_GOTPLT16:
   case R_390_GOTOFF16:
   case R_390_PLTOFF16:
-    checkIntUInt(loc, val, 16, rel);
+    checkIntUInt(ctx, loc, val, 16, rel);
     write16be(loc, val);
     break;
   case R_390_PC16:
-    checkInt(loc, val, 16, rel);
+    checkInt(ctx, loc, val, 16, rel);
     write16be(loc, val);
     break;
   case R_390_PC16DBL:
   case R_390_PLT16DBL:
-    checkInt(loc, val, 17, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 17, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     write16be(loc, val >> 1);
     break;
   case R_390_20:
   case R_390_GOT20:
   case R_390_GOTPLT20:
   case R_390_TLS_GOTIE20:
-    checkInt(loc, val, 20, rel);
+    checkInt(ctx, loc, val, 20, rel);
     write32be(loc, (read32be(loc) & 0xF00000FF) | ((val & 0xFFF) << 16) |
                        ((val & 0xFF000) >> 4));
     break;
   case R_390_PC24DBL:
   case R_390_PLT24DBL:
-    checkInt(loc, val, 25, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 25, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     loc[0] = val >> 17;
     loc[1] = val >> 9;
     loc[2] = val >> 1;
@@ -554,12 +554,12 @@ void SystemZ::relocate(uint8_t *loc, const Relocation &rel,
   case R_390_TLS_LDM32:
   case R_390_TLS_LDO32:
   case R_390_TLS_LE32:
-    checkIntUInt(loc, val, 32, rel);
+    checkIntUInt(ctx, loc, val, 32, rel);
     write32be(loc, val);
     break;
   case R_390_PC32:
   case R_390_PLT32:
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     write32be(loc, val);
     break;
   case R_390_PC32DBL:
@@ -568,8 +568,8 @@ void SystemZ::relocate(uint8_t *loc, const Relocation &rel,
   case R_390_GOTENT:
   case R_390_GOTPLTENT:
   case R_390_TLS_IEENT:
-    checkInt(loc, val, 33, rel);
-    checkAlignment(loc, val, 2, rel);
+    checkInt(ctx, loc, val, 33, rel);
+    checkAlignment(ctx, loc, val, 2, rel);
     write32be(loc, val >> 1);
     break;
   case R_390_64:
diff --git a/lld/ELF/Arch/X86.cpp b/lld/ELF/Arch/X86.cpp
index 4e574a520f1ff1bb1cec614fc5363e93e9605f3b..58199cdb99a2846cae4d3df1c06ac1ba14da133c 100644
--- a/lld/ELF/Arch/X86.cpp
+++ b/lld/ELF/Arch/X86.cpp
@@ -292,15 +292,15 @@ void X86::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     // R_386_{PC,}{8,16} are not part of the i386 psABI, but they are
     // being used for some 16-bit programs such as boot loaders, so
     // we want to support them.
-    checkIntUInt(loc, val, 8, rel);
+    checkIntUInt(ctx, loc, val, 8, rel);
     *loc = val;
     break;
   case R_386_PC8:
-    checkInt(loc, val, 8, rel);
+    checkInt(ctx, loc, val, 8, rel);
     *loc = val;
     break;
   case R_386_16:
-    checkIntUInt(loc, val, 16, rel);
+    checkIntUInt(ctx, loc, val, 16, rel);
     write16le(loc, val);
     break;
   case R_386_PC16:
@@ -314,7 +314,7 @@ void X86::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     // current location subtracted from it.
     // We just check that Val fits in 17 bits. This misses some cases, but
     // should have no false positives.
-    checkInt(loc, val, 17, rel);
+    checkInt(ctx, loc, val, 17, rel);
     write16le(loc, val);
     break;
   case R_386_32:
@@ -338,7 +338,7 @@ void X86::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_386_TLS_LE_32:
   case R_386_TLS_TPOFF:
   case R_386_TLS_TPOFF32:
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     write32le(loc, val);
     break;
   case R_386_TLS_DESC:
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index 121b7d9929b20921d25769fa902fad2b3b36703b..df2983f2022818c64a23881f436ff21e2b3a0d1a 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -186,7 +186,7 @@ static bool isFallThruRelocation(InputSection &is, InputFile *file,
     return false;
 
   uint64_t addrLoc = is.getOutputSection()->addr + is.outSecOff + r.offset;
-  uint64_t targetOffset = is.getRelocTargetVA(ctx, r, addrLoc);
+  uint64_t targetOffset = is.getRelocTargetVA(is.getCtx(), r, addrLoc);
 
   // If this jmp is a fall thru, the target offset is the beginning of the
   // next section.
@@ -777,23 +777,23 @@ static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val);
 void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   switch (rel.type) {
   case R_X86_64_8:
-    checkIntUInt(loc, val, 8, rel);
+    checkIntUInt(ctx, loc, val, 8, rel);
     *loc = val;
     break;
   case R_X86_64_PC8:
-    checkInt(loc, val, 8, rel);
+    checkInt(ctx, loc, val, 8, rel);
     *loc = val;
     break;
   case R_X86_64_16:
-    checkIntUInt(loc, val, 16, rel);
+    checkIntUInt(ctx, loc, val, 16, rel);
     write16le(loc, val);
     break;
   case R_X86_64_PC16:
-    checkInt(loc, val, 16, rel);
+    checkInt(ctx, loc, val, 16, rel);
     write16le(loc, val);
     break;
   case R_X86_64_32:
-    checkUInt(loc, val, 32, rel);
+    checkUInt(ctx, loc, val, 32, rel);
     write32le(loc, val);
     break;
   case R_X86_64_32S:
@@ -804,7 +804,7 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   case R_X86_64_PLT32:
   case R_X86_64_DTPOFF32:
   case R_X86_64_SIZE32:
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     write32le(loc, val);
     break;
   case R_X86_64_64:
@@ -824,7 +824,7 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     if (rel.expr != R_GOT_PC) {
       relaxGot(loc, rel, val);
     } else {
-      checkInt(loc, val, 32, rel);
+      checkInt(ctx, loc, val, 32, rel);
       write32le(loc, val);
     }
     break;
@@ -836,7 +836,7 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     } else if (rel.expr == R_RELAX_TLS_GD_TO_IE) {
       relaxTlsGdToIe(loc, rel, val);
     } else {
-      checkInt(loc, val, 32, rel);
+      checkInt(ctx, loc, val, 32, rel);
       write32le(loc, val);
     }
     break;
@@ -844,7 +844,7 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     if (rel.expr == R_RELAX_TLS_LD_TO_LE) {
       relaxTlsLdToLe(loc, rel, val);
     } else {
-      checkInt(loc, val, 32, rel);
+      checkInt(ctx, loc, val, 32, rel);
       write32le(loc, val);
     }
     break;
@@ -852,12 +852,12 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     if (rel.expr == R_RELAX_TLS_IE_TO_LE) {
       relaxTlsIeToLe(loc, rel, val);
     } else {
-      checkInt(loc, val, 32, rel);
+      checkInt(ctx, loc, val, 32, rel);
       write32le(loc, val);
     }
     break;
   case R_X86_64_TPOFF32:
-    checkInt(loc, val, 32, rel);
+    checkInt(ctx, loc, val, 32, rel);
     write32le(loc, val);
     break;
 
diff --git a/lld/ELF/CallGraphSort.cpp b/lld/ELF/CallGraphSort.cpp
index e73fe12317bd76427f7afe4fd77f313432ded811..537e065081f3fcb902f6fd4b4a00252f6bed26e0 100644
--- a/lld/ELF/CallGraphSort.cpp
+++ b/lld/ELF/CallGraphSort.cpp
@@ -88,11 +88,12 @@ struct Cluster {
 /// * Sort non-empty clusters by density
 class CallGraphSort {
 public:
-  CallGraphSort();
+  CallGraphSort(Ctx &);
 
   DenseMap<const InputSectionBase *, int> run();
 
 private:
+  Ctx &ctx;
   std::vector<Cluster> clusters;
   std::vector<const InputSectionBase *> sections;
 };
@@ -111,7 +112,7 @@ using SectionPair =
 // Take the edge list in ctx.arg.callGraphProfile, resolve symbol names to
 // Symbols, and generate a graph between InputSections with the provided
 // weights.
-CallGraphSort::CallGraphSort() {
+CallGraphSort::CallGraphSort(Ctx &ctx) : ctx(ctx) {
   MapVector<SectionPair, uint64_t> &profile = ctx.arg.callGraphProfile;
   DenseMap<const InputSectionBase *, int> secToCluster;
 
@@ -274,7 +275,8 @@ DenseMap<const InputSectionBase *, int> CallGraphSort::run() {
 // Sort sections by the profile data using the Cache-Directed Sort algorithm.
 // The placement is done by optimizing the locality by co-locating frequently
 // executed code sections together.
-DenseMap<const InputSectionBase *, int> elf::computeCacheDirectedSortOrder() {
+DenseMap<const InputSectionBase *, int>
+elf::computeCacheDirectedSortOrder(Ctx &ctx) {
   SmallVector<uint64_t, 0> funcSizes;
   SmallVector<uint64_t, 0> funcCounts;
   SmallVector<codelayout::EdgeCount, 0> callCounts;
@@ -336,8 +338,9 @@ DenseMap<const InputSectionBase *, int> elf::computeCacheDirectedSortOrder() {
 //
 // This first builds a call graph based on the profile data then merges sections
 // according either to the C³ or Cache-Directed-Sort ordering algorithm.
-DenseMap<const InputSectionBase *, int> elf::computeCallGraphProfileOrder() {
+DenseMap<const InputSectionBase *, int>
+elf::computeCallGraphProfileOrder(Ctx &ctx) {
   if (ctx.arg.callGraphProfileSort == CGProfileSortKind::Cdsort)
-    return computeCacheDirectedSortOrder();
-  return CallGraphSort().run();
+    return computeCacheDirectedSortOrder(ctx);
+  return CallGraphSort(ctx).run();
 }
diff --git a/lld/ELF/CallGraphSort.h b/lld/ELF/CallGraphSort.h
index 1b54f2b62482284bb2d02581dc7481b367ff1760..5f9987ce0973571c1404ef306edeeb9c0fe11ca1 100644
--- a/lld/ELF/CallGraphSort.h
+++ b/lld/ELF/CallGraphSort.h
@@ -12,11 +12,14 @@
 #include "llvm/ADT/DenseMap.h"
 
 namespace lld::elf {
+struct Ctx;
 class InputSectionBase;
 
-llvm::DenseMap<const InputSectionBase *, int> computeCacheDirectedSortOrder();
+llvm::DenseMap<const InputSectionBase *, int>
+computeCacheDirectedSortOrder(Ctx &);
 
-llvm::DenseMap<const InputSectionBase *, int> computeCallGraphProfileOrder();
+llvm::DenseMap<const InputSectionBase *, int>
+computeCallGraphProfileOrder(Ctx &);
 } // namespace lld::elf
 
 #endif
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 019388c9bd2e2c85493b0270aa8afb568a50d534..afacbbfe9099cc80ffc4f23f0ebdb18ca80f62cd 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -155,8 +155,9 @@ bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
 
   context->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput);
   context->e.cleanupCallback = []() {
-    elf::ctx.reset();
-    elf::ctx.partitions.emplace_back();
+    Ctx &ctx = elf::ctx;
+    ctx.reset();
+    ctx.partitions.emplace_back(ctx);
 
     SharedFile::vernauxNum = 0;
   };
@@ -165,13 +166,14 @@ bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
       "too many errors emitted, stopping now (use "
       "--error-limit=0 to see all errors)";
 
+  Ctx &ctx = elf::ctx;
   LinkerScript script(ctx);
   ctx.script = &script;
   ctx.symAux.emplace_back();
   ctx.symtab = std::make_unique<SymbolTable>(ctx);
 
   ctx.partitions.clear();
-  ctx.partitions.emplace_back();
+  ctx.partitions.emplace_back(ctx);
 
   ctx.arg.progName = args[0];
 
@@ -967,7 +969,7 @@ static void readCallGraph(Ctx &ctx, MemoryBufferRef mb) {
         warn(mb.getBufferIdentifier() + ": no such symbol: " + name);
       return nullptr;
     }
-    maybeWarnUnorderableSymbol(sym);
+    maybeWarnUnorderableSymbol(ctx, sym);
 
     if (Defined *dr = dyn_cast_or_null<Defined>(sym))
       return dyn_cast_or_null<InputSectionBase>(dr->section);
@@ -2184,7 +2186,7 @@ static void handleUndefined(Ctx &ctx, Symbol *sym, const char *option) {
 
   if (!sym->isLazy())
     return;
-  sym->extract();
+  sym->extract(ctx);
   if (!ctx.arg.whyExtract.empty())
     ctx.whyExtractRecords.emplace_back(option, sym->file, *sym);
 }
@@ -2215,7 +2217,7 @@ static void handleLibcall(Ctx &ctx, StringRef name) {
   if (sym && sym->isLazy() && isa<BitcodeFile>(sym->file)) {
     if (!ctx.arg.whyExtract.empty())
       ctx.whyExtractRecords.emplace_back("<libcall>", sym->file, *sym);
-    sym->extract();
+    sym->extract(ctx);
   }
 }
 
@@ -2414,7 +2416,7 @@ static void findKeepUniqueSections(Ctx &ctx, opt::InputArgList &args) {
   // or DSOs, so we conservatively mark them as address-significant.
   bool icfSafe = ctx.arg.icf == ICFLevel::Safe;
   for (Symbol *sym : ctx.symtab->getSymbols())
-    if (sym->includeInDynsym())
+    if (sym->includeInDynsym(ctx))
       markAddrsig(icfSafe, sym);
 
   // Visit the address-significance table in each object file and mark each
@@ -2463,7 +2465,7 @@ static void readSymbolPartitionSection(Ctx &ctx, InputSectionBase *s) {
     sym = readEntry(s->file, rels.rels);
   else
     sym = readEntry(s->file, rels.relas);
-  if (!isa_and_nonnull<Defined>(sym) || !sym->includeInDynsym())
+  if (!isa_and_nonnull<Defined>(sym) || !sym->includeInDynsym(ctx))
     return;
 
   StringRef partName = reinterpret_cast<const char *>(s->content().data());
@@ -2495,7 +2497,7 @@ static void readSymbolPartitionSection(Ctx &ctx, InputSectionBase *s) {
   if (ctx.partitions.size() == 254)
     fatal("may not have more than 254 partitions");
 
-  ctx.partitions.emplace_back();
+  ctx.partitions.emplace_back(ctx);
   Partition &newPart = ctx.partitions.back();
   newPart.name = partName;
   sym->partition = newPart.getNumber();
@@ -2549,7 +2551,7 @@ void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) {
     if (!ctx.arg.relocatable)
       for (Symbol *sym : obj->getGlobalSymbols())
         if (sym->hasVersionSuffix)
-          sym->parseSymbolVersion();
+          sym->parseSymbolVersion(ctx);
     ctx.objectFiles.push_back(obj);
   }
 }
@@ -2646,12 +2648,12 @@ static void combineVersionedSymbol(Ctx &ctx, Symbol &sym,
     // If both foo@v1 and foo@@v1 are defined and non-weak, report a
     // duplicate definition error.
     if (sym.isDefined()) {
-      sym2->checkDuplicate(cast<Defined>(sym));
-      sym2->resolve(cast<Defined>(sym));
+      sym2->checkDuplicate(ctx, cast<Defined>(sym));
+      sym2->resolve(ctx, cast<Defined>(sym));
     } else if (sym.isUndefined()) {
-      sym2->resolve(cast<Undefined>(sym));
+      sym2->resolve(ctx, cast<Undefined>(sym));
     } else {
-      sym2->resolve(cast<SharedSymbol>(sym));
+      sym2->resolve(ctx, cast<SharedSymbol>(sym));
     }
     // Eliminate foo@v1 from the symbol table.
     sym.symbolKind = Symbol::PlaceholderKind;
@@ -2973,7 +2975,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
 
   // Create elfHeader early. We need a dummy section in
   // addReservedSymbols to mark the created symbols as not absolute.
-  ctx.out.elfHeader = make<OutputSection>("", 0, SHF_ALLOC);
+  ctx.out.elfHeader = make<OutputSection>(ctx, "", 0, SHF_ALLOC);
 
   // We need to create some reserved symbols such as _end. Create them.
   if (!ctx.arg.relocatable)
@@ -3145,7 +3147,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
 
   // This adds a .comment section containing a version string.
   if (!ctx.arg.relocatable)
-    ctx.inputSections.push_back(createCommentSection());
+    ctx.inputSections.push_back(createCommentSection(ctx));
 
   // Split SHF_MERGE and .eh_frame sections into pieces in preparation for garbage collection.
   splitSections<ELFT>(ctx);
@@ -3157,7 +3159,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // partition.
   copySectionsIntoPartitions(ctx);
 
-  if (canHaveMemtagGlobals()) {
+  if (canHaveMemtagGlobals(ctx)) {
     llvm::TimeTraceScope timeScope("Process memory tagged symbols");
     createTaggedSymbols(ctx);
   }
@@ -3198,7 +3200,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     // sectionBases.
     for (SectionCommand *cmd : ctx.script->sectionCommands)
       if (auto *osd = dyn_cast<OutputDesc>(cmd))
-        osd->osec.finalizeInputSections(ctx);
+        osd->osec.finalizeInputSections();
   }
 
   // Two input sections with different output sections should not be folded.
diff --git a/lld/ELF/DriverUtils.cpp b/lld/ELF/DriverUtils.cpp
index 8b93068f2f9526cd8fccb95183cdbe6ee600fd1e..cd7688ee3f80c03c9e3c173402a3d90c2804b50b 100644
--- a/lld/ELF/DriverUtils.cpp
+++ b/lld/ELF/DriverUtils.cpp
@@ -210,7 +210,7 @@ std::string elf::createResponseFile(const opt::InputArgList &args) {
 
 // Find a file by concatenating given paths. If a resulting path
 // starts with "=", the character is replaced with a --sysroot value.
-static std::optional<std::string> findFile(StringRef path1,
+static std::optional<std::string> findFile(Ctx &ctx, StringRef path1,
                                            const Twine &path2) {
   SmallString<128> s;
   if (path1.starts_with("="))
@@ -225,7 +225,7 @@ static std::optional<std::string> findFile(StringRef path1,
 
 std::optional<std::string> elf::findFromSearchPaths(Ctx &ctx, StringRef path) {
   for (StringRef dir : ctx.arg.searchPaths)
-    if (std::optional<std::string> s = findFile(dir, path))
+    if (std::optional<std::string> s = findFile(ctx, dir, path))
       return s;
   return std::nullopt;
 }
@@ -236,9 +236,10 @@ std::optional<std::string> elf::searchLibraryBaseName(Ctx &ctx,
                                                       StringRef name) {
   for (StringRef dir : ctx.arg.searchPaths) {
     if (!ctx.arg.isStatic)
-      if (std::optional<std::string> s = findFile(dir, "lib" + name + ".so"))
+      if (std::optional<std::string> s =
+              findFile(ctx, dir, "lib" + name + ".so"))
         return s;
-    if (std::optional<std::string> s = findFile(dir, "lib" + name + ".a"))
+    if (std::optional<std::string> s = findFile(ctx, dir, "lib" + name + ".a"))
       return s;
   }
   return std::nullopt;
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 39a78a65bd7cd6186e5fc191dca7fb72d613fa20..3d02ef8b77abaaf32b3ad37e2201af0be74646de 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -649,7 +649,7 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
     // medatada, and we don't want them to end up in the output file for static
     // executables.
     if (sec.sh_type == SHT_AARCH64_MEMTAG_GLOBALS_STATIC &&
-        !canHaveMemtagGlobals()) {
+        !canHaveMemtagGlobals(ctx)) {
       this->sections[i] = &InputSection::discarded;
       continue;
     }
@@ -1156,14 +1156,14 @@ void ObjFile<ELFT>::initializeSymbols(const object::ELFFile<ELFT> &obj) {
         fatal(toString(this) + ": common symbol '" + sym->getName() +
               "' has invalid alignment: " + Twine(value));
       hasCommonSyms = true;
-      sym->resolve(
-          CommonSymbol{this, StringRef(), binding, stOther, type, value, size});
+      sym->resolve(ctx, CommonSymbol{this, StringRef(), binding, stOther, type,
+                                     value, size});
       continue;
     }
 
     // Handle global defined symbols. Defined::section will be set in postParse.
-    sym->resolve(Defined{this, StringRef(), binding, stOther, type, value, size,
-                         nullptr});
+    sym->resolve(ctx, Defined{this, StringRef(), binding, stOther, type, value,
+                              size, nullptr});
   }
 
   // Undefined symbols (excluding those defined relative to non-prevailing
@@ -1175,8 +1175,8 @@ void ObjFile<ELFT>::initializeSymbols(const object::ELFFile<ELFT> &obj) {
   for (unsigned i : undefineds) {
     const Elf_Sym &eSym = eSyms[i];
     Symbol *sym = symbols[i];
-    sym->resolve(Undefined{this, StringRef(), eSym.getBinding(), eSym.st_other,
-                           eSym.getType()});
+    sym->resolve(ctx, Undefined{this, StringRef(), eSym.getBinding(),
+                                eSym.st_other, eSym.getType()});
     sym->isUsedInRegularObj = true;
     sym->referenced = true;
   }
@@ -1324,8 +1324,8 @@ static bool isBitcodeNonCommonDef(MemoryBufferRef mb, StringRef symName,
 }
 
 template <class ELFT>
-static bool isNonCommonDef(ELFKind ekind, MemoryBufferRef mb, StringRef symName,
-                           StringRef archiveName) {
+static bool isNonCommonDef(Ctx &ctx, ELFKind ekind, MemoryBufferRef mb,
+                           StringRef symName, StringRef archiveName) {
   ObjFile<ELFT> *obj = make<ObjFile<ELFT>>(ctx, ekind, mb, archiveName);
   obj->init();
   StringRef stringtable = obj->getStringTable();
@@ -1339,17 +1339,17 @@ static bool isNonCommonDef(ELFKind ekind, MemoryBufferRef mb, StringRef symName,
   return false;
 }
 
-static bool isNonCommonDef(MemoryBufferRef mb, StringRef symName,
+static bool isNonCommonDef(Ctx &ctx, MemoryBufferRef mb, StringRef symName,
                            StringRef archiveName) {
   switch (getELFKind(mb, archiveName)) {
   case ELF32LEKind:
-    return isNonCommonDef<ELF32LE>(ELF32LEKind, mb, symName, archiveName);
+    return isNonCommonDef<ELF32LE>(ctx, ELF32LEKind, mb, symName, archiveName);
   case ELF32BEKind:
-    return isNonCommonDef<ELF32BE>(ELF32BEKind, mb, symName, archiveName);
+    return isNonCommonDef<ELF32BE>(ctx, ELF32BEKind, mb, symName, archiveName);
   case ELF64LEKind:
-    return isNonCommonDef<ELF64LE>(ELF64LEKind, mb, symName, archiveName);
+    return isNonCommonDef<ELF64LE>(ctx, ELF64LEKind, mb, symName, archiveName);
   case ELF64BEKind:
-    return isNonCommonDef<ELF64BE>(ELF64BEKind, mb, symName, archiveName);
+    return isNonCommonDef<ELF64BE>(ctx, ELF64BEKind, mb, symName, archiveName);
   default:
     llvm_unreachable("getELFKind");
   }
@@ -1759,20 +1759,20 @@ static void createBitcodeSymbol(Ctx &ctx, Symbol *&sym,
   int c = objSym.getComdatIndex();
   if (objSym.isUndefined() || (c != -1 && !keptComdats[c])) {
     Undefined newSym(&f, StringRef(), binding, visibility, type);
-    sym->resolve(newSym);
+    sym->resolve(ctx, newSym);
     sym->referenced = true;
     return;
   }
 
   if (objSym.isCommon()) {
-    sym->resolve(CommonSymbol{&f, StringRef(), binding, visibility, STT_OBJECT,
-                              objSym.getCommonAlignment(),
-                              objSym.getCommonSize()});
+    sym->resolve(ctx, CommonSymbol{&f, StringRef(), binding, visibility,
+                                   STT_OBJECT, objSym.getCommonAlignment(),
+                                   objSym.getCommonSize()});
   } else {
     Defined newSym(&f, StringRef(), binding, visibility, type, 0, 0, nullptr);
     if (objSym.canBeOmittedFromSymbolTable())
       newSym.exportDynamic = false;
-    sym->resolve(newSym);
+    sym->resolve(ctx, newSym);
   }
 }
 
@@ -1813,7 +1813,7 @@ void BitcodeFile::parseLazy() {
     irSym.Name = uniqueSaver().save(irSym.getName());
     if (!irSym.isUndefined()) {
       auto *sym = ctx.symtab->insert(irSym.getName());
-      sym->resolve(LazySymbol{*this});
+      sym->resolve(ctx, LazySymbol{*this});
       symbols[i] = sym;
     }
   }
@@ -1849,15 +1849,15 @@ void BinaryFile::parse() {
 
   llvm::StringSaver &saver = lld::saver();
 
-  ctx.symtab->addAndCheckDuplicate(Defined{this, saver.save(s + "_start"),
-                                           STB_GLOBAL, STV_DEFAULT, STT_OBJECT,
-                                           0, 0, section});
-  ctx.symtab->addAndCheckDuplicate(Defined{this, saver.save(s + "_end"),
-                                           STB_GLOBAL, STV_DEFAULT, STT_OBJECT,
-                                           data.size(), 0, section});
-  ctx.symtab->addAndCheckDuplicate(Defined{this, saver.save(s + "_size"),
-                                           STB_GLOBAL, STV_DEFAULT, STT_OBJECT,
-                                           data.size(), 0, nullptr});
+  ctx.symtab->addAndCheckDuplicate(ctx, Defined{this, saver.save(s + "_start"),
+                                                STB_GLOBAL, STV_DEFAULT,
+                                                STT_OBJECT, 0, 0, section});
+  ctx.symtab->addAndCheckDuplicate(
+      ctx, Defined{this, saver.save(s + "_end"), STB_GLOBAL, STV_DEFAULT,
+                   STT_OBJECT, data.size(), 0, section});
+  ctx.symtab->addAndCheckDuplicate(
+      ctx, Defined{this, saver.save(s + "_size"), STB_GLOBAL, STV_DEFAULT,
+                   STT_OBJECT, data.size(), 0, nullptr});
 }
 
 InputFile *elf::createInternalFile(Ctx &ctx, StringRef name) {
@@ -1906,7 +1906,7 @@ template <class ELFT> void ObjFile<ELFT>::parseLazy() {
     if (eSyms[i].st_shndx == SHN_UNDEF)
       continue;
     symbols[i] = symtab->insert(CHECK(eSyms[i].getName(stringTable), this));
-    symbols[i]->resolve(LazySymbol{*this});
+    symbols[i]->resolve(ctx, LazySymbol{*this});
     if (!lazy)
       break;
   }
@@ -1916,7 +1916,7 @@ bool InputFile::shouldExtractForCommon(StringRef name) const {
   if (isa<BitcodeFile>(this))
     return isBitcodeNonCommonDef(mb, name, archiveName);
 
-  return isNonCommonDef(mb, name, archiveName);
+  return isNonCommonDef(ctx, mb, name, archiveName);
 }
 
 std::string elf::replaceThinLTOSuffix(Ctx &ctx, StringRef path) {
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 082fdb9f5c9ac4cf6534c3135b9a7786afc334ea..2e9e8a7007bbf8d905a9cb8bdb8fd747ab85a79e 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -119,6 +119,7 @@ static void decompressAux(const InputSectionBase &sec, uint8_t *out,
 }
 
 void InputSectionBase::decompress() const {
+  Ctx &ctx = getCtx();
   uint8_t *uncompressedBuf;
   {
     static std::mutex mu;
@@ -298,10 +299,6 @@ std::string InputSectionBase::getLocation(uint64_t offset) const {
   std::string secAndOffset =
       (name + "+0x" + Twine::utohexstr(offset) + ")").str();
 
-  // We don't have file for synthetic sections.
-  if (file == nullptr)
-    return (ctx.arg.outputFile + ":(" + secAndOffset).str();
-
   std::string filename = toString(file);
   if (Defined *d = getEnclosingFunction(offset))
     return filename + ":(function " + toString(*d) + ": " + secAndOffset;
@@ -399,18 +396,20 @@ InputSectionBase *InputSection::getRelocatedSection() const {
 }
 
 template <class ELFT, class RelTy>
-void InputSection::copyRelocations(uint8_t *buf) {
+void InputSection::copyRelocations(Ctx &ctx, uint8_t *buf) {
   if (ctx.arg.relax && !ctx.arg.relocatable &&
       (ctx.arg.emachine == EM_RISCV || ctx.arg.emachine == EM_LOONGARCH)) {
     // On LoongArch and RISC-V, relaxation might change relocations: copy
     // from internal ones that are updated by relaxation.
     InputSectionBase *sec = getRelocatedSection();
-    copyRelocations<ELFT, RelTy>(buf, llvm::make_range(sec->relocations.begin(),
-                                                       sec->relocations.end()));
+    copyRelocations<ELFT, RelTy>(
+        ctx, buf,
+        llvm::make_range(sec->relocations.begin(), sec->relocations.end()));
   } else {
     // Convert the raw relocations in the input section into Relocation objects
     // suitable to be used by copyRelocations below.
     struct MapRel {
+      Ctx &ctx;
       const ObjFile<ELFT> &file;
       Relocation operator()(const RelTy &rel) const {
         // RelExpr is not used so set to a dummy value.
@@ -422,11 +421,11 @@ void InputSection::copyRelocations(uint8_t *buf) {
     using RawRels = ArrayRef<RelTy>;
     using MapRelIter =
         llvm::mapped_iterator<typename RawRels::iterator, MapRel>;
-    auto mapRel = MapRel{*getFile<ELFT>()};
+    auto mapRel = MapRel{ctx, *getFile<ELFT>()};
     RawRels rawRels = getDataAs<RelTy>();
     auto rels = llvm::make_range(MapRelIter(rawRels.begin(), mapRel),
                                  MapRelIter(rawRels.end(), mapRel));
-    copyRelocations<ELFT, RelTy>(buf, rels);
+    copyRelocations<ELFT, RelTy>(ctx, buf, rels);
   }
 }
 
@@ -434,9 +433,9 @@ void InputSection::copyRelocations(uint8_t *buf) {
 // relocations because we need to update symbol table offset and section index
 // for each relocation. So we copy relocations one by one.
 template <class ELFT, class RelTy, class RelIt>
-void InputSection::copyRelocations(uint8_t *buf,
+void InputSection::copyRelocations(Ctx &ctx, uint8_t *buf,
                                    llvm::iterator_range<RelIt> rels) {
-  const TargetInfo &target = *elf::ctx.target;
+  const TargetInfo &target = *ctx.target;
   InputSectionBase *sec = getRelocatedSection();
   (void)sec->contentMaybeDecompress(); // uncompress if needed
 
@@ -971,9 +970,10 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
 // So, we handle relocations for non-alloc sections directly in this
 // function as a performance optimization.
 template <class ELFT, class RelTy>
-void InputSection::relocateNonAlloc(uint8_t *buf, Relocs<RelTy> rels) {
+void InputSection::relocateNonAlloc(Ctx &ctx, uint8_t *buf,
+                                    Relocs<RelTy> rels) {
   const unsigned bits = sizeof(typename ELFT::uint) * 8;
-  const TargetInfo &target = *elf::ctx.target;
+  const TargetInfo &target = *ctx.target;
   const auto emachine = ctx.arg.emachine;
   const bool isDebug = isDebugSection(*this);
   const bool isDebugLine = isDebug && name == ".debug_line";
@@ -1121,9 +1121,9 @@ void InputSection::relocateNonAlloc(uint8_t *buf, Relocs<RelTy> rels) {
 }
 
 template <class ELFT>
-void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) {
+void InputSectionBase::relocate(Ctx &ctx, uint8_t *buf, uint8_t *bufEnd) {
   if ((flags & SHF_EXECINSTR) && LLVM_UNLIKELY(getFile<ELFT>()->splitStack))
-    adjustSplitStackFunctionPrologues<ELFT>(buf, bufEnd);
+    adjustSplitStackFunctionPrologues<ELFT>(ctx, buf, bufEnd);
 
   if (flags & SHF_ALLOC) {
     ctx.target->relocateAlloc(*this, buf);
@@ -1133,13 +1133,13 @@ void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) {
   auto *sec = cast<InputSection>(this);
   // For a relocatable link, also call relocateNonAlloc() to rewrite applicable
   // locations with tombstone values.
-  invokeOnRelocs(*sec, sec->relocateNonAlloc<ELFT>, buf);
+  invokeOnRelocs(*sec, sec->relocateNonAlloc<ELFT>, ctx, buf);
 }
 
 // For each function-defining prologue, find any calls to __morestack,
 // and replace them with calls to __morestack_non_split.
 static void switchMorestackCallsToMorestackNonSplit(
-    DenseSet<Defined *> &prologues,
+    Ctx &ctx, DenseSet<Defined *> &prologues,
     SmallVector<Relocation *, 0> &morestackCalls) {
 
   // If the target adjusted a function's prologue, all calls to
@@ -1187,7 +1187,7 @@ static bool enclosingPrologueAttempted(uint64_t offset,
 // adjusted to ensure that the called function will have enough stack
 // available. Find those functions, and adjust their prologues.
 template <class ELFT>
-void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf,
+void InputSectionBase::adjustSplitStackFunctionPrologues(Ctx &ctx, uint8_t *buf,
                                                          uint8_t *end) {
   DenseSet<Defined *> prologues;
   SmallVector<Relocation *, 0> morestackCalls;
@@ -1232,20 +1232,20 @@ void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf,
   }
 
   if (ctx.target->needsMoreStackNonSplit)
-    switchMorestackCallsToMorestackNonSplit(prologues, morestackCalls);
+    switchMorestackCallsToMorestackNonSplit(ctx, prologues, morestackCalls);
 }
 
-template <class ELFT> void InputSection::writeTo(uint8_t *buf) {
+template <class ELFT> void InputSection::writeTo(Ctx &ctx, uint8_t *buf) {
   if (LLVM_UNLIKELY(type == SHT_NOBITS))
     return;
   // If -r or --emit-relocs is given, then an InputSection
   // may be a relocation section.
   if (LLVM_UNLIKELY(type == SHT_RELA)) {
-    copyRelocations<ELFT, typename ELFT::Rela>(buf);
+    copyRelocations<ELFT, typename ELFT::Rela>(ctx, buf);
     return;
   }
   if (LLVM_UNLIKELY(type == SHT_REL)) {
-    copyRelocations<ELFT, typename ELFT::Rel>(buf);
+    copyRelocations<ELFT, typename ELFT::Rel>(ctx, buf);
     return;
   }
 
@@ -1268,14 +1268,14 @@ template <class ELFT> void InputSection::writeTo(uint8_t *buf) {
       fatal(toString(this) +
             ": decompress failed: " + llvm::toString(std::move(e)));
     uint8_t *bufEnd = buf + size;
-    relocate<ELFT>(buf, bufEnd);
+    relocate<ELFT>(ctx, buf, bufEnd);
     return;
   }
 
   // Copy section contents from source object file to output file
   // and then apply relocations.
   memcpy(buf, content().data(), content().size());
-  relocate<ELFT>(buf, buf + content().size());
+  relocate<ELFT>(ctx, buf, buf + content().size());
 }
 
 void InputSection::replace(InputSection *other) {
@@ -1387,7 +1387,7 @@ static size_t findNull(StringRef s, size_t entSize) {
 // Split SHF_STRINGS section. Such section is a sequence of
 // null-terminated strings.
 void MergeInputSection::splitStrings(StringRef s, size_t entSize) {
-  const bool live = !(flags & SHF_ALLOC) || !ctx.arg.gcSections;
+  const bool live = !(flags & SHF_ALLOC) || !getCtx().arg.gcSections;
   const char *p = s.data(), *end = s.data() + s.size();
   if (!std::all_of(end - entSize, end, [](char c) { return c == 0; }))
     fatal(toString(this) + ": string is not null terminated");
@@ -1413,7 +1413,7 @@ void MergeInputSection::splitNonStrings(ArrayRef<uint8_t> data,
                                         size_t entSize) {
   size_t size = data.size();
   assert((size % entSize) == 0);
-  const bool live = !(flags & SHF_ALLOC) || !ctx.arg.gcSections;
+  const bool live = !(flags & SHF_ALLOC) || !getCtx().arg.gcSections;
 
   pieces.resize_for_overwrite(size / entSize);
   for (size_t i = 0, j = 0; i != size; i += entSize, j++)
@@ -1426,11 +1426,12 @@ MergeInputSection::MergeInputSection(ObjFile<ELFT> &f,
                                      StringRef name)
     : InputSectionBase(f, header, name, InputSectionBase::Merge) {}
 
-MergeInputSection::MergeInputSection(uint64_t flags, uint32_t type,
+MergeInputSection::MergeInputSection(Ctx &ctx, uint64_t flags, uint32_t type,
                                      uint64_t entsize, ArrayRef<uint8_t> data,
                                      StringRef name)
-    : InputSectionBase(nullptr, flags, type, entsize, /*Link*/ 0, /*Info*/ 0,
-                       /*Alignment*/ entsize, data, name, SectionBase::Merge) {}
+    : InputSectionBase(ctx.internalFile, flags, type, entsize, /*link=*/0,
+                       /*info=*/0,
+                       /*alignment=*/entsize, data, name, SectionBase::Merge) {}
 
 // This function is called after we obtain a complete list of input sections
 // that need to be linked. This is responsible to split section contents
@@ -1469,10 +1470,10 @@ template InputSection::InputSection(ObjFile<ELF64LE> &, const ELF64LE::Shdr &,
 template InputSection::InputSection(ObjFile<ELF64BE> &, const ELF64BE::Shdr &,
                                     StringRef);
 
-template void InputSection::writeTo<ELF32LE>(uint8_t *);
-template void InputSection::writeTo<ELF32BE>(uint8_t *);
-template void InputSection::writeTo<ELF64LE>(uint8_t *);
-template void InputSection::writeTo<ELF64BE>(uint8_t *);
+template void InputSection::writeTo<ELF32LE>(Ctx &, uint8_t *);
+template void InputSection::writeTo<ELF32BE>(Ctx &, uint8_t *);
+template void InputSection::writeTo<ELF64LE>(Ctx &, uint8_t *);
+template void InputSection::writeTo<ELF64BE>(Ctx &, uint8_t *);
 
 template RelsOrRelas<ELF32LE>
 InputSectionBase::relsOrRelas<ELF32LE>(bool) const;
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 2b34047bc0682a7e54e68955714d66ebb98350d5..1a5bc629d8b093adeabf0941cf5f4dd398e017e9 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -245,7 +245,7 @@ public:
   // Each section knows how to relocate itself. These functions apply
   // relocations, assuming that Buf points to this section's copy in
   // the mmap'ed output buffer.
-  template <class ELFT> void relocate(uint8_t *buf, uint8_t *bufEnd);
+  template <class ELFT> void relocate(Ctx &, uint8_t *buf, uint8_t *bufEnd);
   uint64_t getRelocTargetVA(Ctx &, const Relocation &r, uint64_t p) const;
 
   // The native ELF reloc data type is not very convenient to handle.
@@ -278,8 +278,7 @@ public:
   // to relocation. See https://gcc.gnu.org/wiki/SplitStacks for more
   // information.
   template <typename ELFT>
-  void adjustSplitStackFunctionPrologues(uint8_t *buf, uint8_t *end);
-
+  void adjustSplitStackFunctionPrologues(Ctx &, uint8_t *buf, uint8_t *end);
 
   template <typename T> llvm::ArrayRef<T> getDataAs() const {
     size_t s = content().size();
@@ -317,7 +316,7 @@ public:
   template <class ELFT>
   MergeInputSection(ObjFile<ELFT> &f, const typename ELFT::Shdr &header,
                     StringRef name);
-  MergeInputSection(uint64_t flags, uint32_t type, uint64_t entsize,
+  MergeInputSection(Ctx &, uint64_t flags, uint32_t type, uint64_t entsize,
                     ArrayRef<uint8_t> data, StringRef name);
 
   static bool classof(const SectionBase *s) { return s->kind() == Merge; }
@@ -410,7 +409,7 @@ public:
 
   // Write this section to a mmap'ed file, assuming Buf is pointing to
   // beginning of the output section.
-  template <class ELFT> void writeTo(uint8_t *buf);
+  template <class ELFT> void writeTo(Ctx &, uint8_t *buf);
 
   OutputSection *getParent() const {
     return reinterpret_cast<OutputSection *>(parent);
@@ -425,7 +424,7 @@ public:
   InputSectionBase *getRelocatedSection() const;
 
   template <class ELFT, class RelTy>
-  void relocateNonAlloc(uint8_t *buf, Relocs<RelTy> rels);
+  void relocateNonAlloc(Ctx &, uint8_t *buf, Relocs<RelTy> rels);
 
   // Points to the canonical section. If ICF folds two sections, repl pointer of
   // one section points to the other.
@@ -440,10 +439,10 @@ public:
   static InputSection discarded;
 
 private:
-  template <class ELFT, class RelTy> void copyRelocations(uint8_t *buf);
+  template <class ELFT, class RelTy> void copyRelocations(Ctx &, uint8_t *buf);
 
   template <class ELFT, class RelTy, class RelIt>
-  void copyRelocations(uint8_t *buf, llvm::iterator_range<RelIt> rels);
+  void copyRelocations(Ctx &, uint8_t *buf, llvm::iterator_range<RelIt> rels);
 
   template <class ELFT> void copyShtGroup(uint8_t *buf);
 };
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 627cbbdce83f37c7cd2209c3443bce141c218763..aef058286cea0d6c438e3d11f8b8155a8f4d70d9 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -249,12 +249,12 @@ void BitcodeCompiler::add(BitcodeFile &f) {
     // 5) Symbols that will be referenced after linker wrapping is performed.
     r.VisibleToRegularObj = ctx.arg.relocatable || sym->isUsedInRegularObj ||
                             sym->referencedAfterWrap ||
-                            (r.Prevailing && sym->includeInDynsym()) ||
+                            (r.Prevailing && sym->includeInDynsym(ctx)) ||
                             usedStartStop.count(objSym.getSectionName());
     // Identify symbols exported dynamically, and that therefore could be
     // referenced by a shared library not visible to the linker.
     r.ExportDynamic =
-        sym->computeBinding() != STB_LOCAL &&
+        sym->computeBinding(ctx) != STB_LOCAL &&
         (ctx.arg.exportDynamic || sym->exportDynamic || sym->inDynamicList);
     const auto *dr = dyn_cast<Defined>(sym);
     r.FinalDefinitionInLinkageUnit =
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index b736cb1beef37e22bef04e95897f4c3d214d4c2a..418f689cee2167079f1381191733be694bc14c62 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -140,7 +140,7 @@ OutputDesc *LinkerScript::createOutputSection(StringRef name,
     // There was a forward reference.
     sec = secRef;
   } else {
-    sec = make<OutputDesc>(name, SHT_PROGBITS, 0);
+    sec = make<OutputDesc>(ctx, name, SHT_PROGBITS, 0);
     if (!secRef)
       secRef = sec;
   }
@@ -151,7 +151,7 @@ OutputDesc *LinkerScript::createOutputSection(StringRef name,
 OutputDesc *LinkerScript::getOrCreateOutputSection(StringRef name) {
   OutputDesc *&cmdRef = nameToOutputSection[CachedHashStringRef(name)];
   if (!cmdRef)
-    cmdRef = make<OutputDesc>(name, SHT_PROGBITS, 0);
+    cmdRef = make<OutputDesc>(ctx, name, SHT_PROGBITS, 0);
   return cmdRef;
 }
 
@@ -196,17 +196,17 @@ void LinkerScript::setDot(Expr e, const Twine &loc, bool inSec) {
 // Used for handling linker symbol assignments, for both finalizing
 // their values and doing early declarations. Returns true if symbol
 // should be defined from linker script.
-static bool shouldDefineSym(SymbolAssignment *cmd) {
+static bool shouldDefineSym(Ctx &ctx, SymbolAssignment *cmd) {
   if (cmd->name == ".")
     return false;
 
-  return !cmd->provide || LinkerScript::shouldAddProvideSym(cmd->name);
+  return !cmd->provide || LinkerScript::shouldAddProvideSym(ctx, cmd->name);
 }
 
 // Called by processSymbolAssignments() to assign definitions to
 // linker-script-defined symbols.
 void LinkerScript::addSymbol(SymbolAssignment *cmd) {
-  if (!shouldDefineSym(cmd))
+  if (!shouldDefineSym(ctx, cmd))
     return;
 
   // Define a symbol.
@@ -240,7 +240,7 @@ void LinkerScript::addSymbol(SymbolAssignment *cmd) {
 // This function is called from LinkerScript::declareSymbols.
 // It creates a placeholder symbol if needed.
 void LinkerScript::declareSymbol(SymbolAssignment *cmd) {
-  if (!shouldDefineSym(cmd))
+  if (!shouldDefineSym(ctx, cmd))
     return;
 
   uint8_t visibility = cmd->hidden ? STV_HIDDEN : STV_DEFAULT;
@@ -830,7 +830,7 @@ void LinkerScript::processSymbolAssignments() {
   // sh_shndx should not be SHN_UNDEF or SHN_ABS. Create a dummy aether section
   // that fills the void outside a section. It has an index of one, which is
   // indistinguishable from any other regular section index.
-  aether = make<OutputSection>("", 0, SHF_ALLOC);
+  aether = make<OutputSection>(ctx, "", 0, SHF_ALLOC);
   aether->sectionIndex = 1;
 
   // `st` captures the local AddressState and makes it accessible deliberately.
@@ -861,7 +861,8 @@ static OutputSection *findByName(ArrayRef<SectionCommand *> vec,
   return nullptr;
 }
 
-static OutputDesc *createSection(InputSectionBase *isec, StringRef outsecName) {
+static OutputDesc *createSection(Ctx &ctx, InputSectionBase *isec,
+                                 StringRef outsecName) {
   OutputDesc *osd = ctx.script->createOutputSection(outsecName, "<internal>");
   osd->osec.recordSection(isec);
   return osd;
@@ -878,7 +879,7 @@ static OutputDesc *addInputSec(Ctx &ctx,
   // as-is because adding/removing members or merging them with other groups
   // change their semantics.
   if (isec->type == SHT_GROUP || (isec->flags & SHF_GROUP))
-    return createSection(isec, outsecName);
+    return createSection(ctx, isec, outsecName);
 
   // Imagine .zed : { *(.foo) *(.bar) } script. Both foo and bar may have
   // relocation sections .rela.foo and .rela.bar for example. Most tools do
@@ -895,7 +896,7 @@ static OutputDesc *addInputSec(Ctx &ctx,
       return nullptr;
     }
 
-    OutputDesc *osd = createSection(isec, outsecName);
+    OutputDesc *osd = createSection(ctx, isec, outsecName);
     out->relocationSection = &osd->osec;
     return osd;
   }
@@ -966,7 +967,7 @@ static OutputDesc *addInputSec(Ctx &ctx,
     return nullptr;
   }
 
-  OutputDesc *osd = createSection(isec, outsecName);
+  OutputDesc *osd = createSection(ctx, isec, outsecName);
   v.push_back(&osd->osec);
   return osd;
 }
@@ -982,7 +983,7 @@ void LinkerScript::addOrphanSections() {
 
       StringRef name = getOutputSectionName(s);
       if (ctx.arg.unique) {
-        v.push_back(createSection(s, name));
+        v.push_back(createSection(ctx, s, name));
       } else if (OutputSection *sec = findByName(sectionCommands, name)) {
         sec->recordSection(s);
       } else {
@@ -1797,7 +1798,7 @@ void LinkerScript::addScriptReferencedSymbolsToSymTable() {
   DenseSet<StringRef> added;
   SmallVector<const SmallVector<StringRef, 0> *, 0> symRefsVec;
   for (const auto &[name, symRefs] : provideMap)
-    if (LinkerScript::shouldAddProvideSym(name) && added.insert(name).second)
+    if (shouldAddProvideSym(ctx, name) && added.insert(name).second)
       symRefsVec.push_back(&symRefs);
   while (symRefsVec.size()) {
     for (StringRef name : *symRefsVec.pop_back_val()) {
@@ -1805,7 +1806,7 @@ void LinkerScript::addScriptReferencedSymbolsToSymTable() {
       // Prevent the symbol from being discarded by --gc-sections.
       referencedSymbols.push_back(name);
       auto it = provideMap.find(name);
-      if (it != provideMap.end() && shouldAddProvideSym(name) &&
+      if (it != provideMap.end() && shouldAddProvideSym(ctx, name) &&
           added.insert(name).second) {
         symRefsVec.push_back(&it->second);
       }
@@ -1813,7 +1814,14 @@ void LinkerScript::addScriptReferencedSymbolsToSymTable() {
   }
 }
 
-bool LinkerScript::shouldAddProvideSym(StringRef symName) {
-  Symbol *sym = elf::ctx.symtab->find(symName);
-  return sym && !sym->isDefined() && !sym->isCommon();
+bool LinkerScript::shouldAddProvideSym(Ctx &ctx, StringRef symName) {
+  // This function is called before and after garbage collection. To prevent
+  // undefined references from the RHS, the result of this function for a
+  // symbol must be the same for each call. We use isUsedInRegularObj to not
+  // change the return value of a demoted symbol. The exportDynamic condition,
+  // while not so accurate, allows PROVIDE to define a symbol referenced by a
+  // DSO.
+  Symbol *sym = ctx.symtab->find(symName);
+  return sym && !sym->isDefined() && !sym->isCommon() &&
+         (sym->isUsedInRegularObj || sym->exportDynamic);
 }
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index 4aa030a37fc0f4ec35e5fe8708755771e9eef02e..eda152dd3582ab5c14e1bca11e286ef8ecff6817 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -389,7 +389,7 @@ public:
   // Returns true if the PROVIDE symbol should be added to the link.
   // A PROVIDE symbol is added to the link only if it satisfies an
   // undefined reference.
-  static bool shouldAddProvideSym(StringRef symName);
+  static bool shouldAddProvideSym(Ctx &, StringRef symName);
 
   // SECTIONS command list.
   SmallVector<SectionCommand *, 0> sectionCommands;
diff --git a/lld/ELF/MapFile.cpp b/lld/ELF/MapFile.cpp
index 3495cdb0bc66633f6cc8490b9bb994107af7dec2..6bbc1ecc646fdd74c95d17cd6c58781adb39c8bd 100644
--- a/lld/ELF/MapFile.cpp
+++ b/lld/ELF/MapFile.cpp
@@ -53,7 +53,7 @@ static void writeHeader(Ctx &ctx, raw_ostream &os, uint64_t vma, uint64_t lma,
 }
 
 // Returns a list of all symbols that we want to print out.
-static std::vector<Defined *> getSymbols() {
+static std::vector<Defined *> getSymbols(Ctx &ctx) {
   std::vector<Defined *> v;
   for (ELFFileBase *file : ctx.objectFiles)
     for (Symbol *b : file->getSymbols())
@@ -148,7 +148,7 @@ static void printEhFrame(Ctx &ctx, raw_ostream &os, const EhFrameSection *sec) {
 
 static void writeMapFile(Ctx &ctx, raw_fd_ostream &os) {
   // Collect symbol info that we want to print out.
-  std::vector<Defined *> syms = getSymbols();
+  std::vector<Defined *> syms = getSymbols(ctx);
   SymbolMapTy sectionSyms = getSectionSyms(syms);
   DenseMap<Symbol *, std::string> symStr = getSymbolStrings(ctx, syms);
 
diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp
index 23e23594913701795da78bd63d69a332f7a3e289..9b6aa1982b577e5f2ea5f400b9e6aac020603b9a 100644
--- a/lld/ELF/MarkLive.cpp
+++ b/lld/ELF/MarkLive.cpp
@@ -221,7 +221,7 @@ template <class ELFT> void MarkLive<ELFT>::run() {
   // Preserve externally-visible symbols if the symbols defined by this
   // file can interpose other ELF file's symbols at runtime.
   for (Symbol *sym : ctx.symtab->getSymbols())
-    if (sym->includeInDynsym() && sym->partition == partition)
+    if (sym->includeInDynsym(ctx) && sym->partition == partition)
       markSymbol(sym);
 
   // If this isn't the main partition, that's all that we need to preserve.
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 7a65858a6f8c184964760ce759ec4519cd8cac46..309039fe7e204a6e90d896b3b822a207a64d0fa1 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -65,10 +65,12 @@ void OutputSection::writeHeaderTo(typename ELFT::Shdr *shdr) {
   shdr->sh_name = shName;
 }
 
-OutputSection::OutputSection(StringRef name, uint32_t type, uint64_t flags)
+OutputSection::OutputSection(Ctx &ctx, StringRef name, uint32_t type,
+                             uint64_t flags)
     : SectionBase(Output, ctx.internalFile, name, flags, /*entsize=*/0,
                   /*addralign=*/1, type,
-                  /*info=*/0, /*link=*/0) {}
+                  /*info=*/0, /*link=*/0),
+      ctx(ctx) {}
 
 // We allow sections of types listed below to merged into a
 // single progbits section. This is typically done by linker
@@ -106,7 +108,7 @@ void OutputSection::recordSection(InputSectionBase *isec) {
 // Update fields (type, flags, alignment, etc) according to the InputSection
 // isec. Also check whether the InputSection flags and type are consistent with
 // other InputSections.
-void OutputSection::commitSection(Ctx &ctx, InputSection *isec) {
+void OutputSection::commitSection(InputSection *isec) {
   if (LLVM_UNLIKELY(type != isec->type)) {
     if (!hasInputSections && !typeIsSet) {
       type = isec->type;
@@ -189,7 +191,7 @@ static MergeSyntheticSection *createMergeSynthetic(Ctx &ctx, StringRef name,
 // new synthetic sections at the location of the first input section
 // that it replaces. It then finalizes each synthetic section in order
 // to compute an output offset for each piece of each input section.
-void OutputSection::finalizeInputSections(Ctx &ctx) {
+void OutputSection::finalizeInputSections() {
   auto *script = ctx.script;
   std::vector<MergeSyntheticSection *> mergeSections;
   for (SectionCommand *cmd : commands) {
@@ -245,7 +247,7 @@ void OutputSection::finalizeInputSections(Ctx &ctx) {
 
     // Some input sections may be removed from the list after ICF.
     for (InputSection *s : isd->sections)
-      commitSection(ctx, s);
+      commitSection(s);
   }
   for (auto *ms : mergeSections)
     ms->finalizeContents();
@@ -456,15 +458,15 @@ template <class ELFT> void OutputSection::maybeCompress(Ctx &ctx) {
   flags |= SHF_COMPRESSED;
 }
 
-static void writeInt(uint8_t *buf, uint64_t data, uint64_t size) {
+static void writeInt(Ctx &ctx, uint8_t *buf, uint64_t data, uint64_t size) {
   if (size == 1)
     *buf = data;
   else if (size == 2)
-    write16(buf, data);
+    write16(ctx, buf, data);
   else if (size == 4)
-    write32(buf, data);
+    write32(ctx, buf, data);
   else if (size == 8)
-    write64(buf, data);
+    write64(ctx, buf, data);
   else
     llvm_unreachable("unsupported Size argument");
 }
@@ -510,7 +512,7 @@ void OutputSection::writeTo(Ctx &ctx, uint8_t *buf, parallel::TaskGroup &tg) {
   // Write leading padding.
   ArrayRef<InputSection *> sections = getInputSections(*this, storage);
   std::array<uint8_t, 4> filler = getFiller(ctx);
-  bool nonZeroFiller = read32(filler.data()) != 0;
+  bool nonZeroFiller = read32(ctx, filler.data()) != 0;
   if (nonZeroFiller)
     fill(buf, sections.empty() ? size : sections[0]->outSecOff, filler);
 
@@ -527,7 +529,7 @@ void OutputSection::writeTo(Ctx &ctx, uint8_t *buf, parallel::TaskGroup &tg) {
       if (auto *s = dyn_cast<SyntheticSection>(isec))
         s->writeTo(buf + isec->outSecOff);
       else
-        isec->writeTo<ELFT>(buf + isec->outSecOff);
+        isec->writeTo<ELFT>(ctx, buf + isec->outSecOff);
 
       // When in Arm BE8 mode, the linker has to convert the big-endian
       // instructions to little-endian, leaving the data big-endian.
@@ -561,7 +563,8 @@ void OutputSection::writeTo(Ctx &ctx, uint8_t *buf, parallel::TaskGroup &tg) {
     if (auto *data = dyn_cast<ByteCommand>(cmd)) {
       if (!std::exchange(written, true))
         fn(0, numSections);
-      writeInt(buf + data->offset, data->expression().getValue(), data->size);
+      writeInt(ctx, buf + data->offset, data->expression().getValue(),
+               data->size);
     }
   if (written || !numSections)
     return;
@@ -603,15 +606,16 @@ static void finalizeShtGroup(Ctx &ctx, OutputSection *os,
   DenseSet<uint32_t> seen;
   ArrayRef<InputSectionBase *> sections = section->file->getSections();
   for (const uint32_t &idx : section->getDataAs<uint32_t>().slice(1))
-    if (OutputSection *osec = sections[read32(&idx)]->getOutputSection())
+    if (OutputSection *osec = sections[read32(ctx, &idx)]->getOutputSection())
       seen.insert(osec->sectionIndex);
   os->size = (1 + seen.size()) * sizeof(uint32_t);
 }
 
 template <class uint>
 LLVM_ATTRIBUTE_ALWAYS_INLINE static void
-encodeOneCrel(raw_svector_ostream &os, Elf_Crel<sizeof(uint) == 8> &out,
-              uint offset, const Symbol &sym, uint32_t type, uint addend) {
+encodeOneCrel(Ctx &ctx, raw_svector_ostream &os,
+              Elf_Crel<sizeof(uint) == 8> &out, uint offset, const Symbol &sym,
+              uint32_t type, uint addend) {
   const auto deltaOffset = static_cast<uint64_t>(offset - out.r_offset);
   out.r_offset = offset;
   int64_t symidx = ctx.in.symTab->getSymbolIndex(sym);
@@ -652,8 +656,9 @@ encodeOneCrel(raw_svector_ostream &os, Elf_Crel<sizeof(uint) == 8> &out,
 }
 
 template <class ELFT>
-static size_t relToCrel(raw_svector_ostream &os, Elf_Crel<ELFT::Is64Bits> &out,
-                        InputSection *relSec, InputSectionBase *sec) {
+static size_t relToCrel(Ctx &ctx, raw_svector_ostream &os,
+                        Elf_Crel<ELFT::Is64Bits> &out, InputSection *relSec,
+                        InputSectionBase *sec) {
   const auto &file = *cast<ELFFileBase>(relSec->file);
   if (relSec->type == SHT_REL) {
     // REL conversion is complex and unsupported yet.
@@ -663,7 +668,7 @@ static size_t relToCrel(raw_svector_ostream &os, Elf_Crel<ELFT::Is64Bits> &out,
   auto rels = relSec->getDataAs<typename ELFT::Rela>();
   for (auto rel : rels) {
     encodeOneCrel<typename ELFT::uint>(
-        os, out, sec->getVA(rel.r_offset), file.getRelocTargetSym(rel),
+        ctx, os, out, sec->getVA(rel.r_offset), file.getRelocTargetSym(rel),
         rel.getType(ctx.arg.isMips64EL), getAddend<ELFT>(rel));
   }
   return rels.size();
@@ -685,7 +690,7 @@ template <bool is64> void OutputSection::finalizeNonAllocCrel(Ctx &ctx) {
       RelocsCrel<is64> entries(relSec->content_);
       totalCount += entries.size();
       for (Elf_Crel_Impl<is64> r : entries) {
-        encodeOneCrel<uint>(os, out, uint(sec->getVA(r.r_offset)),
+        encodeOneCrel<uint>(ctx, os, out, uint(sec->getVA(r.r_offset)),
                             file.getSymbol(r.r_symidx), r.r_type, r.r_addend);
       }
       continue;
@@ -693,11 +698,13 @@ template <bool is64> void OutputSection::finalizeNonAllocCrel(Ctx &ctx) {
 
     // Convert REL[A] to CREL.
     if constexpr (is64) {
-      totalCount += ctx.arg.isLE ? relToCrel<ELF64LE>(os, out, relSec, sec)
-                                 : relToCrel<ELF64BE>(os, out, relSec, sec);
+      totalCount += ctx.arg.isLE
+                        ? relToCrel<ELF64LE>(ctx, os, out, relSec, sec)
+                        : relToCrel<ELF64BE>(ctx, os, out, relSec, sec);
     } else {
-      totalCount += ctx.arg.isLE ? relToCrel<ELF32LE>(os, out, relSec, sec)
-                                 : relToCrel<ELF32BE>(os, out, relSec, sec);
+      totalCount += ctx.arg.isLE
+                        ? relToCrel<ELF32LE>(ctx, os, out, relSec, sec)
+                        : relToCrel<ELF32BE>(ctx, os, out, relSec, sec);
     }
   }
 
diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h
index 11977507e9268e6bd09d1c961c0be82cbe9f1eca..f8509d5a7aaaba29b3f6833f72c1fbc32226a9a6 100644
--- a/lld/ELF/OutputSections.h
+++ b/lld/ELF/OutputSections.h
@@ -35,7 +35,7 @@ struct CompressedData {
 // non-overlapping file offsets and VAs.
 class OutputSection final : public SectionBase {
 public:
-  OutputSection(StringRef name, uint32_t type, uint64_t flags);
+  OutputSection(Ctx &, StringRef name, uint32_t type, uint64_t flags);
 
   static bool classof(const SectionBase *s) {
     return s->kind() == SectionBase::Output;
@@ -44,6 +44,7 @@ public:
   uint64_t getLMA() const { return ptLoad ? addr + ptLoad->lmaOffset : addr; }
   template <typename ELFT> void writeHeaderTo(typename ELFT::Shdr *sHdr);
 
+  Ctx &ctx;
   uint32_t sectionIndex = UINT32_MAX;
   unsigned sortRank;
 
@@ -74,8 +75,8 @@ public:
   uint32_t shName = 0;
 
   void recordSection(InputSectionBase *isec);
-  void commitSection(Ctx &ctx, InputSection *isec);
-  void finalizeInputSections(Ctx &ctx);
+  void commitSection(InputSection *isec);
+  void finalizeInputSections();
 
   // The following members are normally only used in linker scripts.
   MemoryRegion *memRegion = nullptr;
@@ -135,8 +136,8 @@ private:
 
 struct OutputDesc final : SectionCommand {
   OutputSection osec;
-  OutputDesc(StringRef name, uint32_t type, uint64_t flags)
-      : SectionCommand(OutputSectionKind), osec(name, type, flags) {}
+  OutputDesc(Ctx &ctx, StringRef name, uint32_t type, uint64_t flags)
+      : SectionCommand(OutputSectionKind), osec(ctx, name, type, flags) {}
 
   static bool classof(const SectionCommand *c) {
     return c->kind == OutputSectionKind;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index cb33f35e59e43b14cf0f59a74ce84721edee3923..73c19c8385a824d5fbfbae120a52904236c1b36e 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -392,7 +392,7 @@ template <class ELFT> static void addCopyRelSymbol(Ctx &ctx, SharedSymbol &ss) {
     osec->commands.push_back(make<InputSectionDescription>(""));
   auto *isd = cast<InputSectionDescription>(osec->commands.back());
   isd->sections.push_back(sec);
-  osec->commitSection(ctx, sec);
+  osec->commitSection(sec);
 
   // Look through the DSO's dynamic symbol table for aliases and create a
   // dynamic symbol for each one. This causes the copy relocation to correctly
@@ -917,7 +917,7 @@ static void addRelativeReloc(Ctx &ctx, InputSectionBase &isec,
 }
 
 template <class PltSection, class GotPltSection>
-static void addPltEntry(PltSection &plt, GotPltSection &gotPlt,
+static void addPltEntry(Ctx &ctx, PltSection &plt, GotPltSection &gotPlt,
                         RelocationBaseSection &rel, RelType type, Symbol &sym) {
   plt.addEntry(sym);
   gotPlt.addEntry(sym);
@@ -1756,7 +1756,7 @@ static bool handleNonPreemptibleIfunc(Ctx &ctx, Symbol &sym, uint16_t flags) {
   directSym->allocateAux(ctx);
   auto &dyn =
       ctx.arg.androidPackDynRelocs ? *ctx.in.relaPlt : *ctx.mainPart->relaDyn;
-  addPltEntry(*ctx.in.iplt, *ctx.in.igotPlt, dyn, ctx.target->iRelativeRel,
+  addPltEntry(ctx, *ctx.in.iplt, *ctx.in.igotPlt, dyn, ctx.target->iRelativeRel,
               *directSym);
   sym.allocateAux(ctx);
   ctx.symAux.back().pltIdx = ctx.symAux[directSym->auxIdx].pltIdx;
@@ -1796,7 +1796,7 @@ void elf::postScanRelocations(Ctx &ctx) {
     if (flags & NEEDS_GOT)
       addGotEntry(ctx, sym);
     if (flags & NEEDS_PLT)
-      addPltEntry(*ctx.in.plt, *ctx.in.gotPlt, *ctx.in.relaPlt,
+      addPltEntry(ctx, *ctx.in.plt, *ctx.in.gotPlt, *ctx.in.relaPlt,
                   ctx.target->pltRel, sym);
     if (flags & NEEDS_COPY) {
       if (sym.isObject()) {
@@ -2427,7 +2427,7 @@ void elf::hexagonTLSSymbolUpdate(Ctx &ctx) {
             if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) {
               if (needEntry) {
                 sym->allocateAux(ctx);
-                addPltEntry(*ctx.in.plt, *ctx.in.gotPlt, *ctx.in.relaPlt,
+                addPltEntry(ctx, *ctx.in.plt, *ctx.in.gotPlt, *ctx.in.relaPlt,
                             ctx.target->pltRel, *sym);
                 needEntry = false;
               }
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index b9ef28f0436f8873699241c69ae7d45e44f7d809..02814e0413909447a196a7a2bbfd15b9b759c6b8 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -98,11 +98,11 @@ Symbol *SymbolTable::insert(StringRef name) {
 
 // This variant of addSymbol is used by BinaryFile::parse to check duplicate
 // symbol errors.
-Symbol *SymbolTable::addAndCheckDuplicate(const Defined &newSym) {
+Symbol *SymbolTable::addAndCheckDuplicate(Ctx &ctx, const Defined &newSym) {
   Symbol *sym = insert(newSym.getName());
   if (sym->isDefined())
-    sym->checkDuplicate(newSym);
-  sym->resolve(newSym);
+    sym->checkDuplicate(ctx, newSym);
+  sym->resolve(ctx, newSym);
   sym->isUsedInRegularObj = true;
   return sym;
 }
@@ -227,7 +227,7 @@ bool SymbolTable::assignExactVersion(SymbolVersion ver, uint16_t versionId,
   for (Symbol *sym : syms) {
     // For a non-local versionId, skip symbols containing version info because
     // symbol versions specified by symbol names take precedence over version
-    // scripts. See parseSymbolVersion().
+    // scripts. See parseSymbolVersion(ctx).
     if (!includeNonDefault && versionId != VER_NDX_LOCAL &&
         sym->getName().contains('@'))
       continue;
@@ -353,10 +353,10 @@ void SymbolTable::scanVersionScript() {
   // Let them parse and update their names to exclude version suffix.
   for (Symbol *sym : symVector)
     if (sym->hasVersionSuffix)
-      sym->parseSymbolVersion();
+      sym->parseSymbolVersion(ctx);
 
   // isPreemptible is false at this point. To correctly compute the binding of a
-  // Defined (which is used by includeInDynsym()), we need to know if it is
+  // Defined (which is used by includeInDynsym(ctx)), we need to know if it is
   // VER_NDX_LOCAL or not. Compute symbol versions before handling
   // --dynamic-list.
   handleDynamicList();
diff --git a/lld/ELF/SymbolTable.h b/lld/ELF/SymbolTable.h
index de00a016615514a2bde86b05bdf7220b1cdeee02..d6443742f7baad17c816cbdac91a1ff91e0e48dd 100644
--- a/lld/ELF/SymbolTable.h
+++ b/lld/ELF/SymbolTable.h
@@ -47,10 +47,10 @@ public:
 
   template <typename T> Symbol *addSymbol(const T &newSym) {
     Symbol *sym = insert(newSym.getName());
-    sym->resolve(newSym);
+    sym->resolve(ctx, newSym);
     return sym;
   }
-  Symbol *addAndCheckDuplicate(const Defined &newSym);
+  Symbol *addAndCheckDuplicate(Ctx &, const Defined &newSym);
 
   void scanVersionScript();
 
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index 0842c15974d1302222803ac698182c82378ae070..3caa609338e0683c1a7190265996bd0a1fd1c34e 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -199,7 +199,7 @@ OutputSection *Symbol::getOutputSection() const {
 
 // If a symbol name contains '@', the characters after that is
 // a symbol version name. This function parses that.
-void Symbol::parseSymbolVersion() {
+void Symbol::parseSymbolVersion(Ctx &ctx) {
   // Return if localized by a local: pattern in a version script.
   if (versionId == VER_NDX_LOCAL)
     return;
@@ -247,14 +247,14 @@ void Symbol::parseSymbolVersion() {
           verstr);
 }
 
-void Symbol::extract() const {
+void Symbol::extract(Ctx &ctx) const {
   if (file->lazy) {
     file->lazy = false;
     parseFile(ctx, file);
   }
 }
 
-uint8_t Symbol::computeBinding() const {
+uint8_t Symbol::computeBinding(Ctx &ctx) const {
   auto v = visibility();
   if ((v != STV_DEFAULT && v != STV_PROTECTED) || versionId == VER_NDX_LOCAL)
     return STB_LOCAL;
@@ -263,8 +263,8 @@ uint8_t Symbol::computeBinding() const {
   return binding;
 }
 
-bool Symbol::includeInDynsym() const {
-  if (computeBinding() == STB_LOCAL)
+bool Symbol::includeInDynsym(Ctx &ctx) const {
+  if (computeBinding(ctx) == STB_LOCAL)
     return false;
   if (!isDefined() && !isCommon())
     // This should unconditionally return true, unfortunately glibc -static-pie
@@ -293,12 +293,12 @@ void elf::printTraceSymbol(const Symbol &sym, StringRef name) {
   message(toString(sym.file) + s + name);
 }
 
-static void recordWhyExtract(const InputFile *reference,
+static void recordWhyExtract(Ctx &ctx, const InputFile *reference,
                              const InputFile &extracted, const Symbol &sym) {
   ctx.whyExtractRecords.emplace_back(toString(reference), &extracted, sym);
 }
 
-void elf::maybeWarnUnorderableSymbol(const Symbol *sym) {
+void elf::maybeWarnUnorderableSymbol(Ctx &ctx, const Symbol *sym) {
   if (!ctx.arg.warnSymbolOrdering)
     return;
 
@@ -339,7 +339,7 @@ bool elf::computeIsPreemptible(Ctx &ctx, const Symbol &sym) {
 
   // Only symbols with default visibility that appear in dynsym can be
   // preempted. Symbols with protected visibility cannot be preempted.
-  if (!sym.includeInDynsym() || sym.visibility() != STV_DEFAULT)
+  if (!sym.includeInDynsym(ctx) || sym.visibility() != STV_DEFAULT)
     return false;
 
   // At this point copy relocations have not been created yet, so any
@@ -380,7 +380,7 @@ void Symbol::mergeProperties(const Symbol &other) {
   }
 }
 
-void Symbol::resolve(const Undefined &other) {
+void Symbol::resolve(Ctx &ctx, const Undefined &other) {
   if (other.visibility() != STV_DEFAULT) {
     uint8_t v = visibility(), ov = other.visibility();
     setVisibility(v == STV_DEFAULT ? ov : std::min(v, ov));
@@ -460,10 +460,10 @@ void Symbol::resolve(const Undefined &other) {
     // group assignment rule simulates the traditional linker's semantics.
     bool backref = ctx.arg.warnBackrefs && other.file &&
                    file->groupId < other.file->groupId;
-    extract();
+    extract(ctx);
 
     if (!ctx.arg.whyExtract.empty())
-      recordWhyExtract(other.file, *file, *this);
+      recordWhyExtract(ctx, other.file, *file, *this);
 
     // We don't report backward references to weak symbols as they can be
     // overridden later.
@@ -493,7 +493,7 @@ void Symbol::resolve(const Undefined &other) {
 }
 
 // Compare two symbols. Return true if the new symbol should win.
-bool Symbol::shouldReplace(const Defined &other) const {
+bool Symbol::shouldReplace(Ctx &ctx, const Defined &other) const {
   if (LLVM_UNLIKELY(isCommon())) {
     if (ctx.arg.warnCommon)
       warn("common " + getName() + " is overridden");
@@ -553,14 +553,14 @@ void elf::reportDuplicate(Ctx &ctx, const Symbol &sym, const InputFile *newFile,
   errorOrWarn(msg);
 }
 
-void Symbol::checkDuplicate(const Defined &other) const {
+void Symbol::checkDuplicate(Ctx &ctx, const Defined &other) const {
   if (isDefined() && !isWeak() && !other.isWeak())
     reportDuplicate(ctx, *this, other.file,
                     dyn_cast_or_null<InputSectionBase>(other.section),
                     other.value);
 }
 
-void Symbol::resolve(const CommonSymbol &other) {
+void Symbol::resolve(Ctx &ctx, const CommonSymbol &other) {
   if (other.exportDynamic)
     exportDynamic = true;
   if (other.visibility() != STV_DEFAULT) {
@@ -598,18 +598,18 @@ void Symbol::resolve(const CommonSymbol &other) {
   }
 }
 
-void Symbol::resolve(const Defined &other) {
+void Symbol::resolve(Ctx &ctx, const Defined &other) {
   if (other.exportDynamic)
     exportDynamic = true;
   if (other.visibility() != STV_DEFAULT) {
     uint8_t v = visibility(), ov = other.visibility();
     setVisibility(v == STV_DEFAULT ? ov : std::min(v, ov));
   }
-  if (shouldReplace(other))
+  if (shouldReplace(ctx, other))
     other.overwrite(*this);
 }
 
-void Symbol::resolve(const LazySymbol &other) {
+void Symbol::resolve(Ctx &ctx, const LazySymbol &other) {
   if (isPlaceholder()) {
     other.overwrite(*this);
     return;
@@ -621,7 +621,7 @@ void Symbol::resolve(const LazySymbol &other) {
       other.file->shouldExtractForCommon(getName())) {
     ctx.backwardReferences.erase(this);
     other.overwrite(*this);
-    other.extract();
+    other.extract(ctx);
     return;
   }
 
@@ -643,12 +643,12 @@ void Symbol::resolve(const LazySymbol &other) {
   }
 
   const InputFile *oldFile = file;
-  other.extract();
+  other.extract(ctx);
   if (!ctx.arg.whyExtract.empty())
-    recordWhyExtract(oldFile, *file, *this);
+    recordWhyExtract(ctx, oldFile, *file, *this);
 }
 
-void Symbol::resolve(const SharedSymbol &other) {
+void Symbol::resolve(Ctx &ctx, const SharedSymbol &other) {
   exportDynamic = true;
   if (isPlaceholder()) {
     other.overwrite(*this);
diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index 424305233ec8623c335acb54af7a0c7b2c672e00..010ae9742378b992d5849f5e0d3213fdec8e813a 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -166,8 +166,8 @@ public:
     stOther = (stOther & ~3) | visibility;
   }
 
-  bool includeInDynsym() const;
-  uint8_t computeBinding() const;
+  bool includeInDynsym(Ctx &) const;
+  uint8_t computeBinding(Ctx &) const;
   bool isGlobal() const { return binding == llvm::ELF::STB_GLOBAL; }
   bool isWeak() const { return binding == llvm::ELF::STB_WEAK; }
 
@@ -192,12 +192,12 @@ public:
     nameSize = s.size();
   }
 
-  void parseSymbolVersion();
+  void parseSymbolVersion(Ctx &);
 
   // Get the NUL-terminated version suffix ("", "@...", or "@@...").
   //
   // For @@, the name has been truncated by insert(). For @, the name has been
-  // truncated by Symbol::parseSymbolVersion().
+  // truncated by Symbol::parseSymbolVersion(ctx).
   const char *getVersionSuffix() const { return nameData + nameSize; }
 
   uint32_t getGotIdx(Ctx &ctx) const { return ctx.symAux[auxIdx].gotIdx; }
@@ -234,21 +234,21 @@ public:
   // For example, if "this" is an undefined symbol and a new symbol is
   // a defined symbol, "this" is replaced with the new symbol.
   void mergeProperties(const Symbol &other);
-  void resolve(const Undefined &other);
-  void resolve(const CommonSymbol &other);
-  void resolve(const Defined &other);
-  void resolve(const LazySymbol &other);
-  void resolve(const SharedSymbol &other);
+  void resolve(Ctx &, const Undefined &other);
+  void resolve(Ctx &, const CommonSymbol &other);
+  void resolve(Ctx &, const Defined &other);
+  void resolve(Ctx &, const LazySymbol &other);
+  void resolve(Ctx &, const SharedSymbol &other);
 
   // If this is a lazy symbol, extract an input file and add the symbol
   // in the file to the symbol table. Calling this function on
   // non-lazy object causes a runtime error.
-  void extract() const;
+  void extract(Ctx &) const;
 
-  void checkDuplicate(const Defined &other) const;
+  void checkDuplicate(Ctx &, const Defined &other) const;
 
 private:
-  bool shouldReplace(const Defined &other) const;
+  bool shouldReplace(Ctx &, const Defined &other) const;
 
 protected:
   Symbol(Kind k, InputFile *file, StringRef name, uint8_t binding,
@@ -527,7 +527,7 @@ template <typename... T> Defined *makeDefined(T &&...args) {
 
 void reportDuplicate(Ctx &, const Symbol &sym, const InputFile *newFile,
                      InputSectionBase *errSec, uint64_t errOffset);
-void maybeWarnUnorderableSymbol(const Symbol *sym);
+void maybeWarnUnorderableSymbol(Ctx &, const Symbol *sym);
 bool computeIsPreemptible(Ctx &, const Symbol &sym);
 
 } // namespace elf
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index ee0e9c513740ac4cbc8c02c0cc0c48d0d2dc1526..d99aeac50ca5ce5c258f83106de4b8460b22725b 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -60,14 +60,14 @@ using llvm::support::endian::write64le;
 constexpr size_t MergeNoTailSection::numShards;
 
 static uint64_t readUint(Ctx &ctx, uint8_t *buf) {
-  return ctx.arg.is64 ? read64(buf) : read32(buf);
+  return ctx.arg.is64 ? read64(ctx, buf) : read32(ctx, buf);
 }
 
-static void writeUint(uint8_t *buf, uint64_t val) {
+static void writeUint(Ctx &ctx, uint8_t *buf, uint64_t val) {
   if (ctx.arg.is64)
-    write64(buf, val);
+    write64(ctx, buf, val);
   else
-    write32(buf, val);
+    write32(ctx, buf, val);
 }
 
 // Returns an LLD version string.
@@ -87,9 +87,9 @@ static ArrayRef<uint8_t> getVersion() {
 // With this feature, you can identify LLD-generated binaries easily
 // by "readelf --string-dump .comment <file>".
 // The returned object is a mergeable string section.
-MergeInputSection *elf::createCommentSection() {
-  auto *sec = make<MergeInputSection>(SHF_MERGE | SHF_STRINGS, SHT_PROGBITS, 1,
-                                      getVersion(), ".comment");
+MergeInputSection *elf::createCommentSection(Ctx &ctx) {
+  auto *sec = make<MergeInputSection>(
+      ctx, SHF_MERGE | SHF_STRINGS, SHT_PROGBITS, 1, getVersion(), ".comment");
   sec->splitIntoPieces();
   return sec;
 }
@@ -264,7 +264,7 @@ MipsReginfoSection<ELFT>::create(Ctx &ctx) {
   return std::make_unique<MipsReginfoSection<ELFT>>(ctx, reginfo);
 }
 
-InputSection *elf::createInterpSection(Ctx &) {
+InputSection *elf::createInterpSection(Ctx &ctx) {
   // StringSaver guarantees that the returned string ends with '\0'.
   StringRef s = saver().save(ctx.arg.dynamicLinker);
   ArrayRef<uint8_t> contents = {(const uint8_t *)s.data(), s.size() + 1};
@@ -273,8 +273,9 @@ InputSection *elf::createInterpSection(Ctx &) {
                             contents, ".interp");
 }
 
-Defined *elf::addSyntheticLocal(StringRef name, uint8_t type, uint64_t value,
-                                uint64_t size, InputSectionBase &section) {
+Defined *elf::addSyntheticLocal(Ctx &ctx, StringRef name, uint8_t type,
+                                uint64_t value, uint64_t size,
+                                InputSectionBase &section) {
   Defined *s = makeDefined(section.file, name, STB_LOCAL, STV_DEFAULT, type,
                            value, size, &section);
   if (ctx.in.symTab)
@@ -289,7 +290,7 @@ Defined *elf::addSyntheticLocal(StringRef name, uint8_t type, uint64_t value,
   return s;
 }
 
-static size_t getHashSize() {
+static size_t getHashSize(Ctx &ctx) {
   switch (ctx.arg.buildId) {
   case BuildIdKind::Fast:
     return 8;
@@ -322,9 +323,9 @@ GnuPropertySection::GnuPropertySection(Ctx &ctx)
                        ctx.arg.wordsize, ".note.gnu.property") {}
 
 void GnuPropertySection::writeTo(uint8_t *buf) {
-  write32(buf, 4);                          // Name size
-  write32(buf + 4, getSize() - 16);         // Content size
-  write32(buf + 8, NT_GNU_PROPERTY_TYPE_0); // Type
+  write32(ctx, buf, 4);                          // Name size
+  write32(ctx, buf + 4, getSize() - 16);         // Content size
+  write32(ctx, buf + 8, NT_GNU_PROPERTY_TYPE_0); // Type
   memcpy(buf + 12, "GNU", 4);               // Name string
 
   uint32_t featureAndType = ctx.arg.emachine == EM_AARCH64
@@ -333,17 +334,17 @@ void GnuPropertySection::writeTo(uint8_t *buf) {
 
   unsigned offset = 16;
   if (ctx.arg.andFeatures != 0) {
-    write32(buf + offset + 0, featureAndType);      // Feature type
-    write32(buf + offset + 4, 4);                   // Feature size
-    write32(buf + offset + 8, ctx.arg.andFeatures); // Feature flags
+    write32(ctx, buf + offset + 0, featureAndType);      // Feature type
+    write32(ctx, buf + offset + 4, 4);                   // Feature size
+    write32(ctx, buf + offset + 8, ctx.arg.andFeatures); // Feature flags
     if (ctx.arg.is64)
-      write32(buf + offset + 12, 0); // Padding
+      write32(ctx, buf + offset + 12, 0); // Padding
     offset += 16;
   }
 
   if (!ctx.aarch64PauthAbiCoreInfo.empty()) {
-    write32(buf + offset + 0, GNU_PROPERTY_AARCH64_FEATURE_PAUTH);
-    write32(buf + offset + 4, ctx.aarch64PauthAbiCoreInfo.size());
+    write32(ctx, buf + offset + 0, GNU_PROPERTY_AARCH64_FEATURE_PAUTH);
+    write32(ctx, buf + offset + 4, ctx.aarch64PauthAbiCoreInfo.size());
     memcpy(buf + offset + 8, ctx.aarch64PauthAbiCoreInfo.data(),
            ctx.aarch64PauthAbiCoreInfo.size());
   }
@@ -361,12 +362,12 @@ size_t GnuPropertySection::getSize() const {
 
 BuildIdSection::BuildIdSection(Ctx &ctx)
     : SyntheticSection(ctx, SHF_ALLOC, SHT_NOTE, 4, ".note.gnu.build-id"),
-      hashSize(getHashSize()) {}
+      hashSize(getHashSize(ctx)) {}
 
 void BuildIdSection::writeTo(uint8_t *buf) {
-  write32(buf, 4);                      // Name size
-  write32(buf + 4, hashSize);           // Content size
-  write32(buf + 8, NT_GNU_BUILD_ID);    // Type
+  write32(ctx, buf, 4);                   // Name size
+  write32(ctx, buf + 4, hashSize);        // Content size
+  write32(ctx, buf + 8, NT_GNU_BUILD_ID); // Type
   memcpy(buf + 12, "GNU", 4);           // Name string
   hashBuf = buf + 16;
 }
@@ -505,10 +506,10 @@ void EhFrameSection::iterateFDEWithLSDA(
   }
 }
 
-static void writeCieFde(uint8_t *buf, ArrayRef<uint8_t> d) {
+static void writeCieFde(Ctx &ctx, uint8_t *buf, ArrayRef<uint8_t> d) {
   memcpy(buf, d.data(), d.size());
   // Fix the size field. -4 since size does not include the size field itself.
-  write32(buf, d.size() - 4);
+  write32(ctx, buf, d.size() - 4);
 }
 
 void EhFrameSection::finalizeContents() {
@@ -592,19 +593,19 @@ SmallVector<EhFrameSection::FdeData, 0> EhFrameSection::getFdeData() const {
   return ret;
 }
 
-static uint64_t readFdeAddr(uint8_t *buf, int size) {
+static uint64_t readFdeAddr(Ctx &ctx, uint8_t *buf, int size) {
   switch (size) {
   case DW_EH_PE_udata2:
-    return read16(buf);
+    return read16(ctx, buf);
   case DW_EH_PE_sdata2:
-    return (int16_t)read16(buf);
+    return (int16_t)read16(ctx, buf);
   case DW_EH_PE_udata4:
-    return read32(buf);
+    return read32(ctx, buf);
   case DW_EH_PE_sdata4:
-    return (int32_t)read32(buf);
+    return (int32_t)read32(ctx, buf);
   case DW_EH_PE_udata8:
   case DW_EH_PE_sdata8:
-    return read64(buf);
+    return read64(ctx, buf);
   case DW_EH_PE_absptr:
     return readUint(ctx, buf);
   }
@@ -619,7 +620,7 @@ uint64_t EhFrameSection::getFdePc(uint8_t *buf, size_t fdeOff,
   // stored at FDE + 8 byte. And this offset is within
   // the .eh_frame section.
   size_t off = fdeOff + 8;
-  uint64_t addr = readFdeAddr(buf + off, enc & 0xf);
+  uint64_t addr = readFdeAddr(ctx, buf + off, enc & 0xf);
   if ((enc & 0x70) == DW_EH_PE_absptr)
     return ctx.arg.is64 ? addr : uint32_t(addr);
   if ((enc & 0x70) == DW_EH_PE_pcrel)
@@ -631,15 +632,15 @@ void EhFrameSection::writeTo(uint8_t *buf) {
   // Write CIE and FDE records.
   for (CieRecord *rec : cieRecords) {
     size_t cieOffset = rec->cie->outputOff;
-    writeCieFde(buf + cieOffset, rec->cie->data());
+    writeCieFde(ctx, buf + cieOffset, rec->cie->data());
 
     for (EhSectionPiece *fde : rec->fdes) {
       size_t off = fde->outputOff;
-      writeCieFde(buf + off, fde->data());
+      writeCieFde(ctx, buf + off, fde->data());
 
       // FDE's second word should have the offset to an associated CIE.
       // Write it.
-      write32(buf + off + 4, off + 4 - cieOffset);
+      write32(ctx, buf + off + 4, off + 4 - cieOffset);
     }
   }
 
@@ -1117,13 +1118,14 @@ void MipsGotSection::writeTo(uint8_t *buf) {
   // we've been doing this for years, it is probably a safe bet to
   // keep doing this for now. We really need to revisit this to see
   // if we had to do this.
-  writeUint(buf + ctx.arg.wordsize, (uint64_t)1 << (ctx.arg.wordsize * 8 - 1));
+  writeUint(ctx, buf + ctx.arg.wordsize,
+            (uint64_t)1 << (ctx.arg.wordsize * 8 - 1));
   for (const FileGot &g : gots) {
     auto write = [&](size_t i, const Symbol *s, int64_t a) {
       uint64_t va = a;
       if (s)
         va = s->getVA(a);
-      writeUint(buf + i * ctx.arg.wordsize, va);
+      writeUint(ctx, buf + i * ctx.arg.wordsize, va);
     };
     // Write 'page address' entries to the local part of the GOT.
     for (const std::pair<const OutputSection *, FileGot::PageBlock> &l :
@@ -1206,7 +1208,7 @@ bool GotPltSection::isNeeded() const {
   return !entries.empty() || hasGotPltOffRel;
 }
 
-static StringRef getIgotPltName() {
+static StringRef getIgotPltName(Ctx &ctx) {
   // On ARM the IgotPltSection is part of the GotSection.
   if (ctx.arg.emachine == EM_ARM)
     return ".got";
@@ -1224,7 +1226,7 @@ static StringRef getIgotPltName() {
 IgotPltSection::IgotPltSection(Ctx &ctx)
     : SyntheticSection(ctx, SHF_ALLOC | SHF_WRITE,
                        ctx.arg.emachine == EM_PPC64 ? SHT_NOBITS : SHT_PROGBITS,
-                       ctx.target->gotEntrySize, getIgotPltName()) {}
+                       ctx.target->gotEntrySize, getIgotPltName(ctx)) {}
 
 void IgotPltSection::addEntry(Symbol &sym) {
   assert(ctx.symAux.back().pltIdx == entries.size());
@@ -1281,7 +1283,9 @@ void StringTableSection::writeTo(uint8_t *buf) {
 // Returns the number of entries in .gnu.version_d: the number of
 // non-VER_NDX_LOCAL-non-VER_NDX_GLOBAL definitions, plus 1.
 // Note that we don't support vd_cnt > 1 yet.
-static unsigned getVerDefNum() { return namedVersionDefs(ctx).size() + 1; }
+static unsigned getVerDefNum(Ctx &ctx) {
+  return namedVersionDefs(ctx).size() + 1;
+}
 
 template <class ELFT>
 DynamicSection<ELFT>::DynamicSection(Ctx &ctx)
@@ -1304,7 +1308,7 @@ DynamicSection<ELFT>::DynamicSection(Ctx &ctx)
 //   .rela.dyn
 //
 // DT_RELASZ is the total size of the included sections.
-static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) {
+static uint64_t addRelaSz(Ctx &ctx, const RelocationBaseSection &relaDyn) {
   size_t size = relaDyn.getSize();
   if (ctx.in.relaPlt->getParent() == relaDyn.getParent())
     size += ctx.in.relaPlt->getSize();
@@ -1315,7 +1319,7 @@ static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) {
 // output section. When this occurs we cannot just use the OutputSection
 // Size. Moreover the [DT_JMPREL, DT_JMPREL + DT_PLTRELSZ) is permitted to
 // overlap with the [DT_RELA, DT_RELA + DT_RELASZ).
-static uint64_t addPltRelSz() { return ctx.in.relaPlt->getSize(); }
+static uint64_t addPltRelSz(Ctx &ctx) { return ctx.in.relaPlt->getSize(); }
 
 // Add remaining entries to complete .dynamic contents.
 template <class ELFT>
@@ -1405,7 +1409,7 @@ DynamicSection<ELFT>::computeContents() {
   if (part.relaDyn->isNeeded()) {
     addInSec(part.relaDyn->dynamicTag, *part.relaDyn);
     entries.emplace_back(part.relaDyn->sizeDynamicTag,
-                         addRelaSz(*part.relaDyn));
+                         addRelaSz(ctx, *part.relaDyn));
 
     bool isRela = ctx.arg.isRela;
     addInt(isRela ? DT_RELAENT : DT_RELENT,
@@ -1437,7 +1441,7 @@ DynamicSection<ELFT>::computeContents() {
   }
   if (isMain && ctx.in.relaPlt->isNeeded()) {
     addInSec(DT_JMPREL, *ctx.in.relaPlt);
-    entries.emplace_back(DT_PLTRELSZ, addPltRelSz());
+    entries.emplace_back(DT_PLTRELSZ, addPltRelSz(ctx));
     switch (ctx.arg.emachine) {
     case EM_MIPS:
       addInSec(DT_MIPS_PLTGOT, *ctx.in.gotPlt);
@@ -1478,7 +1482,7 @@ DynamicSection<ELFT>::computeContents() {
     if (ctx.arg.zPacPlt)
       addInt(DT_AARCH64_PAC_PLT, 0);
 
-    if (hasMemtag()) {
+    if (hasMemtag(ctx)) {
       addInt(DT_AARCH64_MEMTAG_MODE, ctx.arg.androidMemtagMode == NT_MEMTAG_LEVEL_ASYNC);
       addInt(DT_AARCH64_MEMTAG_HEAP, ctx.arg.androidMemtagHeap);
       addInt(DT_AARCH64_MEMTAG_STACK, ctx.arg.androidMemtagStack);
@@ -1528,7 +1532,7 @@ DynamicSection<ELFT>::computeContents() {
     addInSec(DT_VERSYM, *part.verSym);
   if (part.verDef && part.verDef->isLive()) {
     addInSec(DT_VERDEF, *part.verDef);
-    addInt(DT_VERDEFNUM, getVerDefNum());
+    addInt(DT_VERDEFNUM, getVerDefNum(ctx));
   }
   if (part.verNeed && part.verNeed->isNeeded()) {
     addInSec(DT_VERNEED, *part.verNeed);
@@ -1601,7 +1605,7 @@ uint64_t DynamicReloc::getOffset() const {
   return inputSec->getVA(offsetInSec);
 }
 
-int64_t DynamicReloc::computeAddend() const {
+int64_t DynamicReloc::computeAddend(Ctx &ctx) const {
   switch (kind) {
   case AddendOnly:
     assert(sym == nullptr);
@@ -1628,8 +1632,9 @@ uint32_t DynamicReloc::getSymIndex(SymbolTableBaseSection *symTab) const {
 
   size_t index = symTab->getSymbolIndex(*sym);
   assert((index != 0 ||
-          (type != ctx.target->gotRel && type != ctx.target->pltRel) ||
-          !ctx.mainPart->dynSymTab->getParent()) &&
+          (type != symTab->ctx.target->gotRel &&
+           type != symTab->ctx.target->pltRel) ||
+          !symTab->ctx.mainPart->dynSymTab->getParent()) &&
          "GOT or PLT relocation must refer to symbol in dynamic symbol table");
   return index;
 }
@@ -1699,17 +1704,18 @@ void RelocationBaseSection::finalizeContents() {
   }
 }
 
-void DynamicReloc::computeRaw(SymbolTableBaseSection *symt) {
+void DynamicReloc::computeRaw(Ctx &ctx, SymbolTableBaseSection *symt) {
   r_offset = getOffset();
   r_sym = getSymIndex(symt);
-  addend = computeAddend();
+  addend = computeAddend(ctx);
   kind = AddendOnly; // Catch errors
 }
 
 void RelocationBaseSection::computeRels() {
   SymbolTableBaseSection *symTab = getPartition().dynSymTab.get();
-  parallelForEach(relocs,
-                  [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); });
+  parallelForEach(relocs, [&ctx = ctx, symTab](DynamicReloc &rel) {
+    rel.computeRaw(ctx, symTab);
+  });
 
   auto irelative = std::stable_partition(
       relocs.begin() + numRelativeRelocs, relocs.end(),
@@ -1848,7 +1854,7 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize(Ctx &ctx) {
     r.r_offset = rel.getOffset();
     r.setSymbolAndType(rel.getSymIndex(getPartition().dynSymTab.get()),
                        rel.type, false);
-    r.r_addend = ctx.arg.isRela ? rel.computeAddend() : 0;
+    r.r_addend = ctx.arg.isRela ? rel.computeAddend(ctx) : 0;
 
     if (r.getType(ctx.arg.isMips64EL) == ctx.target->relativeRel)
       relatives.push_back(r);
@@ -2126,15 +2132,18 @@ SymbolTableBaseSection::SymbolTableBaseSection(Ctx &ctx,
 // See "Global Offset Table" in Chapter 5 in the following document
 // for detailed description:
 // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
-static bool sortMipsSymbols(const SymbolTableEntry &l,
-                            const SymbolTableEntry &r) {
-  // Sort entries related to non-local preemptible symbols by GOT indexes.
-  // All other entries go to the beginning of a dynsym in arbitrary order.
-  if (l.sym->isInGot(ctx) && r.sym->isInGot(ctx))
-    return l.sym->getGotIdx(ctx) < r.sym->getGotIdx(ctx);
-  if (!l.sym->isInGot(ctx) && !r.sym->isInGot(ctx))
-    return false;
-  return !l.sym->isInGot(ctx);
+static void sortMipsSymbols(Ctx &ctx, SmallVector<SymbolTableEntry, 0> &syms) {
+  llvm::stable_sort(syms,
+                    [&](const SymbolTableEntry &l, const SymbolTableEntry &r) {
+                      // Sort entries related to non-local preemptible symbols
+                      // by GOT indexes. All other entries go to the beginning
+                      // of a dynsym in arbitrary order.
+                      if (l.sym->isInGot(ctx) && r.sym->isInGot(ctx))
+                        return l.sym->getGotIdx(ctx) < r.sym->getGotIdx(ctx);
+                      if (!l.sym->isInGot(ctx) && !r.sym->isInGot(ctx))
+                        return false;
+                      return !l.sym->isInGot(ctx);
+                    });
 }
 
 void SymbolTableBaseSection::finalizeContents() {
@@ -2157,7 +2166,7 @@ void SymbolTableBaseSection::finalizeContents() {
     // NB: It also sorts Symbols to meet the GNU hash table requirements.
     getPartition().gnuHashTab->addSymbols(symbols);
   } else if (ctx.arg.emachine == EM_MIPS) {
-    llvm::stable_sort(symbols, sortMipsSymbols);
+    sortMipsSymbols(ctx, symbols);
   }
 
   // Only the main partition's dynsym indexes are stored in the symbols
@@ -2236,8 +2245,8 @@ SymbolTableSection<ELFT>::SymbolTableSection(Ctx &ctx,
   this->entsize = sizeof(Elf_Sym);
 }
 
-static BssSection *getCommonSec(Symbol *sym) {
-  if (ctx.arg.relocatable)
+static BssSection *getCommonSec(bool relocatable, Symbol *sym) {
+  if (relocatable)
     if (auto *d = dyn_cast<Defined>(sym))
       return dyn_cast_or_null<BssSection>(d->section);
   return nullptr;
@@ -2259,7 +2268,7 @@ template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) {
   buf += sizeof(Elf_Sym);
 
   auto *eSym = reinterpret_cast<Elf_Sym *>(buf);
-
+  bool relocatable = ctx.arg.relocatable;
   for (SymbolTableEntry &ent : symbols) {
     Symbol *sym = ent.sym;
     bool isDefinedHere = type == SHT_SYMTAB || sym->partition == partition;
@@ -2269,7 +2278,7 @@ template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) {
     eSym->setBindingAndType(sym->binding, sym->type);
     eSym->st_other = sym->stOther;
 
-    if (BssSection *commonSec = getCommonSec(sym)) {
+    if (BssSection *commonSec = getCommonSec(relocatable, sym)) {
       // When -r is specified, a COMMON symbol is not allocated. Its st_shndx
       // holds SHN_COMMON and st_value holds the alignment.
       eSym->st_shndx = SHN_COMMON;
@@ -2342,9 +2351,11 @@ void SymtabShndxSection::writeTo(uint8_t *buf) {
   // SHN_XINDEX, we need to write actual index, otherwise, we must write
   // SHN_UNDEF(0).
   buf += 4; // Ignore .symtab[0] entry.
+  bool relocatable = ctx.arg.relocatable;
   for (const SymbolTableEntry &entry : ctx.in.symTab->getSymbols()) {
-    if (!getCommonSec(entry.sym) && getSymSectionIndex(entry.sym) == SHN_XINDEX)
-      write32(buf, entry.sym->getOutputSection()->sectionIndex);
+    if (!getCommonSec(relocatable, entry.sym) &&
+        getSymSectionIndex(entry.sym) == SHN_XINDEX)
+      write32(ctx, buf, entry.sym->getOutputSection()->sectionIndex);
     buf += 4;
   }
 }
@@ -2425,10 +2436,11 @@ void GnuHashTableSection::finalizeContents() {
 
 void GnuHashTableSection::writeTo(uint8_t *buf) {
   // Write a header.
-  write32(buf, nBuckets);
-  write32(buf + 4, getPartition().dynSymTab->getNumSymbols() - symbols.size());
-  write32(buf + 8, maskWords);
-  write32(buf + 12, Shift2);
+  write32(ctx, buf, nBuckets);
+  write32(ctx, buf + 4,
+          getPartition().dynSymTab->getNumSymbols() - symbols.size());
+  write32(ctx, buf + 8, maskWords);
+  write32(ctx, buf + 12, Shift2);
   buf += 16;
 
   // Write the 2-bit bloom filter.
@@ -2440,7 +2452,7 @@ void GnuHashTableSection::writeTo(uint8_t *buf) {
     uint64_t val = readUint(ctx, buf + i * ctx.arg.wordsize);
     val |= uint64_t(1) << (sym.hash % c);
     val |= uint64_t(1) << ((sym.hash >> Shift2) % c);
-    writeUint(buf + i * ctx.arg.wordsize, val);
+    writeUint(ctx, buf + i * ctx.arg.wordsize, val);
   }
   buf += ctx.arg.wordsize * maskWords;
 
@@ -2455,13 +2467,13 @@ void GnuHashTableSection::writeTo(uint8_t *buf) {
     uint32_t hash = i->hash;
     bool isLastInChain = (i + 1) == e || i->bucketIdx != (i + 1)->bucketIdx;
     hash = isLastInChain ? hash | 1 : hash & ~1;
-    write32(values++, hash);
+    write32(ctx, values++, hash);
 
     if (i->bucketIdx == oldBucket)
       continue;
     // Write a hash bucket. Hash buckets contain indices in the following hash
     // value table.
-    write32(buckets + i->bucketIdx,
+    write32(ctx, buckets + i->bucketIdx,
             getPartition().dynSymTab->getSymbolIndex(*i->sym));
     oldBucket = i->bucketIdx;
   }
@@ -2533,8 +2545,8 @@ void HashTableSection::writeTo(uint8_t *buf) {
   unsigned numSymbols = symTab->getNumSymbols();
 
   uint32_t *p = reinterpret_cast<uint32_t *>(buf);
-  write32(p++, numSymbols); // nbucket
-  write32(p++, numSymbols); // nchain
+  write32(ctx, p++, numSymbols); // nbucket
+  write32(ctx, p++, numSymbols); // nchain
 
   uint32_t *buckets = p;
   uint32_t *chains = p + numSymbols;
@@ -2545,7 +2557,7 @@ void HashTableSection::writeTo(uint8_t *buf) {
     unsigned i = sym->dynsymIndex;
     uint32_t hash = hashSysV(name) % numSymbols;
     chains[i] = buckets[hash];
-    write32(buckets + hash, i);
+    write32(ctx, buckets + hash, i);
   }
 }
 
@@ -3428,6 +3440,7 @@ readPubNamesAndTypes(const LLDDwarfObj<ELFT> &obj,
 // by uniquifying them by name.
 static std::pair<SmallVector<GdbIndexSection::GdbSymbol, 0>, size_t>
 createSymbols(
+    Ctx &ctx,
     ArrayRef<SmallVector<GdbIndexSection::NameAttrEntry, 0>> nameAttrs,
     const SmallVector<GdbIndexSection::GdbChunk, 0> &chunks) {
   using GdbSymbol = GdbIndexSection::GdbSymbol;
@@ -3553,7 +3566,8 @@ std::unique_ptr<GdbIndexSection> GdbIndexSection::create(Ctx &ctx) {
 
   auto ret = std::make_unique<GdbIndexSection>(ctx);
   ret->chunks = std::move(chunks);
-  std::tie(ret->symbols, ret->size) = createSymbols(nameAttrs, ret->chunks);
+  std::tie(ret->symbols, ret->size) =
+      createSymbols(ctx, nameAttrs, ret->chunks);
 
   // Count the areas other than the constant pool.
   ret->size += sizeof(GdbIndexHeader) + ret->computeSymtabSize() * 8;
@@ -3659,14 +3673,14 @@ void EhFrameHeader::write() {
   buf[1] = DW_EH_PE_pcrel | DW_EH_PE_sdata4;
   buf[2] = DW_EH_PE_udata4;
   buf[3] = DW_EH_PE_datarel | DW_EH_PE_sdata4;
-  write32(buf + 4,
+  write32(ctx, buf + 4,
           getPartition().ehFrame->getParent()->addr - this->getVA() - 4);
-  write32(buf + 8, fdes.size());
+  write32(ctx, buf + 8, fdes.size());
   buf += 12;
 
   for (FdeData &fde : fdes) {
-    write32(buf, fde.pcRel);
-    write32(buf + 4, fde.fdeVARel);
+    write32(ctx, buf, fde.pcRel);
+    write32(ctx, buf + 4, fde.fdeVARel);
     buf += 8;
   }
 }
@@ -3703,7 +3717,7 @@ void VersionDefinitionSection::finalizeContents() {
   // sh_info should be set to the number of definitions. This fact is missed in
   // documentation, but confirmed by binutils community:
   // https://sourceware.org/ml/binutils/2014-11/msg00355.html
-  getParent()->info = getVerDefNum();
+  getParent()->info = getVerDefNum(ctx);
 }
 
 void VersionDefinitionSection::writeOne(uint8_t *buf, uint32_t index,
@@ -3711,17 +3725,17 @@ void VersionDefinitionSection::writeOne(uint8_t *buf, uint32_t index,
   uint16_t flags = index == 1 ? VER_FLG_BASE : 0;
 
   // Write a verdef.
-  write16(buf, 1);                  // vd_version
-  write16(buf + 2, flags);          // vd_flags
-  write16(buf + 4, index);          // vd_ndx
-  write16(buf + 6, 1);              // vd_cnt
-  write32(buf + 8, hashSysV(name)); // vd_hash
-  write32(buf + 12, 20);            // vd_aux
-  write32(buf + 16, 28);            // vd_next
+  write16(ctx, buf, 1);                  // vd_version
+  write16(ctx, buf + 2, flags);          // vd_flags
+  write16(ctx, buf + 4, index);          // vd_ndx
+  write16(ctx, buf + 6, 1);              // vd_cnt
+  write32(ctx, buf + 8, hashSysV(name)); // vd_hash
+  write32(ctx, buf + 12, 20);            // vd_aux
+  write32(ctx, buf + 16, 28);            // vd_next
 
   // Write a veraux.
-  write32(buf + 20, nameOff); // vda_name
-  write32(buf + 24, 0);       // vda_next
+  write32(ctx, buf + 20, nameOff); // vda_name
+  write32(ctx, buf + 24, 0);       // vda_next
 }
 
 void VersionDefinitionSection::writeTo(uint8_t *buf) {
@@ -3734,11 +3748,11 @@ void VersionDefinitionSection::writeTo(uint8_t *buf) {
   }
 
   // Need to terminate the last version definition.
-  write32(buf + 16, 0); // vd_next
+  write32(ctx, buf + 16, 0); // vd_next
 }
 
 size_t VersionDefinitionSection::getSize() const {
-  return EntrySize * getVerDefNum();
+  return EntrySize * getVerDefNum(ctx);
 }
 
 // .gnu.version is a table where each entry is 2 byte long.
@@ -3764,7 +3778,7 @@ void VersionTableSection::writeTo(uint8_t *buf) {
     // For an unextracted lazy symbol (undefined weak), it must have been
     // converted to Undefined and have VER_NDX_GLOBAL version here.
     assert(!s.sym->isLazy());
-    write16(buf, s.sym->versionId);
+    write16(ctx, buf, s.sym->versionId);
     buf += 2;
   }
 }
@@ -3774,9 +3788,9 @@ bool VersionTableSection::isNeeded() const {
          (getPartition().verDef || getPartition().verNeed->isNeeded());
 }
 
-void elf::addVerneed(Symbol *ss) {
-  auto &file = cast<SharedFile>(*ss->file);
-  if (ss->versionId == VER_NDX_GLOBAL)
+void elf::addVerneed(Ctx &ctx, Symbol &ss) {
+  auto &file = cast<SharedFile>(*ss.file);
+  if (ss.versionId == VER_NDX_GLOBAL)
     return;
 
   if (file.vernauxs.empty())
@@ -3784,12 +3798,12 @@ void elf::addVerneed(Symbol *ss) {
 
   // Select a version identifier for the vernaux data structure, if we haven't
   // already allocated one. The verdef identifiers cover the range
-  // [1..getVerDefNum()]; this causes the vernaux identifiers to start from
-  // getVerDefNum()+1.
-  if (file.vernauxs[ss->versionId] == 0)
-    file.vernauxs[ss->versionId] = ++SharedFile::vernauxNum + getVerDefNum();
+  // [1..getVerDefNum(ctx)]; this causes the vernaux identifiers to start from
+  // getVerDefNum(ctx)+1.
+  if (file.vernauxs[ss.versionId] == 0)
+    file.vernauxs[ss.versionId] = ++SharedFile::vernauxNum + getVerDefNum(ctx);
 
-  ss->versionId = file.vernauxs[ss->versionId];
+  ss.versionId = file.vernauxs[ss.versionId];
 }
 
 template <class ELFT>
@@ -3820,7 +3834,7 @@ template <class ELFT> void VersionNeedSection<ELFT>::finalizeContents() {
     if (isGlibc2) {
       const char *ver = "GLIBC_ABI_DT_RELR";
       vn.vernauxs.push_back({hashSysV(ver),
-                             ++SharedFile::vernauxNum + getVerDefNum(),
+                             ++SharedFile::vernauxNum + getVerDefNum(ctx),
                              getPartition().dynStrTab->addString(ver)});
     }
   }
@@ -4063,12 +4077,14 @@ static bool isExtabRef(uint32_t unwind) {
 // unwinding instructions in Cur are identical to Prev. Linker generated
 // EXIDX_CANTUNWIND entries are represented by nullptr as they do not have an
 // InputSection.
-static bool isDuplicateArmExidxSec(InputSection *prev, InputSection *cur) {
+static bool isDuplicateArmExidxSec(Ctx &ctx, InputSection *prev,
+                                   InputSection *cur) {
   // Get the last table Entry from the previous .ARM.exidx section. If Prev is
   // nullptr then it will be a synthesized EXIDX_CANTUNWIND entry.
   uint32_t prevUnwind = 1;
   if (prev)
-    prevUnwind = read32(prev->content().data() + prev->content().size() - 4);
+    prevUnwind =
+        read32(ctx, prev->content().data() + prev->content().size() - 4);
   if (isExtabRef(prevUnwind))
     return false;
 
@@ -4085,7 +4101,7 @@ static bool isDuplicateArmExidxSec(InputSection *prev, InputSection *cur) {
     return prevUnwind == 1;
 
   for (uint32_t offset = 4; offset < (uint32_t)cur->content().size(); offset +=8) {
-    uint32_t curUnwind = read32(cur->content().data() + offset);
+    uint32_t curUnwind = read32(ctx, cur->content().data() + offset);
     if (isExtabRef(curUnwind) || curUnwind != prevUnwind)
       return false;
   }
@@ -4151,7 +4167,7 @@ void ARMExidxSyntheticSection::finalizeContents() {
     for (size_t i = 1; i < executableSections.size(); ++i) {
       InputSection *ex1 = findExidxSection(executableSections[prev]);
       InputSection *ex2 = findExidxSection(executableSections[i]);
-      if (!isDuplicateArmExidxSec(ex1, ex2)) {
+      if (!isDuplicateArmExidxSec(ctx, ex1, ex2)) {
         selectedSections.push_back(executableSections[i]);
         prev = i;
       }
@@ -4198,8 +4214,8 @@ void ARMExidxSyntheticSection::writeTo(uint8_t *buf) {
     if (InputSection *d = findExidxSection(isec)) {
       for (int dataOffset = 0; dataOffset != (int)d->content().size();
            dataOffset += 4)
-        write32(buf + offset + dataOffset,
-                read32(d->content().data() + dataOffset));
+        write32(ctx, buf + offset + dataOffset,
+                read32(ctx, d->content().data() + dataOffset));
       // Recalculate outSecOff as finalizeAddressDependentContent()
       // may have altered syntheticSection outSecOff.
       d->outSecOff = offset + outSecOff;
@@ -4207,8 +4223,8 @@ void ARMExidxSyntheticSection::writeTo(uint8_t *buf) {
       offset += d->getSize();
     } else {
       // A Linker generated CANTUNWIND section.
-      write32(buf + offset + 0, 0x0);
-      write32(buf + offset + 4, 0x1);
+      write32(ctx, buf + offset + 0, 0x0);
+      write32(ctx, buf + offset + 4, 0x1);
       uint64_t s = isec->getVA();
       uint64_t p = getVA() + offset;
       ctx.target->relocateNoSym(buf + offset, R_ARM_PREL31, s - p);
@@ -4216,8 +4232,8 @@ void ARMExidxSyntheticSection::writeTo(uint8_t *buf) {
     }
   }
   // Write Sentinel CANTUNWIND entry.
-  write32(buf + offset + 0, 0x0);
-  write32(buf + offset + 4, 0x1);
+  write32(ctx, buf + offset + 0, 0x0);
+  write32(ctx, buf + offset + 4, 0x1);
   uint64_t s = sentinel->getVA(sentinel->getSize());
   uint64_t p = getVA() + offset;
   ctx.target->relocateNoSym(buf + offset, R_ARM_PREL31, s - p);
@@ -4343,8 +4359,9 @@ void PPC64LongBranchTargetSection::writeTo(uint8_t *buf) {
     assert(sym->getVA());
     // Need calls to branch to the local entry-point since a long-branch
     // must be a local-call.
-    write64(buf, sym->getVA(addend) +
-                     getPPC64GlobalEntryToLocalEntryOffset(sym->stOther));
+    write64(ctx, buf,
+            sym->getVA(addend) +
+                getPPC64GlobalEntryToLocalEntryOffset(sym->stOther));
     buf += 8;
   }
 }
@@ -4359,7 +4376,7 @@ bool PPC64LongBranchTargetSection::isNeeded() const {
   return !finalized || !entries.empty();
 }
 
-static uint8_t getAbiVersion() {
+static uint8_t getAbiVersion(Ctx &ctx) {
   // MIPS non-PIC executable gets ABI version 1.
   if (ctx.arg.emachine == EM_MIPS) {
     if (!ctx.arg.isPic && !ctx.arg.relocatable &&
@@ -4379,7 +4396,8 @@ static uint8_t getAbiVersion() {
   return 0;
 }
 
-template <typename ELFT> void elf::writeEhdr(uint8_t *buf, Partition &part) {
+template <typename ELFT>
+void elf::writeEhdr(Ctx &ctx, uint8_t *buf, Partition &part) {
   memcpy(buf, "\177ELF", 4);
 
   auto *eHdr = reinterpret_cast<typename ELFT::Ehdr *>(buf);
@@ -4388,7 +4406,7 @@ template <typename ELFT> void elf::writeEhdr(uint8_t *buf, Partition &part) {
       ELFT::Endianness == endianness::little ? ELFDATA2LSB : ELFDATA2MSB;
   eHdr->e_ident[EI_VERSION] = EV_CURRENT;
   eHdr->e_ident[EI_OSABI] = ctx.arg.osabi;
-  eHdr->e_ident[EI_ABIVERSION] = getAbiVersion();
+  eHdr->e_ident[EI_ABIVERSION] = getAbiVersion(ctx);
   eHdr->e_machine = ctx.arg.emachine;
   eHdr->e_version = EV_CURRENT;
   eHdr->e_flags = ctx.arg.eflags;
@@ -4429,7 +4447,7 @@ size_t PartitionElfHeaderSection<ELFT>::getSize() const {
 
 template <typename ELFT>
 void PartitionElfHeaderSection<ELFT>::writeTo(uint8_t *buf) {
-  writeEhdr<ELFT>(buf, getPartition());
+  writeEhdr<ELFT>(ctx, buf, getPartition());
 
   // Loadable partitions are always ET_DYN.
   auto *eHdr = reinterpret_cast<typename ELFT::Ehdr *>(buf);
@@ -4466,14 +4484,15 @@ void PartitionIndexSection::finalizeContents() {
 void PartitionIndexSection::writeTo(uint8_t *buf) {
   uint64_t va = getVA();
   for (size_t i = 1; i != ctx.partitions.size(); ++i) {
-    write32(buf, ctx.mainPart->dynStrTab->getVA() +
-                     ctx.partitions[i].nameStrTab - va);
-    write32(buf + 4, ctx.partitions[i].elfHeader->getVA() - (va + 4));
+    write32(ctx, buf,
+            ctx.mainPart->dynStrTab->getVA() + ctx.partitions[i].nameStrTab -
+                va);
+    write32(ctx, buf + 4, ctx.partitions[i].elfHeader->getVA() - (va + 4));
 
     SyntheticSection *next = i == ctx.partitions.size() - 1
                                  ? ctx.in.partEnd.get()
                                  : ctx.partitions[i + 1].elfHeader.get();
-    write32(buf + 8, next->getVA() - ctx.partitions[i].elfHeader->getVA());
+    write32(ctx, buf + 8, next->getVA() - ctx.partitions[i].elfHeader->getVA());
 
     va += 12;
     buf += 12;
@@ -4516,7 +4535,7 @@ static bool needsInterpSection(Ctx &ctx) {
          !ctx.arg.dynamicLinker.empty() && ctx.script->needsInterpSection();
 }
 
-bool elf::hasMemtag() {
+bool elf::hasMemtag(Ctx &ctx) {
   return ctx.arg.emachine == EM_AARCH64 &&
          ctx.arg.androidMemtagMode != ELF::NT_MEMTAG_LEVEL_NONE;
 }
@@ -4527,8 +4546,8 @@ bool elf::hasMemtag() {
 //   - Dynamic entries.
 // This restriction could be removed in future by re-using some of the ideas
 // that ifuncs use in fully static executables.
-bool elf::canHaveMemtagGlobals() {
-  return hasMemtag() &&
+bool elf::canHaveMemtagGlobals(Ctx &ctx) {
+  return hasMemtag(ctx) &&
          (ctx.arg.relocatable || ctx.arg.shared || needsInterpSection(ctx));
 }
 
@@ -4539,9 +4558,9 @@ void MemtagAndroidNote::writeTo(uint8_t *buf) {
       "Android 11 & 12 have an ABI that the note name is 8 bytes long. Keep it "
       "that way for backwards compatibility.");
 
-  write32(buf, sizeof(kMemtagAndroidNoteName));
-  write32(buf + 4, sizeof(uint32_t));
-  write32(buf + 8, ELF::NT_ANDROID_TYPE_MEMTAG);
+  write32(ctx, buf, sizeof(kMemtagAndroidNoteName));
+  write32(ctx, buf + 4, sizeof(uint32_t));
+  write32(ctx, buf + 8, ELF::NT_ANDROID_TYPE_MEMTAG);
   memcpy(buf + 12, kMemtagAndroidNoteName, sizeof(kMemtagAndroidNoteName));
   buf += 12 + alignTo(sizeof(kMemtagAndroidNoteName), 4);
 
@@ -4553,7 +4572,7 @@ void MemtagAndroidNote::writeTo(uint8_t *buf) {
   // binary on Android 11 or 12 will result in a checkfail in the loader.
   if (ctx.arg.androidMemtagStack)
     value |= ELF::NT_MEMTAG_STACK;
-  write32(buf, value); // note value
+  write32(ctx, buf, value); // note value
 }
 
 size_t MemtagAndroidNote::getSize() const {
@@ -4563,9 +4582,9 @@ size_t MemtagAndroidNote::getSize() const {
 }
 
 void PackageMetadataNote::writeTo(uint8_t *buf) {
-  write32(buf, 4);
-  write32(buf + 4, ctx.arg.packageMetadata.size() + 1);
-  write32(buf + 8, FDO_PACKAGING_METADATA);
+  write32(ctx, buf, 4);
+  write32(ctx, buf + 4, ctx.arg.packageMetadata.size() + 1);
+  write32(ctx, buf + 8, FDO_PACKAGING_METADATA);
   memcpy(buf + 12, "FDO", 4);
   memcpy(buf + 16, ctx.arg.packageMetadata.data(),
          ctx.arg.packageMetadata.size());
@@ -4648,7 +4667,7 @@ size_t MemtagGlobalDescriptors::getSize() const {
   return createMemtagGlobalDescriptors(ctx, symbols);
 }
 
-static OutputSection *findSection(StringRef name) {
+static OutputSection *findSection(Ctx &ctx, StringRef name) {
   for (SectionCommand *cmd : ctx.script->sectionCommands)
     if (auto *osd = dyn_cast<OutputDesc>(cmd))
       if (osd->osec.name == name)
@@ -4656,15 +4675,15 @@ static OutputSection *findSection(StringRef name) {
   return nullptr;
 }
 
-static Defined *addOptionalRegular(StringRef name, SectionBase *sec,
+static Defined *addOptionalRegular(Ctx &ctx, StringRef name, SectionBase *sec,
                                    uint64_t val, uint8_t stOther = STV_HIDDEN) {
   Symbol *s = ctx.symtab->find(name);
   if (!s || s->isDefined() || s->isCommon())
     return nullptr;
 
-  s->resolve(Defined{ctx.internalFile, StringRef(), STB_GLOBAL, stOther,
-                     STT_NOTYPE, val,
-                     /*size=*/0, sec});
+  s->resolve(ctx, Defined{ctx.internalFile, StringRef(), STB_GLOBAL, stOther,
+                          STT_NOTYPE, val,
+                          /*size=*/0, sec});
   s->isUsedInRegularObj = true;
   return cast<Defined>(s);
 }
@@ -4687,7 +4706,7 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
     ctx.in.shStrTab =
         std::make_unique<StringTableSection>(ctx, ".shstrtab", false);
 
-  ctx.out.programHeaders = make<OutputSection>("", 0, SHF_ALLOC);
+  ctx.out.programHeaders = make<OutputSection>(ctx, "", 0, SHF_ALLOC);
   ctx.out.programHeaders->addralign = ctx.arg.wordsize;
 
   if (ctx.arg.strip != StripPolicy::All) {
@@ -4704,7 +4723,7 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
   // .data.rel.ro.bss so that we match in the .data.rel.ro output section.
   // This makes sure our relro is contiguous.
   bool hasDataRelRo =
-      ctx.script->hasSectionsCommand && findSection(".data.rel.ro");
+      ctx.script->hasSectionsCommand && findSection(ctx, ".data.rel.ro");
   ctx.in.bssRelRo = std::make_unique<BssSection>(
       ctx, hasDataRelRo ? ".data.rel.ro.bss" : ".bss.rel.ro", 0, 1);
   add(*ctx.in.bssRelRo);
@@ -4747,7 +4766,7 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
       add(*part.buildId);
     }
 
-    // dynSymTab is always present to simplify sym->includeInDynsym() in
+    // dynSymTab is always present to simplify sym->includeInDynsym(ctx) in
     // finalizeSections.
     part.dynStrTab = std::make_unique<StringTableSection>(ctx, ".dynstr", true);
     part.dynSymTab =
@@ -4757,10 +4776,10 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
       continue;
     part.dynamic = std::make_unique<DynamicSection<ELFT>>(ctx);
 
-    if (hasMemtag()) {
+    if (hasMemtag(ctx)) {
       part.memtagAndroidNote = std::make_unique<MemtagAndroidNote>(ctx);
       add(*part.memtagAndroidNote);
-      if (canHaveMemtagGlobals()) {
+      if (canHaveMemtagGlobals(ctx)) {
         part.memtagGlobalDescriptors =
             std::make_unique<MemtagGlobalDescriptors>(ctx);
         add(*part.memtagGlobalDescriptors);
@@ -4840,8 +4859,8 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
     add(*ctx.in.partEnd);
 
     ctx.in.partIndex = std::make_unique<PartitionIndexSection>(ctx);
-    addOptionalRegular("__part_index_begin", ctx.in.partIndex.get(), 0);
-    addOptionalRegular("__part_index_end", ctx.in.partIndex.get(),
+    addOptionalRegular(ctx, "__part_index_begin", ctx.in.partIndex.get(), 0);
+    addOptionalRegular(ctx, "__part_index_end", ctx.in.partIndex.get(),
                        ctx.in.partIndex->getSize());
     add(*ctx.in.partIndex);
   }
@@ -4965,10 +4984,10 @@ template class elf::SymbolTableSection<ELF32BE>;
 template class elf::SymbolTableSection<ELF64LE>;
 template class elf::SymbolTableSection<ELF64BE>;
 
-template void elf::writeEhdr<ELF32LE>(uint8_t *Buf, Partition &Part);
-template void elf::writeEhdr<ELF32BE>(uint8_t *Buf, Partition &Part);
-template void elf::writeEhdr<ELF64LE>(uint8_t *Buf, Partition &Part);
-template void elf::writeEhdr<ELF64BE>(uint8_t *Buf, Partition &Part);
+template void elf::writeEhdr<ELF32LE>(Ctx &, uint8_t *Buf, Partition &Part);
+template void elf::writeEhdr<ELF32BE>(Ctx &, uint8_t *Buf, Partition &Part);
+template void elf::writeEhdr<ELF64LE>(Ctx &, uint8_t *Buf, Partition &Part);
+template void elf::writeEhdr<ELF64BE>(Ctx &, uint8_t *Buf, Partition &Part);
 
 template void elf::writePhdrs<ELF32LE>(uint8_t *Buf, Partition &Part);
 template void elf::writePhdrs<ELF32BE>(uint8_t *Buf, Partition &Part);
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 421ef760ef4a092ac08d9090840f298831bf6665..7ddbaff5573fdbd0dc38cc2c389c50962f77f5c1 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -411,14 +411,14 @@ class DynamicReloc {
 public:
   enum Kind {
     /// The resulting dynamic relocation does not reference a symbol (#sym must
-    /// be nullptr) and uses #addend as the result of computeAddend().
+    /// be nullptr) and uses #addend as the result of computeAddend(ctx).
     AddendOnly,
     /// The resulting dynamic relocation will not reference a symbol: #sym is
     /// only used to compute the addend with InputSection::getRelocTargetVA().
     /// Useful for various relative and TLS relocations (e.g. R_X86_64_TPOFF64).
     AddendOnlyWithTargetVA,
     /// The resulting dynamic relocation references symbol #sym from the dynamic
-    /// symbol table and uses #addend as the value of computeAddend().
+    /// symbol table and uses #addend as the value of computeAddend(ctx).
     AgainstSymbol,
     /// The resulting dynamic relocation references symbol #sym from the dynamic
     /// symbol table and uses InputSection::getRelocTargetVA() + #addend for the
@@ -458,9 +458,9 @@ public:
   /// Computes the addend of the dynamic relocation. Note that this is not the
   /// same as the #addend member variable as it may also include the symbol
   /// address/the address of the corresponding GOT entry/etc.
-  int64_t computeAddend() const;
+  int64_t computeAddend(Ctx &) const;
 
-  void computeRaw(SymbolTableBaseSection *symt);
+  void computeRaw(Ctx &, SymbolTableBaseSection *symt);
 
   Symbol *sym;
   const OutputSection *outputSec = nullptr;
@@ -1429,23 +1429,25 @@ private:
 
 template <class ELFT> void createSyntheticSections(Ctx &);
 InputSection *createInterpSection(Ctx &);
-MergeInputSection *createCommentSection();
+MergeInputSection *createCommentSection(Ctx &);
 template <class ELFT> void splitSections(Ctx &);
 void combineEhSections(Ctx &);
 
-bool hasMemtag();
-bool canHaveMemtagGlobals();
+bool hasMemtag(Ctx &);
+bool canHaveMemtagGlobals(Ctx &);
 
-template <typename ELFT> void writeEhdr(uint8_t *buf, Partition &part);
+template <typename ELFT> void writeEhdr(Ctx &, uint8_t *buf, Partition &part);
 template <typename ELFT> void writePhdrs(uint8_t *buf, Partition &part);
 
-Defined *addSyntheticLocal(StringRef name, uint8_t type, uint64_t value,
-                           uint64_t size, InputSectionBase &section);
+Defined *addSyntheticLocal(Ctx &ctx, StringRef name, uint8_t type,
+                           uint64_t value, uint64_t size,
+                           InputSectionBase &section);
 
-void addVerneed(Symbol *ss);
+void addVerneed(Ctx &, Symbol &ss);
 
 // Linker generated per-partition sections.
 struct Partition {
+  Ctx &ctx;
   StringRef name;
   uint64_t nameStrTab;
 
@@ -1472,6 +1474,7 @@ struct Partition {
   std::unique_ptr<SyntheticSection> verNeed;
   std::unique_ptr<VersionTableSection> verSym;
 
+  Partition(Ctx &ctx) : ctx(ctx) {}
   unsigned getNumber() const { return this - &ctx.partitions[0] + 1; }
 };
 
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index 8a415c84cb57429da660ad1b7e774ca06c5628ee..27986b0a127e8ebec56003dfc3814c42b245b386 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -261,20 +261,22 @@ void reportRangeError(Ctx &ctx, uint8_t *loc, int64_t v, int n,
                       const Symbol &sym, const Twine &msg);
 
 // Make sure that V can be represented as an N bit signed integer.
-inline void checkInt(uint8_t *loc, int64_t v, int n, const Relocation &rel) {
+inline void checkInt(Ctx &ctx, uint8_t *loc, int64_t v, int n,
+                     const Relocation &rel) {
   if (v != llvm::SignExtend64(v, n))
     reportRangeError(ctx, loc, rel, Twine(v), llvm::minIntN(n),
                      llvm::maxIntN(n));
 }
 
 // Make sure that V can be represented as an N bit unsigned integer.
-inline void checkUInt(uint8_t *loc, uint64_t v, int n, const Relocation &rel) {
+inline void checkUInt(Ctx &ctx, uint8_t *loc, uint64_t v, int n,
+                      const Relocation &rel) {
   if ((v >> n) != 0)
     reportRangeError(ctx, loc, rel, Twine(v), 0, llvm::maxUIntN(n));
 }
 
 // Make sure that V can be represented as an N bit signed or unsigned integer.
-inline void checkIntUInt(uint8_t *loc, uint64_t v, int n,
+inline void checkIntUInt(Ctx &ctx, uint8_t *loc, uint64_t v, int n,
                          const Relocation &rel) {
   // For the error message we should cast V to a signed integer so that error
   // messages show a small negative value rather than an extremely large one
@@ -283,7 +285,7 @@ inline void checkIntUInt(uint8_t *loc, uint64_t v, int n,
                      llvm::maxUIntN(n));
 }
 
-inline void checkAlignment(uint8_t *loc, uint64_t v, int n,
+inline void checkAlignment(Ctx &ctx, uint8_t *loc, uint64_t v, int n,
                            const Relocation &rel) {
   if ((v & (n - 1)) != 0)
     error(getErrorLoc(ctx, loc) + "improper alignment for relocation " +
@@ -292,27 +294,27 @@ inline void checkAlignment(uint8_t *loc, uint64_t v, int n,
 }
 
 // Endianness-aware read/write.
-inline uint16_t read16(const void *p) {
+inline uint16_t read16(Ctx &ctx, const void *p) {
   return llvm::support::endian::read16(p, ctx.arg.endianness);
 }
 
-inline uint32_t read32(const void *p) {
+inline uint32_t read32(Ctx &ctx, const void *p) {
   return llvm::support::endian::read32(p, ctx.arg.endianness);
 }
 
-inline uint64_t read64(const void *p) {
+inline uint64_t read64(Ctx &ctx, const void *p) {
   return llvm::support::endian::read64(p, ctx.arg.endianness);
 }
 
-inline void write16(void *p, uint16_t v) {
+inline void write16(Ctx &ctx, void *p, uint16_t v) {
   llvm::support::endian::write16(p, v, ctx.arg.endianness);
 }
 
-inline void write32(void *p, uint32_t v) {
+inline void write32(Ctx &ctx, void *p, uint32_t v) {
   llvm::support::endian::write32(p, v, ctx.arg.endianness);
 }
 
-inline void write64(void *p, uint64_t v) {
+inline void write64(Ctx &ctx, void *p, uint64_t v) {
   llvm::support::endian::write64(p, v, ctx.arg.endianness);
 }
 
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index 19fe25a2688f4a19f469ab732262d33711003fbe..971b2724b3e26f794bea9687ad861f8739d3ecf5 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -537,7 +537,7 @@ public:
 
 Defined *Thunk::addSymbol(StringRef name, uint8_t type, uint64_t value,
                           InputSectionBase &section) {
-  Defined *d = addSyntheticLocal(name, type, value, /*size=*/0, section);
+  Defined *d = addSyntheticLocal(ctx, name, type, value, /*size=*/0, section);
   syms.push_back(d);
   return d;
 }
@@ -570,7 +570,7 @@ void AArch64Thunk::writeTo(uint8_t *buf) {
   }
   uint64_t s = getAArch64ThunkDestVA(ctx, destination, addend);
   uint64_t p = getThunkTargetSym()->getVA();
-  write32(buf, 0x14000000); // b S
+  write32(ctx, buf, 0x14000000); // b S
   ctx.target->relocateNoSym(buf, R_AARCH64_CALL26, s - p);
 }
 
@@ -647,7 +647,7 @@ void AArch64BTILandingPadThunk::writeTo(uint8_t *buf) {
     writeLong(buf);
     return;
   }
-  write32(buf, 0xd503245f); // BTI c
+  write32(ctx, buf, 0xd503245f); // BTI c
   // Control falls through to target in following section.
 }
 
@@ -672,8 +672,8 @@ bool AArch64BTILandingPadThunk::getMayUseShortThunk() {
 void AArch64BTILandingPadThunk::writeLong(uint8_t *buf) {
   uint64_t s = destination.getVA(addend);
   uint64_t p = getThunkTargetSym()->getVA() + 4;
-  write32(buf, 0xd503245f);     // BTI c
-  write32(buf + 4, 0x14000000); // B S
+  write32(ctx, buf, 0xd503245f);     // BTI c
+  write32(ctx, buf + 4, 0x14000000); // B S
   ctx.target->relocateNoSym(buf + 4, R_AARCH64_CALL26, s - p);
 }
 
@@ -708,7 +708,7 @@ void ARMThunk::writeTo(uint8_t *buf) {
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA();
   int64_t offset = s - p - 8;
-  write32(buf, 0xea000000); // b S
+  write32(ctx, buf, 0xea000000); // b S
   ctx.target->relocateNoSym(buf, R_ARM_JUMP24, offset);
 }
 
@@ -751,8 +751,8 @@ void ThumbThunk::writeTo(uint8_t *buf) {
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA();
   int64_t offset = s - p - 4;
-  write16(buf + 0, 0xf000); // b.w S
-  write16(buf + 2, 0xb000);
+  write16(ctx, buf + 0, 0xf000); // b.w S
+  write16(ctx, buf + 2, 0xb000);
   ctx.target->relocateNoSym(buf, R_ARM_THM_JUMP24, offset);
 }
 
@@ -767,9 +767,9 @@ bool ThumbThunk::isCompatibleWith(const InputSection &isec,
 }
 
 void ARMV7ABSLongThunk::writeLong(uint8_t *buf) {
-  write32(buf + 0, 0xe300c000); // movw ip,:lower16:S
-  write32(buf + 4, 0xe340c000); // movt ip,:upper16:S
-  write32(buf + 8, 0xe12fff1c); // bx   ip
+  write32(ctx, buf + 0, 0xe300c000); // movw ip,:lower16:S
+  write32(ctx, buf + 4, 0xe340c000); // movt ip,:upper16:S
+  write32(ctx, buf + 8, 0xe12fff1c); // bx   ip
   uint64_t s = getARMThunkDestVA(ctx, destination);
   ctx.target->relocateNoSym(buf, R_ARM_MOVW_ABS_NC, s);
   ctx.target->relocateNoSym(buf + 4, R_ARM_MOVT_ABS, s);
@@ -782,11 +782,11 @@ void ARMV7ABSLongThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ThumbV7ABSLongThunk::writeLong(uint8_t *buf) {
-  write16(buf + 0, 0xf240); // movw ip, :lower16:S
-  write16(buf + 2, 0x0c00);
-  write16(buf + 4, 0xf2c0); // movt ip, :upper16:S
-  write16(buf + 6, 0x0c00);
-  write16(buf + 8, 0x4760); // bx   ip
+  write16(ctx, buf + 0, 0xf240); // movw ip, :lower16:S
+  write16(ctx, buf + 2, 0x0c00);
+  write16(ctx, buf + 4, 0xf2c0); // movt ip, :upper16:S
+  write16(ctx, buf + 6, 0x0c00);
+  write16(ctx, buf + 8, 0x4760); // bx   ip
   uint64_t s = getARMThunkDestVA(ctx, destination);
   ctx.target->relocateNoSym(buf, R_ARM_THM_MOVW_ABS_NC, s);
   ctx.target->relocateNoSym(buf + 4, R_ARM_THM_MOVT_ABS, s);
@@ -799,10 +799,12 @@ void ThumbV7ABSLongThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ARMV7PILongThunk::writeLong(uint8_t *buf) {
-  write32(buf + 0, 0xe30fcff0);   // P:  movw ip,:lower16:S - (P + (L1-P) + 8)
-  write32(buf + 4, 0xe340c000);   //     movt ip,:upper16:S - (P + (L1-P) + 8)
-  write32(buf + 8, 0xe08cc00f);   // L1: add  ip, ip, pc
-  write32(buf + 12, 0xe12fff1c);  //     bx   ip
+  write32(ctx, buf + 0,
+          0xe30fcff0); // P:  movw ip,:lower16:S - (P + (L1-P) + 8)
+  write32(ctx, buf + 4,
+          0xe340c000); //     movt ip,:upper16:S - (P + (L1-P) + 8)
+  write32(ctx, buf + 8, 0xe08cc00f);  // L1: add  ip, ip, pc
+  write32(ctx, buf + 12, 0xe12fff1c); //     bx   ip
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA();
   int64_t offset = s - p - 16;
@@ -817,12 +819,12 @@ void ARMV7PILongThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ThumbV7PILongThunk::writeLong(uint8_t *buf) {
-  write16(buf + 0, 0xf64f);   // P:  movw ip,:lower16:S - (P + (L1-P) + 4)
-  write16(buf + 2, 0x7cf4);
-  write16(buf + 4, 0xf2c0);   //     movt ip,:upper16:S - (P + (L1-P) + 4)
-  write16(buf + 6, 0x0c00);
-  write16(buf + 8, 0x44fc);   // L1: add  ip, pc
-  write16(buf + 10, 0x4760);  //     bx   ip
+  write16(ctx, buf + 0, 0xf64f); // P:  movw ip,:lower16:S - (P + (L1-P) + 4)
+  write16(ctx, buf + 2, 0x7cf4);
+  write16(ctx, buf + 4, 0xf2c0); //     movt ip,:upper16:S - (P + (L1-P) + 4)
+  write16(ctx, buf + 6, 0x0c00);
+  write16(ctx, buf + 8, 0x44fc);  // L1: add  ip, pc
+  write16(ctx, buf + 10, 0x4760); //     bx   ip
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA() & ~0x1;
   int64_t offset = s - p - 12;
@@ -841,11 +843,11 @@ void ThumbV6MABSLongThunk::writeLong(uint8_t *buf) {
   // only register we can corrupt is r12 we must instead spill a low register
   // to the stack to use as a scratch register. We push r1 even though we
   // don't need to get some space to use for the return address.
-  write16(buf + 0, 0xb403);   // push {r0, r1} ; Obtain scratch registers
-  write16(buf + 2, 0x4801);   // ldr r0, [pc, #4] ; L1
-  write16(buf + 4, 0x9001);   // str r0, [sp, #4] ; SP + 4 = S
-  write16(buf + 6, 0xbd01);   // pop {r0, pc} ; restore r0 and branch to dest
-  write32(buf + 8, 0x00000000);   // L1: .word S
+  write16(ctx, buf + 0, 0xb403); // push {r0, r1} ; Obtain scratch registers
+  write16(ctx, buf + 2, 0x4801); // ldr r0, [pc, #4] ; L1
+  write16(ctx, buf + 4, 0x9001); // str r0, [sp, #4] ; SP + 4 = S
+  write16(ctx, buf + 6, 0xbd01); // pop {r0, pc} ; restore r0 and branch to dest
+  write32(ctx, buf + 8, 0x00000000); // L1: .word S
   uint64_t s = getARMThunkDestVA(ctx, destination);
   ctx.target->relocateNoSym(buf + 8, R_ARM_ABS32, s);
 }
@@ -863,16 +865,17 @@ void ThumbV6MABSXOLongThunk::writeLong(uint8_t *buf) {
   // only register we can corrupt is r12 we must instead spill a low register
   // to the stack to use as a scratch register. We push r1 even though we
   // don't need to get some space to use for the return address.
-  write16(buf + 0, 0xb403);  // push {r0, r1} ; Obtain scratch registers
-  write16(buf + 2, 0x2000);  // movs r0, :upper8_15:S
-  write16(buf + 4, 0x0200);  // lsls r0, r0, #8
-  write16(buf + 6, 0x3000);  // adds r0, :upper0_7:S
-  write16(buf + 8, 0x0200);  // lsls r0, r0, #8
-  write16(buf + 10, 0x3000); // adds r0, :lower8_15:S
-  write16(buf + 12, 0x0200); // lsls r0, r0, #8
-  write16(buf + 14, 0x3000); // adds r0, :lower0_7:S
-  write16(buf + 16, 0x9001); // str r0, [sp, #4] ; SP + 4 = S
-  write16(buf + 18, 0xbd01); // pop {r0, pc} ; restore r0 and branch to dest
+  write16(ctx, buf + 0, 0xb403);  // push {r0, r1} ; Obtain scratch registers
+  write16(ctx, buf + 2, 0x2000);  // movs r0, :upper8_15:S
+  write16(ctx, buf + 4, 0x0200);  // lsls r0, r0, #8
+  write16(ctx, buf + 6, 0x3000);  // adds r0, :upper0_7:S
+  write16(ctx, buf + 8, 0x0200);  // lsls r0, r0, #8
+  write16(ctx, buf + 10, 0x3000); // adds r0, :lower8_15:S
+  write16(ctx, buf + 12, 0x0200); // lsls r0, r0, #8
+  write16(ctx, buf + 14, 0x3000); // adds r0, :lower0_7:S
+  write16(ctx, buf + 16, 0x9001); // str r0, [sp, #4] ; SP + 4 = S
+  write16(ctx, buf + 18,
+          0xbd01); // pop {r0, pc} ; restore r0 and branch to dest
   uint64_t s = getARMThunkDestVA(ctx, destination);
   ctx.target->relocateNoSym(buf + 2, R_ARM_THM_ALU_ABS_G3, s);
   ctx.target->relocateNoSym(buf + 6, R_ARM_THM_ALU_ABS_G2_NC, s);
@@ -890,13 +893,16 @@ void ThumbV6MPILongThunk::writeLong(uint8_t *buf) {
   // Most Thumb instructions cannot access the high registers r8 - r15. As the
   // only register we can corrupt is ip (r12) we must instead spill a low
   // register to the stack to use as a scratch register.
-  write16(buf + 0, 0xb401);   // P:  push {r0}        ; Obtain scratch register
-  write16(buf + 2, 0x4802);   //     ldr r0, [pc, #8] ; L2
-  write16(buf + 4, 0x4684);   //     mov ip, r0       ; high to low register
-  write16(buf + 6, 0xbc01);   //     pop {r0}         ; restore scratch register
-  write16(buf + 8, 0x44e7);   // L1: add pc, ip       ; transfer control
-  write16(buf + 10, 0x46c0);  //     nop              ; pad to 4-byte boundary
-  write32(buf + 12, 0x00000000);  // L2: .word S - (P + (L1 - P) + 4)
+  write16(ctx, buf + 0,
+          0xb401); // P:  push {r0}        ; Obtain scratch register
+  write16(ctx, buf + 2, 0x4802); //     ldr r0, [pc, #8] ; L2
+  write16(ctx, buf + 4, 0x4684); //     mov ip, r0       ; high to low register
+  write16(ctx, buf + 6,
+          0xbc01); //     pop {r0}         ; restore scratch register
+  write16(ctx, buf + 8, 0x44e7); // L1: add pc, ip       ; transfer control
+  write16(ctx, buf + 10,
+          0x46c0); //     nop              ; pad to 4-byte boundary
+  write32(ctx, buf + 12, 0x00000000); // L2: .word S - (P + (L1 - P) + 4)
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA() & ~0x1;
   ctx.target->relocateNoSym(buf + 12, R_ARM_REL32, s - p - 12);
@@ -911,8 +917,8 @@ void ThumbV6MPILongThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ARMV5LongLdrPcThunk::writeLong(uint8_t *buf) {
-  write32(buf + 0, 0xe51ff004); // ldr pc, [pc,#-4] ; L1
-  write32(buf + 4, 0x00000000); // L1: .word S
+  write32(ctx, buf + 0, 0xe51ff004); // ldr pc, [pc,#-4] ; L1
+  write32(ctx, buf + 4, 0x00000000); // L1: .word S
   ctx.target->relocateNoSym(buf + 4, R_ARM_ABS32,
                             getARMThunkDestVA(ctx, destination));
 }
@@ -926,9 +932,9 @@ void ARMV5LongLdrPcThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ARMV4ABSLongBXThunk::writeLong(uint8_t *buf) {
-  write32(buf + 0, 0xe59fc000); // ldr r12, [pc] ; L1
-  write32(buf + 4, 0xe12fff1c); // bx r12
-  write32(buf + 8, 0x00000000); // L1: .word S
+  write32(ctx, buf + 0, 0xe59fc000); // ldr r12, [pc] ; L1
+  write32(ctx, buf + 4, 0xe12fff1c); // bx r12
+  write32(ctx, buf + 8, 0x00000000); // L1: .word S
   ctx.target->relocateNoSym(buf + 8, R_ARM_ABS32,
                             getARMThunkDestVA(ctx, destination));
 }
@@ -942,10 +948,11 @@ void ARMV4ABSLongBXThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ThumbV4ABSLongBXThunk::writeLong(uint8_t *buf) {
-  write16(buf + 0, 0x4778); // bx pc
-  write16(buf + 2, 0xe7fd); // b #-6 ; Arm recommended sequence to follow bx pc
-  write32(buf + 4, 0xe51ff004); // ldr pc, [pc, #-4] ; L1
-  write32(buf + 8, 0x00000000); // L1: .word S
+  write16(ctx, buf + 0, 0x4778); // bx pc
+  write16(ctx, buf + 2,
+          0xe7fd); // b #-6 ; Arm recommended sequence to follow bx pc
+  write32(ctx, buf + 4, 0xe51ff004); // ldr pc, [pc, #-4] ; L1
+  write32(ctx, buf + 8, 0x00000000); // L1: .word S
   ctx.target->relocateNoSym(buf + 8, R_ARM_ABS32,
                             getARMThunkDestVA(ctx, destination));
 }
@@ -960,11 +967,12 @@ void ThumbV4ABSLongBXThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ThumbV4ABSLongThunk::writeLong(uint8_t *buf) {
-  write16(buf + 0, 0x4778); // bx pc
-  write16(buf + 2, 0xe7fd); // b #-6 ; Arm recommended sequence to follow bx pc
-  write32(buf + 4, 0xe59fc000); // ldr r12, [pc] ; L1
-  write32(buf + 8, 0xe12fff1c); // bx r12
-  write32(buf + 12, 0x00000000); // L1: .word S
+  write16(ctx, buf + 0, 0x4778); // bx pc
+  write16(ctx, buf + 2,
+          0xe7fd); // b #-6 ; Arm recommended sequence to follow bx pc
+  write32(ctx, buf + 4, 0xe59fc000);  // ldr r12, [pc] ; L1
+  write32(ctx, buf + 8, 0xe12fff1c);  // bx r12
+  write32(ctx, buf + 12, 0x00000000); // L1: .word S
   ctx.target->relocateNoSym(buf + 12, R_ARM_ABS32,
                             getARMThunkDestVA(ctx, destination));
 }
@@ -979,10 +987,10 @@ void ThumbV4ABSLongThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ARMV4PILongBXThunk::writeLong(uint8_t *buf) {
-  write32(buf + 0, 0xe59fc004); // P:  ldr ip, [pc,#4] ; L2
-  write32(buf + 4, 0xe08fc00c);	// L1: add ip, pc, ip
-  write32(buf + 8, 0xe12fff1c);	//     bx ip
-  write32(buf + 12, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
+  write32(ctx, buf + 0, 0xe59fc004);  // P:  ldr ip, [pc,#4] ; L2
+  write32(ctx, buf + 4, 0xe08fc00c);  // L1: add ip, pc, ip
+  write32(ctx, buf + 8, 0xe12fff1c);  //     bx ip
+  write32(ctx, buf + 12, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA() & ~0x1;
   ctx.target->relocateNoSym(buf + 12, R_ARM_REL32, s - p - 12);
@@ -997,9 +1005,9 @@ void ARMV4PILongBXThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ARMV4PILongThunk::writeLong(uint8_t *buf) {
-  write32(buf + 0, 0xe59fc000); // P:  ldr ip, [pc] ; L2
-  write32(buf + 4, 0xe08ff00c); // L1: add pc, pc, r12
-  write32(buf + 8, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
+  write32(ctx, buf + 0, 0xe59fc000); // P:  ldr ip, [pc] ; L2
+  write32(ctx, buf + 4, 0xe08ff00c); // L1: add pc, pc, r12
+  write32(ctx, buf + 8, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA() & ~0x1;
   ctx.target->relocateNoSym(buf + 8, R_ARM_REL32, s - p - 12);
@@ -1014,11 +1022,12 @@ void ARMV4PILongThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ThumbV4PILongBXThunk::writeLong(uint8_t *buf) {
-  write16(buf + 0, 0x4778); // P:  bx pc
-  write16(buf + 2, 0xe7fd); //     b #-6 ; Arm recommended sequence to follow bx pc
-  write32(buf + 4, 0xe59fc000); //     ldr r12, [pc] ; L2
-  write32(buf + 8, 0xe08cf00f); // L1: add pc, r12, pc
-  write32(buf + 12, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
+  write16(ctx, buf + 0, 0x4778); // P:  bx pc
+  write16(ctx, buf + 2,
+          0xe7fd); //     b #-6 ; Arm recommended sequence to follow bx pc
+  write32(ctx, buf + 4, 0xe59fc000);  //     ldr r12, [pc] ; L2
+  write32(ctx, buf + 8, 0xe08cf00f);  // L1: add pc, r12, pc
+  write32(ctx, buf + 12, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA() & ~0x1;
   ctx.target->relocateNoSym(buf + 12, R_ARM_REL32, s - p - 16);
@@ -1034,12 +1043,13 @@ void ThumbV4PILongBXThunk::addSymbols(ThunkSection &isec) {
 }
 
 void ThumbV4PILongThunk::writeLong(uint8_t *buf) {
-  write16(buf + 0, 0x4778); // P:  bx pc
-  write16(buf + 2, 0xe7fd); //     b #-6 ; Arm recommended sequence to follow bx pc
-  write32(buf + 4, 0xe59fc004); //     ldr ip, [pc,#4] ; L2
-  write32(buf + 8, 0xe08fc00c); // L1: add ip, pc, ip
-  write32(buf + 12, 0xe12fff1c); //     bx ip
-  write32(buf + 16, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
+  write16(ctx, buf + 0, 0x4778); // P:  bx pc
+  write16(ctx, buf + 2,
+          0xe7fd); //     b #-6 ; Arm recommended sequence to follow bx pc
+  write32(ctx, buf + 4, 0xe59fc004);  //     ldr ip, [pc,#4] ; L2
+  write32(ctx, buf + 8, 0xe08fc00c);  // L1: add ip, pc, ip
+  write32(ctx, buf + 12, 0xe12fff1c); //     bx ip
+  write32(ctx, buf + 16, 0x00000000); // L2: .word S - (P + (L1 - P) + 8)
   uint64_t s = getARMThunkDestVA(ctx, destination);
   uint64_t p = getThunkTargetSym()->getVA() & ~0x1;
   ctx.target->relocateNoSym(buf + 16, R_ARM_REL32, s - p - 16);
@@ -1056,7 +1066,7 @@ void ThumbV4PILongThunk::addSymbols(ThunkSection &isec) {
 
 // Use the long jump which covers a range up to 8MiB.
 void AVRThunk::writeTo(uint8_t *buf) {
-  write32(buf, 0x940c); // jmp func
+  write32(ctx, buf, 0x940c); // jmp func
   ctx.target->relocateNoSym(buf, R_AVR_CALL, destination.getVA());
 }
 
@@ -1068,10 +1078,10 @@ void AVRThunk::addSymbols(ThunkSection &isec) {
 // Write MIPS LA25 thunk code to call PIC function from the non-PIC one.
 void MipsThunk::writeTo(uint8_t *buf) {
   uint64_t s = destination.getVA();
-  write32(buf, 0x3c190000); // lui   $25, %hi(func)
-  write32(buf + 4, 0x08000000 | (s >> 2)); // j     func
-  write32(buf + 8, 0x27390000); // addiu $25, $25, %lo(func)
-  write32(buf + 12, 0x00000000); // nop
+  write32(ctx, buf, 0x3c190000);                // lui   $25, %hi(func)
+  write32(ctx, buf + 4, 0x08000000 | (s >> 2)); // j     func
+  write32(ctx, buf + 8, 0x27390000);            // addiu $25, $25, %lo(func)
+  write32(ctx, buf + 12, 0x00000000);           // nop
   ctx.target->relocateNoSym(buf, R_MIPS_HI16, s);
   ctx.target->relocateNoSym(buf + 8, R_MIPS_LO16, s);
 }
@@ -1090,10 +1100,10 @@ InputSection *MipsThunk::getTargetInputSection() const {
 // to call PIC function from the non-PIC one.
 void MicroMipsThunk::writeTo(uint8_t *buf) {
   uint64_t s = destination.getVA();
-  write16(buf, 0x41b9);       // lui   $25, %hi(func)
-  write16(buf + 4, 0xd400);   // j     func
-  write16(buf + 8, 0x3339);   // addiu $25, $25, %lo(func)
-  write16(buf + 12, 0x0c00);  // nop
+  write16(ctx, buf, 0x41b9);      // lui   $25, %hi(func)
+  write16(ctx, buf + 4, 0xd400);  // j     func
+  write16(ctx, buf + 8, 0x3339);  // addiu $25, $25, %lo(func)
+  write16(ctx, buf + 12, 0x0c00); // nop
   ctx.target->relocateNoSym(buf, R_MICROMIPS_HI16, s);
   ctx.target->relocateNoSym(buf + 4, R_MICROMIPS_26_S1, s);
   ctx.target->relocateNoSym(buf + 8, R_MICROMIPS_LO16, s);
@@ -1116,9 +1126,9 @@ InputSection *MicroMipsThunk::getTargetInputSection() const {
 void MicroMipsR6Thunk::writeTo(uint8_t *buf) {
   uint64_t s = destination.getVA();
   uint64_t p = getThunkTargetSym()->getVA();
-  write16(buf, 0x1320);       // lui   $25, %hi(func)
-  write16(buf + 4, 0x3339);   // addiu $25, $25, %lo(func)
-  write16(buf + 8, 0x9400);   // bc    func
+  write16(ctx, buf, 0x1320);     // lui   $25, %hi(func)
+  write16(ctx, buf + 4, 0x3339); // addiu $25, $25, %lo(func)
+  write16(ctx, buf + 8, 0x9400); // bc    func
   ctx.target->relocateNoSym(buf, R_MICROMIPS_HI16, s);
   ctx.target->relocateNoSym(buf + 4, R_MICROMIPS_LO16, s);
   ctx.target->relocateNoSym(buf + 8, R_MICROMIPS_PC26_S1, s - p - 12);
@@ -1139,10 +1149,10 @@ InputSection *MicroMipsR6Thunk::getTargetInputSection() const {
 void elf::writePPC32PltCallStub(Ctx &ctx, uint8_t *buf, uint64_t gotPltVA,
                                 const InputFile *file, int64_t addend) {
   if (!ctx.arg.isPic) {
-    write32(buf + 0, 0x3d600000 | (gotPltVA + 0x8000) >> 16); // lis r11,ha
-    write32(buf + 4, 0x816b0000 | (uint16_t)gotPltVA);        // lwz r11,l(r11)
-    write32(buf + 8, 0x7d6903a6);                             // mtctr r11
-    write32(buf + 12, 0x4e800420);                            // bctr
+    write32(ctx, buf + 0, 0x3d600000 | (gotPltVA + 0x8000) >> 16); // lis r11,ha
+    write32(ctx, buf + 4, 0x816b0000 | (uint16_t)gotPltVA); // lwz r11,l(r11)
+    write32(ctx, buf + 8, 0x7d6903a6);                      // mtctr r11
+    write32(ctx, buf + 12, 0x4e800420);                     // bctr
     return;
   }
   uint32_t offset;
@@ -1160,15 +1170,15 @@ void elf::writePPC32PltCallStub(Ctx &ctx, uint8_t *buf, uint64_t gotPltVA,
   }
   uint16_t ha = (offset + 0x8000) >> 16, l = (uint16_t)offset;
   if (ha == 0) {
-    write32(buf + 0, 0x817e0000 | l); // lwz r11,l(r30)
-    write32(buf + 4, 0x7d6903a6);     // mtctr r11
-    write32(buf + 8, 0x4e800420);     // bctr
-    write32(buf + 12, 0x60000000);    // nop
+    write32(ctx, buf + 0, 0x817e0000 | l); // lwz r11,l(r30)
+    write32(ctx, buf + 4, 0x7d6903a6);     // mtctr r11
+    write32(ctx, buf + 8, 0x4e800420);     // bctr
+    write32(ctx, buf + 12, 0x60000000);    // nop
   } else {
-    write32(buf + 0, 0x3d7e0000 | ha); // addis r11,r30,ha
-    write32(buf + 4, 0x816b0000 | l);  // lwz r11,l(r11)
-    write32(buf + 8, 0x7d6903a6);      // mtctr r11
-    write32(buf + 12, 0x4e800420);     // bctr
+    write32(ctx, buf + 0, 0x3d7e0000 | ha); // addis r11,r30,ha
+    write32(ctx, buf + 4, 0x816b0000 | l);  // lwz r11,l(r11)
+    write32(ctx, buf + 8, 0x7d6903a6);      // mtctr r11
+    write32(ctx, buf + 12, 0x4e800420);     // bctr
   }
 }
 
@@ -1206,37 +1216,37 @@ void PPC32LongThunk::writeTo(uint8_t *buf) {
   uint32_t d = destination.getVA(addend);
   if (ctx.arg.isPic) {
     uint32_t off = d - (getThunkTargetSym()->getVA() + 8);
-    write32(buf + 0, 0x7c0802a6);            // mflr r12,0
-    write32(buf + 4, 0x429f0005);            // bcl r20,r31,.+4
-    write32(buf + 8, 0x7d8802a6);            // mtctr r12
-    write32(buf + 12, 0x3d8c0000 | ha(off)); // addis r12,r12,off@ha
-    write32(buf + 16, 0x398c0000 | lo(off)); // addi r12,r12,off@l
-    write32(buf + 20, 0x7c0803a6);           // mtlr r0
+    write32(ctx, buf + 0, 0x7c0802a6);            // mflr r12,0
+    write32(ctx, buf + 4, 0x429f0005);            // bcl r20,r31,.+4
+    write32(ctx, buf + 8, 0x7d8802a6);            // mtctr r12
+    write32(ctx, buf + 12, 0x3d8c0000 | ha(off)); // addis r12,r12,off@ha
+    write32(ctx, buf + 16, 0x398c0000 | lo(off)); // addi r12,r12,off@l
+    write32(ctx, buf + 20, 0x7c0803a6);           // mtlr r0
     buf += 24;
   } else {
-    write32(buf + 0, 0x3d800000 | ha(d));    // lis r12,d@ha
-    write32(buf + 4, 0x398c0000 | lo(d));    // addi r12,r12,d@l
+    write32(ctx, buf + 0, 0x3d800000 | ha(d)); // lis r12,d@ha
+    write32(ctx, buf + 4, 0x398c0000 | lo(d)); // addi r12,r12,d@l
     buf += 8;
   }
-  write32(buf + 0, 0x7d8903a6);              // mtctr r12
-  write32(buf + 4, 0x4e800420);              // bctr
+  write32(ctx, buf + 0, 0x7d8903a6); // mtctr r12
+  write32(ctx, buf + 4, 0x4e800420); // bctr
 }
 
-void elf::writePPC64LoadAndBranch(uint8_t *buf, int64_t offset) {
+void elf::writePPC64LoadAndBranch(Ctx &ctx, uint8_t *buf, int64_t offset) {
   uint16_t offHa = (offset + 0x8000) >> 16;
   uint16_t offLo = offset & 0xffff;
 
-  write32(buf + 0, 0x3d820000 | offHa); // addis r12, r2, OffHa
-  write32(buf + 4, 0xe98c0000 | offLo); // ld    r12, OffLo(r12)
-  write32(buf + 8, 0x7d8903a6);         // mtctr r12
-  write32(buf + 12, 0x4e800420);        // bctr
+  write32(ctx, buf + 0, 0x3d820000 | offHa); // addis r12, r2, OffHa
+  write32(ctx, buf + 4, 0xe98c0000 | offLo); // ld    r12, OffLo(r12)
+  write32(ctx, buf + 8, 0x7d8903a6);         // mtctr r12
+  write32(ctx, buf + 12, 0x4e800420);        // bctr
 }
 
 void PPC64PltCallStub::writeTo(uint8_t *buf) {
   int64_t offset = destination.getGotPltVA(ctx) - getPPC64TocBase(ctx);
   // Save the TOC pointer to the save-slot reserved in the call frame.
-  write32(buf + 0, 0xf8410018); // std     r2,24(r1)
-  writePPC64LoadAndBranch(buf + 4, offset);
+  write32(ctx, buf + 0, 0xf8410018); // std     r2,24(r1)
+  writePPC64LoadAndBranch(ctx, buf + 4, offset);
 }
 
 void PPC64PltCallStub::addSymbols(ThunkSection &isec) {
@@ -1253,10 +1263,10 @@ bool PPC64PltCallStub::isCompatibleWith(const InputSection &isec,
 
 void PPC64R2SaveStub::writeTo(uint8_t *buf) {
   const int64_t offset = computeOffset();
-  write32(buf + 0, 0xf8410018); // std  r2,24(r1)
+  write32(ctx, buf + 0, 0xf8410018); // std  r2,24(r1)
   // The branch offset needs to fit in 26 bits.
   if (getMayUseShortThunk()) {
-    write32(buf + 4, 0x48000000 | (offset & 0x03fffffc)); // b    <offset>
+    write32(ctx, buf + 4, 0x48000000 | (offset & 0x03fffffc)); // b    <offset>
   } else if (isInt<34>(offset)) {
     int nextInstOffset;
     uint64_t tocOffset = destination.getVA() - getPPC64TocBase(ctx);
@@ -1264,22 +1274,22 @@ void PPC64R2SaveStub::writeTo(uint8_t *buf) {
       const uint64_t addi = ADDI_R12_TO_R12_NO_DISP | (tocOffset & 0xffff);
       const uint64_t addis =
           ADDIS_R12_TO_R2_NO_DISP | ((tocOffset >> 16) & 0xffff);
-      write32(buf + 4, addis); // addis r12, r2 , top of offset
-      write32(buf + 8, addi);  // addi  r12, r12, bottom of offset
+      write32(ctx, buf + 4, addis); // addis r12, r2 , top of offset
+      write32(ctx, buf + 8, addi);  // addi  r12, r12, bottom of offset
       nextInstOffset = 12;
     } else {
       const uint64_t addi = ADDI_R12_TO_R2_NO_DISP | (tocOffset & 0xffff);
-      write32(buf + 4, addi); // addi r12, r2, offset
+      write32(ctx, buf + 4, addi); // addi r12, r2, offset
       nextInstOffset = 8;
     }
-    write32(buf + nextInstOffset, MTCTR_R12); // mtctr r12
-    write32(buf + nextInstOffset + 4, BCTR);  // bctr
+    write32(ctx, buf + nextInstOffset, MTCTR_R12); // mtctr r12
+    write32(ctx, buf + nextInstOffset + 4, BCTR);  // bctr
   } else {
     ctx.in.ppc64LongBranchTarget->addEntry(&destination, addend);
     const int64_t offsetFromTOC =
         ctx.in.ppc64LongBranchTarget->getEntryVA(&destination, addend) -
         getPPC64TocBase(ctx);
-    writePPC64LoadAndBranch(buf + 4, offsetFromTOC);
+    writePPC64LoadAndBranch(ctx, buf + 4, offsetFromTOC);
   }
 }
 
@@ -1311,20 +1321,20 @@ void PPC64R12SetupStub::writeTo(uint8_t *buf) {
     nextInstOffset = 8;
   } else {
     uint32_t off = offset - 8;
-    write32(buf + 0, 0x7d8802a6);                     // mflr 12
-    write32(buf + 4, 0x429f0005);                     // bcl 20,31,.+4
-    write32(buf + 8, 0x7d6802a6);                     // mflr 11
-    write32(buf + 12, 0x7d8803a6);                    // mtlr 12
-    write32(buf + 16,
-            0x3d8b0000 | ((off + 0x8000) >> 16));     // addis 12,11,off@ha
+    write32(ctx, buf + 0, 0x7d8802a6);  // mflr 12
+    write32(ctx, buf + 4, 0x429f0005);  // bcl 20,31,.+4
+    write32(ctx, buf + 8, 0x7d6802a6);  // mflr 11
+    write32(ctx, buf + 12, 0x7d8803a6); // mtlr 12
+    write32(ctx, buf + 16,
+            0x3d8b0000 | ((off + 0x8000) >> 16)); // addis 12,11,off@ha
     if (gotPlt)
-      write32(buf + 20, 0xe98c0000 | (off & 0xffff)); // ld 12, off@l(12)
+      write32(ctx, buf + 20, 0xe98c0000 | (off & 0xffff)); // ld 12, off@l(12)
     else
-      write32(buf + 20, 0x398c0000 | (off & 0xffff)); // addi 12,12,off@l
+      write32(ctx, buf + 20, 0x398c0000 | (off & 0xffff)); // addi 12,12,off@l
     nextInstOffset = 24;
   }
-  write32(buf + nextInstOffset, MTCTR_R12); // mtctr r12
-  write32(buf + nextInstOffset + 4, BCTR);  // bctr
+  write32(ctx, buf + nextInstOffset, MTCTR_R12); // mtctr r12
+  write32(ctx, buf + nextInstOffset + 4, BCTR);  // bctr
 }
 
 void PPC64R12SetupStub::addSymbols(ThunkSection &isec) {
@@ -1342,7 +1352,7 @@ void PPC64LongBranchThunk::writeTo(uint8_t *buf) {
   int64_t offset =
       ctx.in.ppc64LongBranchTarget->getEntryVA(&destination, addend) -
       getPPC64TocBase(ctx);
-  writePPC64LoadAndBranch(buf, offset);
+  writePPC64LoadAndBranch(ctx, buf, offset);
 }
 
 void PPC64LongBranchThunk::addSymbols(ThunkSection &isec) {
diff --git a/lld/ELF/Thunks.h b/lld/ELF/Thunks.h
index 3929aa0aee8114831fcdc8c23f07f630e8e79b84..247b500580325e94a12a447dd274fc388f8871d6 100644
--- a/lld/ELF/Thunks.h
+++ b/lld/ELF/Thunks.h
@@ -84,7 +84,7 @@ Thunk *addLandingPadThunk(Ctx &, Symbol &s, int64_t a);
 
 void writePPC32PltCallStub(Ctx &, uint8_t *buf, uint64_t gotPltVA,
                            const InputFile *file, int64_t addend);
-void writePPC64LoadAndBranch(uint8_t *buf, int64_t offset);
+void writePPC64LoadAndBranch(Ctx &, uint8_t *buf, int64_t offset);
 
 } // namespace lld::elf
 
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index f9a21b6745fdd15f94fe956a5a657e045c529e04..49b6ab4a48b55a9e731691b0df49c03f958bab87 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -145,9 +145,9 @@ static Defined *addOptionalRegular(Ctx &ctx, StringRef name, SectionBase *sec,
   if (!s || s->isDefined() || s->isCommon())
     return nullptr;
 
-  s->resolve(Defined{ctx.internalFile, StringRef(), STB_GLOBAL, stOther,
-                     STT_NOTYPE, val,
-                     /*size=*/0, sec});
+  s->resolve(ctx, Defined{ctx.internalFile, StringRef(), STB_GLOBAL, stOther,
+                          STT_NOTYPE, val,
+                          /*size=*/0, sec});
   s->isUsedInRegularObj = true;
   return cast<Defined>(s);
 }
@@ -211,7 +211,8 @@ void elf::addReservedSymbols(Ctx &ctx) {
     if (ctx.arg.emachine == EM_PPC64)
       gotOff = 0x8000;
 
-    s->resolve(Defined{ctx.internalFile, StringRef(), STB_GLOBAL, STV_HIDDEN,
+    s->resolve(ctx,
+               Defined{ctx.internalFile, StringRef(), STB_GLOBAL, STV_HIDDEN,
                        STT_NOTYPE, gotOff, /*size=*/0, ctx.out.elfHeader});
     ctx.sym.globalOffsetTable = cast<Defined>(s);
   }
@@ -777,7 +778,7 @@ unsigned elf::getSectionRank(Ctx &ctx, OutputSection &osec) {
   return rank;
 }
 
-static bool compareSections(const SectionCommand *aCmd,
+static bool compareSections(Ctx &ctx, const SectionCommand *aCmd,
                             const SectionCommand *bCmd) {
   const OutputSection *a = &cast<OutputDesc>(aCmd)->osec;
   const OutputSection *b = &cast<OutputDesc>(bCmd)->osec;
@@ -1072,7 +1073,7 @@ static DenseMap<const InputSectionBase *, int> buildSectionOrder(Ctx &ctx) {
   DenseMap<const InputSectionBase *, int> sectionOrder;
   // Use the rarely used option --call-graph-ordering-file to sort sections.
   if (!ctx.arg.callGraphProfile.empty())
-    return computeCallGraphProfileOrder();
+    return computeCallGraphProfileOrder(ctx);
 
   if (ctx.arg.symbolOrderingFile.empty())
     return sectionOrder;
@@ -1098,7 +1099,7 @@ static DenseMap<const InputSectionBase *, int> buildSectionOrder(Ctx &ctx) {
     SymbolOrderEntry &ent = it->second;
     ent.present = true;
 
-    maybeWarnUnorderableSymbol(&sym);
+    maybeWarnUnorderableSymbol(ctx, &sym);
 
     if (auto *d = dyn_cast<Defined>(&sym)) {
       if (auto *sec = dyn_cast_or_null<InputSectionBase>(d->section)) {
@@ -1273,7 +1274,9 @@ template <class ELFT> void Writer<ELFT>::sortSections() {
     auto mid = std::stable_partition(
         ctx.script->sectionCommands.begin(), ctx.script->sectionCommands.end(),
         [](SectionCommand *cmd) { return isa<OutputDesc>(cmd); });
-    std::stable_sort(ctx.script->sectionCommands.begin(), mid, compareSections);
+    std::stable_sort(
+        ctx.script->sectionCommands.begin(), mid,
+        [&ctx = ctx](auto *l, auto *r) { return compareSections(ctx, l, r); });
   }
 
   // Process INSERT commands and update output section attributes. From this
@@ -1336,7 +1339,9 @@ template <class ELFT> void Writer<ELFT>::sortOrphanSections() {
   });
 
   // Sort the orphan sections.
-  std::stable_sort(nonScriptI, e, compareSections);
+  std::stable_sort(nonScriptI, e, [&ctx = ctx](auto *l, auto *r) {
+    return compareSections(ctx, l, r);
+  });
 
   // As a horrible special case, skip the first . assignment if it is before any
   // section. We do this because it is common to set a load address by starting
@@ -1770,9 +1775,9 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
       // define _TLS_MODULE_BASE_ relative to the first TLS section.
       Symbol *s = ctx.symtab->find("_TLS_MODULE_BASE_");
       if (s && s->isUndefined()) {
-        s->resolve(Defined{ctx.internalFile, StringRef(), STB_GLOBAL,
-                           STV_HIDDEN, STT_TLS, /*value=*/0, 0,
-                           /*section=*/nullptr});
+        s->resolve(ctx, Defined{ctx.internalFile, StringRef(), STB_GLOBAL,
+                                STV_HIDDEN, STT_TLS, /*value=*/0, 0,
+                                /*section=*/nullptr});
         ctx.sym.tlsModuleBase = cast<Defined>(s);
       }
     }
@@ -1847,7 +1852,8 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
             diagnose("undefined reference: " + toString(*sym) +
                      "\n>>> referenced by " + toString(file) +
                      " (disallowed by --no-allow-shlib-undefined)");
-          } else if (sym->isDefined() && sym->computeBinding() == STB_LOCAL) {
+          } else if (sym->isDefined() &&
+                     sym->computeBinding(ctx) == STB_LOCAL) {
             diagnose("non-exported symbol '" + toString(*sym) + "' in '" +
                      toString(sym->file) + "' is referenced by DSO '" +
                      toString(file) + "'");
@@ -1865,15 +1871,15 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
       if (!sym->isUsedInRegularObj || !includeInSymtab(ctx, *sym))
         continue;
       if (!ctx.arg.relocatable)
-        sym->binding = sym->computeBinding();
+        sym->binding = sym->computeBinding(ctx);
       if (ctx.in.symTab)
         ctx.in.symTab->addSymbol(sym);
 
-      if (sym->includeInDynsym()) {
+      if (sym->includeInDynsym(ctx)) {
         ctx.partitions[sym->partition - 1].dynSymTab->addSymbol(sym);
         if (auto *file = dyn_cast_or_null<SharedFile>(sym->file))
           if (file->isNeeded && !sym->isUndefined())
-            addVerneed(sym);
+            addVerneed(ctx, *sym);
       }
     }
 
@@ -2730,7 +2736,7 @@ static uint16_t getELFType(Ctx &ctx) {
 }
 
 template <class ELFT> void Writer<ELFT>::writeHeader() {
-  writeEhdr<ELFT>(ctx.bufferStart, *ctx.mainPart);
+  writeEhdr<ELFT>(ctx, ctx.bufferStart, *ctx.mainPart);
   writePhdrs<ELFT>(ctx.bufferStart + sizeof(Elf_Ehdr), *ctx.mainPart);
 
   auto *eHdr = reinterpret_cast<Elf_Ehdr *>(ctx.bufferStart);
diff --git a/lld/test/ELF/hexagon-shared.s b/lld/test/ELF/hexagon-shared.s
index 747822039e839af89ad5f29abfcf2cb3d6efe189..01f7286584705686610ab997de0a9e38e3d77dd2 100644
--- a/lld/test/ELF/hexagon-shared.s
+++ b/lld/test/ELF/hexagon-shared.s
@@ -42,6 +42,13 @@ r0 = add(r1,##bar@GOT)
 { r0 = add(r0,##bar@GOT)
   memw(r0) = r2 }
 
+# R_HEX_GOT_16_X, pred add
+if (p0) r0 = add(r0,##bar@GOT)
+if (!p0) r0 = add(r0,##bar@GOT)
+{ p0 = cmp.gtu(r0, r1)
+  if (p0.new) r0 = add(r0,##bar@GOT) }
+{ p0 = cmp.gtu(r0, r1)
+  if (!p0.new) r0 = add(r0,##bar@GOT) }
 
 # foo is local so no plt will be generated
 foo:
@@ -78,12 +85,16 @@ pvar:
 # PLT-NEXT: r28 = memw(r14+#0) }
 # PLT-NEXT: jumpr r28 }
 
-# TEXT:  8c 00 01 00 0001008c
-# TEXT: { 	call 0x102d0 }
-# TEXT: if (p0) jump:nt 0x102d0
-# TEXT: r0 = #0 ; jump 0x102d0
+# TEXT:  bc 00 01 00 000100bc
+# TEXT: { 	call 0x10300 }
+# TEXT: if (p0) jump:nt 0x10300
+# TEXT: r0 = #0 ; jump 0x10300
 # TEXT: r0 = add(r1,##-65548)
 # TEXT: r0 = add(r0,##-65548); memw(r0+#0) = r2 }
+# TEXT: if (p0) r0 = add(r0,##-65548)
+# TEXT: if (!p0) r0 = add(r0,##-65548)
+# TEXT: if (p0.new) r0 = add(r0,##-65548)
+# TEXT: if (!p0.new) r0 = add(r0,##-65548)
 
 # GOT: .got:
 # GOT:  00 00 00 00 00000000 <unknown>
diff --git a/lld/test/ELF/linkerscript/provide-defined.s b/lld/test/ELF/linkerscript/provide-defined.s
new file mode 100644
index 0000000000000000000000000000000000000000..1d44bef3d4068ddcb68d6ab4324f97e7a53bede1
--- /dev/null
+++ b/lld/test/ELF/linkerscript/provide-defined.s
@@ -0,0 +1,36 @@
+# REQUIRES: x86
+## Test the GC behavior when the PROVIDE symbol is defined by a relocatable file.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 b.s -o b.o
+# RUN: ld.lld -T a.t --gc-sections a.o b.o -o a
+# RUN: llvm-readelf -s a | FileCheck %s
+
+# CHECK:     1: {{.*}}               0 NOTYPE  GLOBAL DEFAULT     1 _start
+# CHECK-NEXT:2: {{.*}}               0 NOTYPE  GLOBAL DEFAULT     2 f3
+# CHECK-NOT: {{.}}
+
+#--- a.s
+.global _start, f1, f2, f3, bar
+_start:
+  call f3
+
+.section .text.f1,"ax"; f1:
+.section .text.f2,"ax"; f2: # referenced by another relocatable file
+.section .text.f3,"ax"; f3: # live
+.section .text.bar,"ax"; bar:
+
+.comm comm,4,4
+
+#--- b.s
+  call f2
+
+#--- a.t
+SECTIONS {
+  . = . + SIZEOF_HEADERS;
+  PROVIDE(f1 = bar+1);
+  PROVIDE(f2 = bar+2);
+  PROVIDE(f3 = bar+3);
+  PROVIDE(f4 = comm+4);
+}
diff --git a/lld/test/wasm/custom-section-align.s b/lld/test/wasm/custom-section-align.s
new file mode 100644
index 0000000000000000000000000000000000000000..0e46177f4cdb7983f3b4ab2896f51d6437b49d64
--- /dev/null
+++ b/lld/test/wasm/custom-section-align.s
@@ -0,0 +1,31 @@
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.o %s
+# RUN: wasm-ld --no-entry %t.o -o %t.wasm
+# RUN: obj2yaml %t.wasm | FileCheck %s
+
+# Check that "__llvm_covfun" custom section is aligned to 8 bytes.
+
+        .section        .custom_section.__llvm_covfun,"GR",@,__covrec_A
+        .int32  1
+        .int8   2
+# pad   .int8   0
+#       .int8   0
+#       .int8   0
+
+        .section        .custom_section.__llvm_covfun,"GR",@,__covrec_B
+        .int32  3
+
+# CHECK:      - Type:            CUSTOM
+# CHECK-NEXT:   Name:            __llvm_covfun
+# CHECK-NEXT:   Payload:         '010000000200000003000000'
+
+# Check that regular custom sections are not aligned.
+        .section        .custom_section.foo,"GR",@,foo_A
+        .int32  1
+        .int8   2
+
+        .section        .custom_section.foo,"GR",@,foo_B
+        .int32  3
+
+# CHECK:      - Type:            CUSTOM
+# CHECK-NEXT:   Name:            foo
+# CHECK-NEXT:   Payload:         '010000000203000000'
diff --git a/lld/wasm/CMakeLists.txt b/lld/wasm/CMakeLists.txt
index 6033bfbf9a50a3010b534182c39ad6349faf8cbf..ba89a6fee3504ed4ad769cd42158ba70208aaa36 100644
--- a/lld/wasm/CMakeLists.txt
+++ b/lld/wasm/CMakeLists.txt
@@ -28,6 +28,7 @@ add_lld_library(lldWasm
   Object
   Option
   Passes
+  ProfileData
   Support
   TargetParser
 
diff --git a/lld/wasm/InputChunks.h b/lld/wasm/InputChunks.h
index 14eb008c212fb57e19eb7a2d2df25ebea1328b75..d6769bcf5c82325001d6f866bb28776d41f2b22d 100644
--- a/lld/wasm/InputChunks.h
+++ b/lld/wasm/InputChunks.h
@@ -177,8 +177,9 @@ public:
     inputSectionOffset = seg.SectionOffset;
   }
 
-  MergeInputChunk(const WasmSection &s, ObjFile *f)
-      : InputChunk(f, Merge, s.Name, 0, llvm::wasm::WASM_SEG_FLAG_STRINGS) {
+  MergeInputChunk(const WasmSection &s, ObjFile *f, uint32_t alignment)
+      : InputChunk(f, Merge, s.Name, alignment,
+                   llvm::wasm::WASM_SEG_FLAG_STRINGS) {
     assert(s.Type == llvm::wasm::WASM_SEC_CUSTOM);
     comdat = s.Comdat;
     rawData = s.Content;
@@ -234,6 +235,7 @@ public:
 
   void addMergeChunk(MergeInputChunk *ms) {
     comdat = ms->getComdat();
+    alignment = std::max(alignment, ms->alignment);
     ms->parent = this;
     chunks.push_back(ms);
   }
@@ -337,8 +339,8 @@ public:
 // Represents a single Wasm Section within an input file.
 class InputSection : public InputChunk {
 public:
-  InputSection(const WasmSection &s, ObjFile *f)
-      : InputChunk(f, InputChunk::Section, s.Name),
+  InputSection(const WasmSection &s, ObjFile *f, uint32_t alignment)
+      : InputChunk(f, InputChunk::Section, s.Name, alignment),
         tombstoneValue(getTombstoneForSection(s.Name)), section(s) {
     assert(section.Type == llvm::wasm::WASM_SEC_CUSTOM);
     comdat = section.Comdat;
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index de8e707ab2b497d44cdad068e3be335780e4ab03..420865e2aea8e38cb189dc561a14df96f32ad83d 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -18,6 +18,7 @@
 #include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/Wasm.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/raw_ostream.h"
@@ -451,6 +452,18 @@ void SharedFile::parse() {
   }
 }
 
+// Returns the alignment for a custom section. This is used to concatenate
+// custom sections with the same name into a single custom section.
+static uint32_t getCustomSectionAlignment(const WasmSection &sec) {
+  // TODO: Add a section attribute for alignment in the linking spec.
+  if (sec.Name == getInstrProfSectionName(IPSK_covfun, Triple::Wasm) ||
+      sec.Name == getInstrProfSectionName(IPSK_covmap, Triple::Wasm)) {
+    // llvm-cov assumes that coverage metadata sections are 8-byte aligned.
+    return 8;
+  }
+  return 1;
+}
+
 WasmFileBase::WasmFileBase(Kind k, MemoryBufferRef m) : InputFile(k, m) {
   // Parse a memory buffer as a wasm file.
   LLVM_DEBUG(dbgs() << "Reading object: " << toString(this) << "\n");
@@ -520,10 +533,11 @@ void ObjFile::parse(bool ignoreComdats) {
       dataSection = &section;
     } else if (section.Type == WASM_SEC_CUSTOM) {
       InputChunk *customSec;
+      uint32_t alignment = getCustomSectionAlignment(section);
       if (shouldMerge(section))
-        customSec = make<MergeInputChunk>(section, this);
+        customSec = make<MergeInputChunk>(section, this, alignment);
       else
-        customSec = make<InputSection>(section, this);
+        customSec = make<InputSection>(section, this, alignment);
       customSec->discarded = isExcludedByComdat(customSec);
       customSections.emplace_back(customSec);
       customSections.back()->setRelocations(section.Relocations);
diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp
index b0b2446cb56bfc70d187ee3c0d4dd40b8cd9df0d..e4f75829ec4c3e1a4db51e970d34a3b0996083fb 100644
--- a/lld/wasm/OutputSections.cpp
+++ b/lld/wasm/OutputSections.cpp
@@ -244,6 +244,7 @@ void CustomSection::finalizeContents() {
 
   for (InputChunk *section : inputSections) {
     assert(!section->discarded);
+    payloadSize = alignTo(payloadSize, section->alignment);
     section->outSecOff = payloadSize;
     payloadSize += section->getSize();
   }
diff --git a/lldb/CodeOwners.rst b/lldb/Maintainers.rst
similarity index 91%
rename from lldb/CodeOwners.rst
rename to lldb/Maintainers.rst
index 3c10c2a28da9e74d221fb46f2c719882c115e3a9..e83591d5f162b4cf99a9f9f3bb9fab11172744da 100644
--- a/lldb/CodeOwners.rst
+++ b/lldb/Maintainers.rst
@@ -1,27 +1,31 @@
 ================
-LLDB Code Owners
+LLDB Maintainers
 ================
 
-This file is a list of the `code owners <https://llvm.org/docs/DeveloperPolicy.html#code-owners>`_ for LLDB.
+This file is a list of the `maintainers <https://llvm.org/docs/DeveloperPolicy.html#maintainers>`_ for LLDB.
 
 .. contents::
    :depth: 2
    :local:
 
-Current Code Owners
+Current Maintainers
 ===================
-The following people are the active code owners for the project. Please reach
+The following people are the active maintainers for the project. Please reach
 out to them for code reviews, questions about their area of expertise, or other
 assistance.
 
-All parts of LLDB not covered by someone else
-----------------------------------------------
+Lead Maintainer
+---------------
+
+Responsible for project as a whole, and for any areas not covered by a specific
+maintainer.
+
 | Jonas Devlieghere
 | jonas\@devlieghere.com (email), jdevlieghere (GitHub), jdevlieghere (Discourse), jdevlieghere (Discord)
 
 Components
 ----------
-These code owners are responsible for particular high-level components within
+These maintainers are responsible for particular high-level components within
 LLDB.
 
 ABI
@@ -242,10 +246,10 @@ lldb-dap
 | Walter Erquinigo
 | a20012251\@gmail.com (email), walter-erquinigo (GitHub), wallace (Discourse), werquinigo (Discord)
 
-Former Code Owners
+Former Maintainers
 ==================
-The following people have graciously spent time performing code ownership
-responsibilities but are no longer active in that role. Thank you for all your
+The following people have graciously spent time performing maintainership
+duties but are no longer active in that role. Thank you for all your
 help with the success of the project!
 
 | Kamil Rytarowski (kamil\@netbsd.org)
diff --git a/lldb/include/lldb/API/SBCommandReturnObject.h b/lldb/include/lldb/API/SBCommandReturnObject.h
index f96384a4710b16c0f14cd2078ad6d2300e7b9ba0..e8e20a3f3016b82721a2eab314e529b6aeaff225 100644
--- a/lldb/include/lldb/API/SBCommandReturnObject.h
+++ b/lldb/include/lldb/API/SBCommandReturnObject.h
@@ -45,6 +45,7 @@ public:
   const char *GetOutput();
 
   const char *GetError();
+  SBStructuredData GetErrorData();
 
 #ifndef SWIG
   LLDB_DEPRECATED_FIXME("Use PutOutput(SBFile) or PutOutput(FileSP)",
diff --git a/lldb/include/lldb/API/SBStructuredData.h b/lldb/include/lldb/API/SBStructuredData.h
index fc6e1ec95c7b86e9c49342ee85e76b2332b18a4c..ccdd12cab94b2fb590d78e1a2d486c9114655b67 100644
--- a/lldb/include/lldb/API/SBStructuredData.h
+++ b/lldb/include/lldb/API/SBStructuredData.h
@@ -9,6 +9,7 @@
 #ifndef LLDB_API_SBSTRUCTUREDDATA_H
 #define LLDB_API_SBSTRUCTUREDDATA_H
 
+#include "lldb/API/SBCommandReturnObject.h"
 #include "lldb/API/SBDefines.h"
 #include "lldb/API/SBModule.h"
 #include "lldb/API/SBScriptObject.h"
@@ -110,6 +111,7 @@ public:
 
 protected:
   friend class SBAttachInfo;
+  friend class SBCommandReturnObject;
   friend class SBLaunchInfo;
   friend class SBDebugger;
   friend class SBTarget;
diff --git a/lldb/include/lldb/Breakpoint/StopPointSiteList.h b/lldb/include/lldb/Breakpoint/StopPointSiteList.h
index 9ff151fd01b69ac4581952ff1bbcd2b4b3419434..b929c37a12f0922d7828d36fc7d3fc3057ce8f72 100644
--- a/lldb/include/lldb/Breakpoint/StopPointSiteList.h
+++ b/lldb/include/lldb/Breakpoint/StopPointSiteList.h
@@ -38,16 +38,10 @@ public:
   typename StopPointSite::SiteID Add(const StopPointSiteSP &site_sp) {
     lldb::addr_t site_load_addr = site_sp->GetLoadAddress();
     std::lock_guard<std::recursive_mutex> guard(m_mutex);
-    typename collection::iterator iter = m_site_list.find(site_load_addr);
-
     // Add site to the list.  However, if the element already exists in
     // the list, then we don't add it, and return InvalidSiteID.
-    if (iter == m_site_list.end()) {
-      m_site_list[site_load_addr] = site_sp;
-      return site_sp->GetID();
-    } else {
-      return UINT32_MAX;
-    }
+    bool inserted = m_site_list.try_emplace(site_load_addr, site_sp).second;
+    return inserted ? site_sp->GetID() : UINT32_MAX;
   }
 
   /// Standard Dump routine, doesn't do anything at present.
diff --git a/lldb/include/lldb/Expression/DiagnosticManager.h b/lldb/include/lldb/Expression/DiagnosticManager.h
index b9a6421577781e228c0b27b9116fa7317216fc42..cc802b6f48334a74cc15833e8fb98f47ad69012f 100644
--- a/lldb/include/lldb/Expression/DiagnosticManager.h
+++ b/lldb/include/lldb/Expression/DiagnosticManager.h
@@ -12,6 +12,7 @@
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-types.h"
 
+#include "lldb/Utility/DiagnosticsRendering.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Status.h"
 
@@ -23,49 +24,22 @@
 
 namespace lldb_private {
 
-/// A compiler-independent representation of a Diagnostic. Expression
-/// evaluation failures often have more than one diagnostic that a UI
-/// layer might want to render differently, for example to colorize
-/// it.
-///
-/// Running example:
-///   (lldb) expr 1+foo
-///   error: <user expression 0>:1:3: use of undeclared identifier 'foo'
-///   1+foo
-///     ^
-struct DiagnosticDetail {
-  struct SourceLocation {
-    FileSpec file;
-    unsigned line = 0;
-    uint16_t column = 0;
-    uint16_t length = 0;
-    bool hidden = false;
-    bool in_user_input = false;
-  };
-  /// Contains {{}, 1, 3, 3, true} in the example above.
-  std::optional<SourceLocation> source_location;
-  /// Contains eSeverityError in the example above.
-  lldb::Severity severity = lldb::eSeverityInfo;
-  /// Contains "use of undeclared identifier 'x'" in the example above.
-  std::string message;
-  /// Contains the fully rendered error message.
-  std::string rendered;
-};
-
 /// An llvm::Error used to communicate diagnostics in Status. Multiple
 /// diagnostics may be chained in an llvm::ErrorList.
 class ExpressionError
-    : public llvm::ErrorInfo<ExpressionError, ExpressionErrorBase> {
+    : public llvm::ErrorInfo<ExpressionError, DiagnosticError> {
   std::string m_message;
   std::vector<DiagnosticDetail> m_details;
 
 public:
   static char ID;
-  using llvm::ErrorInfo<ExpressionError, ExpressionErrorBase>::ErrorInfo;
+  using llvm::ErrorInfo<ExpressionError, DiagnosticError>::ErrorInfo;
   ExpressionError(lldb::ExpressionResults result, std::string msg,
                   std::vector<DiagnosticDetail> details = {});
   std::string message() const override;
-  llvm::ArrayRef<DiagnosticDetail> GetDetails() const { return m_details; }
+  llvm::ArrayRef<DiagnosticDetail> GetDetails() const override {
+    return m_details;
+  }
   std::error_code convertToErrorCode() const override;
   void log(llvm::raw_ostream &OS) const override;
   std::unique_ptr<CloneableError> Clone() const override;
diff --git a/lldb/include/lldb/Interpreter/CommandReturnObject.h b/lldb/include/lldb/Interpreter/CommandReturnObject.h
index 8f6c9f123b76905907cd2c18f2dcf84e0fa4c203..a491a6c1535b11f3752f446e21f3fb0393385c97 100644
--- a/lldb/include/lldb/Interpreter/CommandReturnObject.h
+++ b/lldb/include/lldb/Interpreter/CommandReturnObject.h
@@ -10,8 +10,10 @@
 #define LLDB_INTERPRETER_COMMANDRETURNOBJECT_H
 
 #include "lldb/Host/StreamFile.h"
+#include "lldb/Utility/DiagnosticsRendering.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StreamTee.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/lldb-private.h"
 
 #include "llvm/ADT/StringRef.h"
@@ -29,19 +31,23 @@ public:
 
   ~CommandReturnObject() = default;
 
-  llvm::StringRef GetOutputData() {
+  /// Format any inline diagnostics with an indentation of \c indent.
+  std::string GetInlineDiagnosticString(unsigned indent);
+
+  llvm::StringRef GetOutputString() {
     lldb::StreamSP stream_sp(m_out_stream.GetStreamAtIndex(eStreamStringIndex));
     if (stream_sp)
       return std::static_pointer_cast<StreamString>(stream_sp)->GetString();
     return llvm::StringRef();
   }
 
-  llvm::StringRef GetErrorData() {
-    lldb::StreamSP stream_sp(m_err_stream.GetStreamAtIndex(eStreamStringIndex));
-    if (stream_sp)
-      return std::static_pointer_cast<StreamString>(stream_sp)->GetString();
-    return llvm::StringRef();
-  }
+  /// Return the errors as a string.
+  ///
+  /// If \c with_diagnostics is true, all diagnostics are also
+  /// rendered into the string. Otherwise the expectation is that they
+  /// are fetched with \ref GetInlineDiagnosticString().
+  std::string GetErrorString(bool with_diagnostics = true);
+  StructuredData::ObjectSP GetErrorData();
 
   Stream &GetOutputStream() {
     // Make sure we at least have our normal string stream output stream
@@ -135,6 +141,14 @@ public:
 
   void SetError(llvm::Error error);
 
+  void SetDiagnosticIndent(std::optional<uint16_t> indent) {
+    m_diagnostic_indent = indent;
+  }
+
+  std::optional<uint16_t> GetDiagnosticIndent() const {
+    return m_diagnostic_indent;
+  }
+
   lldb::ReturnStatus GetStatus() const;
 
   void SetStatus(lldb::ReturnStatus status);
@@ -160,6 +174,8 @@ private:
 
   StreamTee m_out_stream;
   StreamTee m_err_stream;
+  std::vector<DiagnosticDetail> m_diagnostics;
+  std::optional<uint16_t> m_diagnostic_indent;
 
   lldb::ReturnStatus m_status = lldb::eReturnStatusStarted;
 
@@ -168,6 +184,7 @@ private:
 
   /// If true, then the input handle from the debugger will be hooked up.
   bool m_interactive = true;
+  bool m_colors;
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Utility/Args.h b/lldb/include/lldb/Utility/Args.h
index 757f7e2ba5ec6787135732187e770153137c54ba..094bb290691198a346e9b34ad9de70f43d644aa9 100644
--- a/lldb/include/lldb/Utility/Args.h
+++ b/lldb/include/lldb/Utility/Args.h
@@ -38,12 +38,14 @@ public:
 
     std::unique_ptr<char[]> ptr;
     char quote = '\0';
+    /// The position of the argument in the original argument string.
+    std::optional<uint16_t> column;
 
     char *data() { return ptr.get(); }
 
   public:
     ArgEntry() = default;
-    ArgEntry(llvm::StringRef str, char quote);
+    ArgEntry(llvm::StringRef str, char quote, std::optional<uint16_t> column);
 
     llvm::StringRef ref() const { return c_str(); }
     const char *c_str() const { return ptr.get(); }
@@ -51,6 +53,8 @@ public:
     /// Returns true if this argument was quoted in any way.
     bool IsQuoted() const { return quote != '\0'; }
     char GetQuoteChar() const { return quote; }
+    std::optional<uint16_t> GetPos() const { return column; }
+    size_t GetLength() const { return ref().size(); }
   };
 
   /// Construct with an option command string.
diff --git a/lldb/include/lldb/Utility/DiagnosticsRendering.h b/lldb/include/lldb/Utility/DiagnosticsRendering.h
new file mode 100644
index 0000000000000000000000000000000000000000..33aded602e3a77ef2b460fd545f0cab4d264c7da
--- /dev/null
+++ b/lldb/include/lldb/Utility/DiagnosticsRendering.h
@@ -0,0 +1,75 @@
+//===-- DiagnosticsRendering.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UTILITY_DIAGNOSTICSRENDERING_H
+#define LLDB_UTILITY_DIAGNOSTICSRENDERING_H
+
+#include "lldb/Utility/Status.h"
+#include "lldb/Utility/Stream.h"
+#include "llvm/Support/WithColor.h"
+
+namespace lldb_private {
+
+/// A compiler-independent representation of an \c
+/// lldb_private::Diagnostic. Expression evaluation failures often
+/// have more than one diagnostic that a UI layer might want to render
+/// differently, for example to colorize it.
+///
+/// Running example:
+///   (lldb) expr 1 + foo
+///   error: <user expression 0>:1:3: use of undeclared identifier 'foo'
+///   1 + foo
+///       ^~~
+struct DiagnosticDetail {
+  /// A source location consisting of a file name and position.
+  struct SourceLocation {
+    /// \c "<user expression 0>" in the example above.
+    FileSpec file;
+    /// \c 1 in the example above.
+    unsigned line = 0;
+    /// \c 5 in the example above.
+    uint16_t column = 0;
+    /// \c 3 in the example above.
+    uint16_t length = 0;
+    /// Whether this source location should be surfaced to the
+    /// user. For example, syntax errors diagnosed in LLDB's
+    /// expression wrapper code have this set to true.
+    bool hidden = false;
+    /// Whether this source location refers to something the user
+    /// typed as part of the command, i.e., if this qualifies for
+    /// inline display, or if the source line would need to be echoed
+    /// again for the message to make sense.
+    bool in_user_input = false;
+  };
+  /// Contains this diagnostic's source location, if applicable.
+  std::optional<SourceLocation> source_location;
+  /// Contains \c eSeverityError in the example above.
+  lldb::Severity severity = lldb::eSeverityInfo;
+  /// Contains "use of undeclared identifier 'foo'" in the example above.
+  std::string message;
+  /// Contains the fully rendered error message, without "error: ",
+  /// but including the source context.
+  std::string rendered;
+};
+
+class DiagnosticError
+    : public llvm::ErrorInfo<DiagnosticError, CloneableECError> {
+public:
+  using llvm::ErrorInfo<DiagnosticError, CloneableECError>::ErrorInfo;
+  DiagnosticError(std::error_code ec) : ErrorInfo(ec) {}
+  lldb::ErrorType GetErrorType() const override;
+  virtual llvm::ArrayRef<DiagnosticDetail> GetDetails() const = 0;
+  static char ID;
+};
+
+void RenderDiagnosticDetails(Stream &stream,
+                             std::optional<uint16_t> offset_in_command,
+                             bool show_inline,
+                             llvm::ArrayRef<DiagnosticDetail> details);
+} // namespace lldb_private
+#endif
diff --git a/lldb/include/lldb/Utility/Status.h b/lldb/include/lldb/Utility/Status.h
index 2911ffb804c6fb1af7d7bd2ca75506d23ba4083f..b8993dff3bb18a44a6afa3426d1ec56c039381fb 100644
--- a/lldb/include/lldb/Utility/Status.h
+++ b/lldb/include/lldb/Utility/Status.h
@@ -9,6 +9,7 @@
 #ifndef LLDB_UTILITY_STATUS_H
 #define LLDB_UTILITY_STATUS_H
 
+#include "lldb/Utility/FileSpec.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "llvm/ADT/StringRef.h"
@@ -26,8 +27,6 @@ class raw_ostream;
 
 namespace lldb_private {
 
-const char *ExpressionResultAsCString(lldb::ExpressionResults result);
-
 /// Going a bit against the spirit of llvm::Error,
 /// lldb_private::Status need to store errors long-term and sometimes
 /// copy them. This base class defines an interface for this
@@ -79,15 +78,6 @@ public:
   static char ID;
 };
 
-class ExpressionErrorBase
-    : public llvm::ErrorInfo<ExpressionErrorBase, CloneableECError> {
-public:
-  using llvm::ErrorInfo<ExpressionErrorBase, CloneableECError>::ErrorInfo;
-  ExpressionErrorBase(std::error_code ec) : ErrorInfo(ec) {}
-  lldb::ErrorType GetErrorType() const override;
-  static char ID;
-};
-
 /// \class Status Status.h "lldb/Utility/Status.h" An error handling class.
 ///
 /// This class is designed to be able to hold any error code that can be
diff --git a/lldb/source/API/SBCommandReturnObject.cpp b/lldb/source/API/SBCommandReturnObject.cpp
index d0cdebe8c64911db2214e5cbedcb68c4ce9a1cd5..9df8aa48b99366414346468b24c0483bc653f54e 100644
--- a/lldb/source/API/SBCommandReturnObject.cpp
+++ b/lldb/source/API/SBCommandReturnObject.cpp
@@ -11,6 +11,8 @@
 #include "lldb/API/SBError.h"
 #include "lldb/API/SBFile.h"
 #include "lldb/API/SBStream.h"
+#include "lldb/API/SBStructuredData.h"
+#include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Instrumentation.h"
@@ -85,27 +87,36 @@ SBCommandReturnObject::operator bool() const {
 const char *SBCommandReturnObject::GetOutput() {
   LLDB_INSTRUMENT_VA(this);
 
-  ConstString output(ref().GetOutputData());
+  ConstString output(ref().GetOutputString());
   return output.AsCString(/*value_if_empty*/ "");
 }
 
 const char *SBCommandReturnObject::GetError() {
   LLDB_INSTRUMENT_VA(this);
 
-  ConstString output(ref().GetErrorData());
+  ConstString output(ref().GetErrorString());
   return output.AsCString(/*value_if_empty*/ "");
 }
 
+SBStructuredData SBCommandReturnObject::GetErrorData() {
+  LLDB_INSTRUMENT_VA(this);
+
+  StructuredData::ObjectSP data(ref().GetErrorData());
+  SBStructuredData sb_data;
+  sb_data.m_impl_up->SetObjectSP(data);
+  return sb_data;
+}
+
 size_t SBCommandReturnObject::GetOutputSize() {
   LLDB_INSTRUMENT_VA(this);
 
-  return ref().GetOutputData().size();
+  return ref().GetOutputString().size();
 }
 
 size_t SBCommandReturnObject::GetErrorSize() {
   LLDB_INSTRUMENT_VA(this);
 
-  return ref().GetErrorData().size();
+  return ref().GetErrorString().size();
 }
 
 size_t SBCommandReturnObject::PutOutput(FILE *fh) {
diff --git a/lldb/source/Commands/CommandObjectCommands.cpp b/lldb/source/Commands/CommandObjectCommands.cpp
index 845b89a75b7b3923329775c6876fe9ef9a86ce59..f069b2feb5cb7bd33e88cceb508b6fe2323eae85 100644
--- a/lldb/source/Commands/CommandObjectCommands.cpp
+++ b/lldb/source/Commands/CommandObjectCommands.cpp
@@ -1099,7 +1099,7 @@ protected:
     } else {
       // Don't change the status if the command already set it...
       if (result.GetStatus() == eReturnStatusInvalid) {
-        if (result.GetOutputData().empty())
+        if (result.GetOutputString().empty())
           result.SetStatus(eReturnStatusSuccessFinishNoResult);
         else
           result.SetStatus(eReturnStatusSuccessFinishResult);
@@ -1205,7 +1205,7 @@ protected:
     } else {
       // Don't change the status if the command already set it...
       if (result.GetStatus() == eReturnStatusInvalid) {
-        if (result.GetOutputData().empty())
+        if (result.GetOutputString().empty())
           result.SetStatus(eReturnStatusSuccessFinishNoResult);
         else
           result.SetStatus(eReturnStatusSuccessFinishResult);
@@ -2137,7 +2137,7 @@ protected:
     } else {
       // Don't change the status if the command already set it...
       if (result.GetStatus() == eReturnStatusInvalid) {
-        if (result.GetOutputData().empty())
+        if (result.GetOutputString().empty())
           result.SetStatus(eReturnStatusSuccessFinishNoResult);
         else
           result.SetStatus(eReturnStatusSuccessFinishResult);
diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
index 2e1d6e6e5af99620b0156e99510f88c75cd4a500..ddf0aca25fb370b46dd5cfa7a0f7dfbcbb94d3ce 100644
--- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp
+++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
@@ -191,6 +191,17 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
     ExpressionResults expr_result = target.EvaluateExpression(
         expr, exe_scope, valobj_sp, eval_options, &fixed_expression);
 
+    // Record the position of the expression in the command.
+    std::optional<uint16_t> indent;
+    if (fixed_expression.empty()) {
+      size_t pos = m_original_command.rfind(expr);
+      if (pos != llvm::StringRef::npos)
+        indent = pos;
+    }
+    // Previously the indent was set up for diagnosing command line
+    // parsing errors. Now point it to the expression.
+    result.SetDiagnosticIndent(indent);
+
     // Only mention Fix-Its if the expression evaluator applied them.
     // Compiler errors refer to the final expression after applying Fix-It(s).
     if (!fixed_expression.empty() && target.GetEnableNotifyAboutFixIts()) {
diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp
index a72b409d21ed8301451c60184b3fb135cae91fad..13491b5c794425d54bed93b12bb53ec102b6f99b 100644
--- a/lldb/source/Commands/CommandObjectExpression.cpp
+++ b/lldb/source/Commands/CommandObjectExpression.cpp
@@ -7,9 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CommandObjectExpression.h"
-#include "DiagnosticRendering.h"
 #include "lldb/Core/Debugger.h"
-#include "lldb/Expression/DiagnosticManager.h"
 #include "lldb/Expression/ExpressionVariable.h"
 #include "lldb/Expression/REPL.h"
 #include "lldb/Expression/UserExpression.h"
@@ -22,6 +20,7 @@
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
+#include "lldb/Utility/DiagnosticsRendering.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-private-enumerations.h"
 
@@ -486,35 +485,8 @@ bool CommandObjectExpression::EvaluateExpression(llvm::StringRef expr,
 
         result.SetStatus(eReturnStatusSuccessFinishResult);
       } else {
-        // Retrieve the diagnostics.
-        std::vector<DiagnosticDetail> details;
-        llvm::consumeError(llvm::handleErrors(
-            result_valobj_sp->GetError().ToError(),
-            [&](ExpressionError &error) { details = error.GetDetails(); }));
-        // Find the position of the expression in the command.
-        std::optional<uint16_t> expr_pos;
-        size_t nchar = m_original_command.find(expr);
-        if (nchar != std::string::npos)
-          expr_pos = nchar + GetDebugger().GetPrompt().size();
-
-        if (!details.empty()) {
-          bool show_inline =
-              GetDebugger().GetShowInlineDiagnostics() && !expr.contains('\n');
-          RenderDiagnosticDetails(error_stream, expr_pos, show_inline, details);
-        } else {
-          const char *error_cstr = result_valobj_sp->GetError().AsCString();
-          llvm::StringRef error(error_cstr);
-          if (!error.empty()) {
-            if (!error.starts_with("error:"))
-              error_stream << "error: ";
-            error_stream << error;
-            if (!error.ends_with('\n'))
-              error_stream.EOL();
-          } else {
-            error_stream << "error: unknown error\n";
-          }
-        }
         result.SetStatus(eReturnStatusFailed);
+        result.SetError(result_valobj_sp->GetError().ToError());
       }
     }
   } else {
@@ -534,10 +506,13 @@ void CommandObjectExpression::IOHandlerInputComplete(IOHandler &io_handler,
   CommandReturnObject return_obj(
       GetCommandInterpreter().GetDebugger().GetUseColor());
   EvaluateExpression(line.c_str(), *output_sp, *error_sp, return_obj);
+
   if (output_sp)
     output_sp->Flush();
-  if (error_sp)
+  if (error_sp) {
+    *error_sp << return_obj.GetErrorString();
     error_sp->Flush();
+  }
 }
 
 bool CommandObjectExpression::IOHandlerIsInputComplete(IOHandler &io_handler,
@@ -680,6 +655,14 @@ void CommandObjectExpression::DoExecute(llvm::StringRef command,
     }
   }
 
+  // Previously the indent was set up for diagnosing command line
+  // parsing errors. Now point it to the expression.
+  std::optional<uint16_t> indent;
+  size_t pos = m_original_command.rfind(expr);
+  if (pos != llvm::StringRef::npos)
+    indent = pos;
+  result.SetDiagnosticIndent(indent);
+
   Target &target = GetTarget();
   if (EvaluateExpression(expr, result.GetOutputStream(),
                          result.GetErrorStream(), result)) {
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index e6b9eedd89b4e371dada93e20f6554084c2469b7..c666a753343c9d9c2bb827c69f507553c4bc5bf0 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -784,9 +784,10 @@ void Debugger::Destroy(DebuggerSP &debugger_sp) {
     CommandReturnObject result(debugger_sp->GetUseColor());
     cmd_interpreter.SaveTranscript(result);
     if (result.Succeeded())
-      (*debugger_sp->GetAsyncOutputStream()) << result.GetOutputData() << '\n';
+      (*debugger_sp->GetAsyncOutputStream())
+          << result.GetOutputString() << '\n';
     else
-      (*debugger_sp->GetAsyncErrorStream()) << result.GetErrorData() << '\n';
+      (*debugger_sp->GetAsyncErrorStream()) << result.GetErrorString() << '\n';
   }
 
   debugger_sp->Clear();
diff --git a/lldb/source/Expression/DiagnosticManager.cpp b/lldb/source/Expression/DiagnosticManager.cpp
index 7c67a0ce4aa026482ed1d6964975cf3a1c2de71b..b1fa14bccc17bd9452ebe58eace2b5507d21700f 100644
--- a/lldb/source/Expression/DiagnosticManager.cpp
+++ b/lldb/source/Expression/DiagnosticManager.cpp
@@ -10,6 +10,7 @@
 
 #include "llvm/Support/ErrorHandling.h"
 
+#include "lldb/Utility/ErrorMessages.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
 
@@ -22,8 +23,7 @@ class ExpressionCategory : public std::error_category {
     return "LLDBExpressionCategory";
   }
   std::string message(int __ev) const override {
-    return ExpressionResultAsCString(
-        static_cast<lldb::ExpressionResults>(__ev));
+    return toString(static_cast<lldb::ExpressionResults>(__ev));
   };
 };
 ExpressionCategory &expression_category() {
diff --git a/lldb/source/Expression/FunctionCaller.cpp b/lldb/source/Expression/FunctionCaller.cpp
index 5ce0175fedf45357461ada28c8a0688df049275c..d1dd350b09f25e8b0d2674eb7bb7e3f01d8592bd 100644
--- a/lldb/source/Expression/FunctionCaller.cpp
+++ b/lldb/source/Expression/FunctionCaller.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "lldb/Expression/FunctionCaller.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ValueObject.h"
@@ -24,6 +23,7 @@
 #include "lldb/Target/ThreadPlan.h"
 #include "lldb/Target/ThreadPlanCallFunction.h"
 #include "lldb/Utility/DataExtractor.h"
+#include "lldb/Utility/ErrorMessages.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/State.h"
@@ -390,7 +390,7 @@ lldb::ExpressionResults FunctionCaller::ExecuteFunction(
       LLDB_LOGF(log,
                 "== [FunctionCaller::ExecuteFunction] Execution of \"%s\" "
                 "completed abnormally: %s ==",
-                m_name.c_str(), ExpressionResultAsCString(return_value));
+                m_name.c_str(), toString(return_value).c_str());
     } else {
       LLDB_LOGF(log,
                 "== [FunctionCaller::ExecuteFunction] Execution of \"%s\" "
diff --git a/lldb/source/Expression/LLVMUserExpression.cpp b/lldb/source/Expression/LLVMUserExpression.cpp
index 9e07d4a7374a782c40c891c996e08d476d54f993..d76f04d9b86e2e13efd6720ac971ba886ce70a8f 100644
--- a/lldb/source/Expression/LLVMUserExpression.cpp
+++ b/lldb/source/Expression/LLVMUserExpression.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "lldb/Expression/LLVMUserExpression.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ValueObjectConstResult.h"
@@ -30,6 +29,7 @@
 #include "lldb/Target/ThreadPlan.h"
 #include "lldb/Target/ThreadPlanCallUserExpression.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/ErrorMessages.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
@@ -237,7 +237,7 @@ LLVMUserExpression::DoExecute(DiagnosticManager &diagnostic_manager,
     } else if (execution_result != lldb::eExpressionCompleted) {
       diagnostic_manager.Printf(lldb::eSeverityError,
                                 "Couldn't execute function; result was %s",
-                                ExpressionResultAsCString(execution_result));
+                                toString(execution_result).c_str());
       return execution_result;
     }
   }
diff --git a/lldb/source/Host/common/Host.cpp b/lldb/source/Host/common/Host.cpp
index abca6068d3604afb949fc3038f0cc1996400ac52..6857a29b0399028e82c2d0c9f630e7e8d7a3f21c 100644
--- a/lldb/source/Host/common/Host.cpp
+++ b/lldb/source/Host/common/Host.cpp
@@ -111,17 +111,7 @@ void Host::SystemLog(Severity severity, llvm::StringRef message) {
   syslog(level, "%s", message.data());
 }
 #else
-void Host::SystemLog(Severity severity, llvm::StringRef message) {
-  switch (severity) {
-  case lldb::eSeverityInfo:
-  case lldb::eSeverityWarning:
-    llvm::outs() << message;
-    break;
-  case lldb::eSeverityError:
-    llvm::errs() << message;
-    break;
-  }
-}
+void Host::SystemLog(Severity severity, llvm::StringRef message) {}
 #endif
 #endif
 
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index 8d3a82ef6c990ad0d91f1d2b00171edd9e4523ff..bfac3f4fea8d40dfcd99c51e50441cebd9174f52 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -2069,7 +2069,7 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
         remainder.c_str());
 
     // To test whether or not transcript should be saved, `transcript_item` is
-    // used instead of `GetSaveTrasncript()`. This is because the latter will
+    // used instead of `GetSaveTranscript()`. This is because the latter will
     // fail when the command is "settings set interpreter.save-transcript true".
     if (transcript_item) {
       transcript_item->AddStringItem("commandName", cmd_obj->GetCommandName());
@@ -2078,6 +2078,12 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
 
     ElapsedTime elapsed(execute_time);
     cmd_obj->SetOriginalCommandString(real_original_command_string);
+    // Set the indent to the position of the command in the command line.
+    pos = real_original_command_string.rfind(remainder);
+    std::optional<uint16_t> indent;
+    if (pos != std::string::npos)
+      indent = pos;
+    result.SetDiagnosticIndent(indent);
     cmd_obj->Execute(remainder.c_str(), result);
   }
 
@@ -2088,11 +2094,11 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   // used instead of `GetSaveTrasncript()`. This is because the latter will
   // fail when the command is "settings set interpreter.save-transcript true".
   if (transcript_item) {
-    m_transcript_stream << result.GetOutputData();
-    m_transcript_stream << result.GetErrorData();
+    m_transcript_stream << result.GetOutputString();
+    m_transcript_stream << result.GetErrorString();
 
-    transcript_item->AddStringItem("output", result.GetOutputData());
-    transcript_item->AddStringItem("error", result.GetErrorData());
+    transcript_item->AddStringItem("output", result.GetOutputString());
+    transcript_item->AddStringItem("error", result.GetErrorString());
     transcript_item->AddFloatItem("durationInSeconds",
                                   execute_time.get().count());
   }
@@ -2626,24 +2632,22 @@ void CommandInterpreter::HandleCommands(const StringList &commands,
 
     if (options.GetPrintResults()) {
       if (tmp_result.Succeeded())
-        result.AppendMessage(tmp_result.GetOutputData());
+        result.AppendMessage(tmp_result.GetOutputString());
     }
 
     if (!success || !tmp_result.Succeeded()) {
-      llvm::StringRef error_msg = tmp_result.GetErrorData();
+      std::string error_msg = tmp_result.GetErrorString();
       if (error_msg.empty())
         error_msg = "<unknown error>.\n";
       if (options.GetStopOnError()) {
-        result.AppendErrorWithFormat(
-            "Aborting reading of commands after command #%" PRIu64
-            ": '%s' failed with %s",
-            (uint64_t)idx, cmd, error_msg.str().c_str());
+        result.AppendErrorWithFormatv("Aborting reading of commands after "
+                                      "command #{0}: '{1}' failed with {2}",
+                                      (uint64_t)idx, cmd, error_msg);
         m_debugger.SetAsyncExecution(old_async_execution);
         return;
       } else if (options.GetPrintResults()) {
-        result.AppendMessageWithFormat(
-            "Command #%" PRIu64 " '%s' failed with %s", (uint64_t)idx + 1, cmd,
-            error_msg.str().c_str());
+        result.AppendMessageWithFormatv("Command #{0} '{1}' failed with {2}",
+                                        (uint64_t)idx + 1, cmd, error_msg);
       }
     }
 
@@ -3180,17 +3184,29 @@ void CommandInterpreter::IOHandlerInputComplete(IOHandler &io_handler,
   if ((result.Succeeded() &&
        io_handler.GetFlags().Test(eHandleCommandFlagPrintResult)) ||
       io_handler.GetFlags().Test(eHandleCommandFlagPrintErrors)) {
-    // Display any STDOUT/STDERR _prior_ to emitting the command result text
+    // Display any inline diagnostics first.
+    const bool inline_diagnostics = !result.GetImmediateErrorStream() &&
+                                    GetDebugger().GetShowInlineDiagnostics();
+    if (inline_diagnostics) {
+      unsigned prompt_len = m_debugger.GetPrompt().size();
+      if (auto indent = result.GetDiagnosticIndent()) {
+        std::string diags =
+            result.GetInlineDiagnosticString(prompt_len + *indent);
+        PrintCommandOutput(io_handler, diags, true);
+      }
+    }
+
+    // Display any STDOUT/STDERR _prior_ to emitting the command result text.
     GetProcessOutput();
 
     if (!result.GetImmediateOutputStream()) {
-      llvm::StringRef output = result.GetOutputData();
+      llvm::StringRef output = result.GetOutputString();
       PrintCommandOutput(io_handler, output, true);
     }
 
-    // Now emit the command error text from the command we just executed
+    // Now emit the command error text from the command we just executed.
     if (!result.GetImmediateErrorStream()) {
-      llvm::StringRef error = result.GetErrorData();
+      std::string error = result.GetErrorString(!inline_diagnostics);
       PrintCommandOutput(io_handler, error, false);
     }
   }
diff --git a/lldb/source/Interpreter/CommandObject.cpp b/lldb/source/Interpreter/CommandObject.cpp
index cf2682cd26faa0c3d22abc7d64f69fdc91f09003..6b044a28eb37eead4dd489880a62d2b00f7f9296 100644
--- a/lldb/source/Interpreter/CommandObject.cpp
+++ b/lldb/source/Interpreter/CommandObject.cpp
@@ -127,16 +127,7 @@ bool CommandObject::ParseOptions(Args &args, CommandReturnObject &result) {
       if (options->VerifyOptions(result))
         return true;
     } else {
-      const char *error_cstr = error.AsCString();
-      if (error_cstr) {
-        // We got an error string, lets use that
-        result.AppendError(error_cstr);
-      } else {
-        // No error string, output the usage information into result
-        options->GenerateOptionUsage(
-            result.GetErrorStream(), *this,
-            GetCommandInterpreter().GetDebugger().GetTerminalWidth());
-      }
+      result.SetError(error.takeError());
     }
     result.SetStatus(eReturnStatusFailed);
     return false;
diff --git a/lldb/source/Interpreter/CommandReturnObject.cpp b/lldb/source/Interpreter/CommandReturnObject.cpp
index d5da73c00a52093f7f7a4f081c32f5331be55984..94f5ff608b2aea6412ad517273c0c719680ef8a9 100644
--- a/lldb/source/Interpreter/CommandReturnObject.cpp
+++ b/lldb/source/Interpreter/CommandReturnObject.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Interpreter/CommandReturnObject.h"
 
+#include "lldb/Utility/DiagnosticsRendering.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StreamString.h"
 
@@ -41,7 +42,7 @@ static void DumpStringToStreamWithNewline(Stream &strm, const std::string &s) {
 }
 
 CommandReturnObject::CommandReturnObject(bool colors)
-    : m_out_stream(colors), m_err_stream(colors) {}
+    : m_out_stream(colors), m_err_stream(colors), m_colors(colors) {}
 
 void CommandReturnObject::AppendErrorWithFormat(const char *format, ...) {
   SetStatus(eReturnStatusFailed);
@@ -112,8 +113,89 @@ void CommandReturnObject::SetError(Status error) {
 }
 
 void CommandReturnObject::SetError(llvm::Error error) {
-  if (error)
+  // Retrieve any diagnostics.
+  error = llvm::handleErrors(std::move(error), [&](DiagnosticError &error) {
+    SetStatus(eReturnStatusFailed);
+    m_diagnostics = error.GetDetails();
+  });
+  if (error) {
     AppendError(llvm::toString(std::move(error)));
+  }
+}
+
+std::string CommandReturnObject::GetInlineDiagnosticString(unsigned indent) {
+  StreamString diag_stream(m_colors);
+  RenderDiagnosticDetails(diag_stream, indent, true, m_diagnostics);
+  // Duplex the diagnostics to the secondary stream (but not inlined).
+  if (auto stream_sp = m_err_stream.GetStreamAtIndex(eImmediateStreamIndex))
+    RenderDiagnosticDetails(*stream_sp, std::nullopt, false, m_diagnostics);
+
+  return diag_stream.GetString().str();
+}
+
+std::string CommandReturnObject::GetErrorString(bool with_diagnostics) {
+  StreamString stream(m_colors);
+  if (with_diagnostics)
+    RenderDiagnosticDetails(stream, std::nullopt, false, m_diagnostics);
+
+  lldb::StreamSP stream_sp(m_err_stream.GetStreamAtIndex(eStreamStringIndex));
+  if (stream_sp)
+    stream << std::static_pointer_cast<StreamString>(stream_sp)->GetString();
+  return stream.GetString().str();
+}
+
+StructuredData::ObjectSP CommandReturnObject::GetErrorData() {
+  auto make_array = []() { return std::make_unique<StructuredData::Array>(); };
+  auto make_bool = [](bool b) {
+    return std::make_unique<StructuredData::Boolean>(b);
+  };
+  auto make_dict = []() {
+    return std::make_unique<StructuredData::Dictionary>();
+  };
+  auto make_int = [](unsigned i) {
+    return std::make_unique<StructuredData::UnsignedInteger>(i);
+  };
+  auto make_string = [](llvm::StringRef s) {
+    return std::make_unique<StructuredData::String>(s);
+  };
+  auto dict_up = make_dict();
+  dict_up->AddItem("version", make_int(1));
+  auto array_up = make_array();
+  for (const DiagnosticDetail &diag : m_diagnostics) {
+    auto detail_up = make_dict();
+    if (auto &sloc = diag.source_location) {
+      auto sloc_up = make_dict();
+      sloc_up->AddItem("file", make_string(sloc->file.GetPath()));
+      sloc_up->AddItem("line", make_int(sloc->line));
+      sloc_up->AddItem("length", make_int(sloc->length));
+      sloc_up->AddItem("hidden", make_bool(sloc->hidden));
+      sloc_up->AddItem("in_user_input", make_bool(sloc->in_user_input));
+      detail_up->AddItem("source_location", std::move(sloc_up));
+    }
+    llvm::StringRef severity = "unknown";
+    switch (diag.severity) {
+    case lldb::eSeverityError:
+      severity = "error";
+      break;
+    case lldb::eSeverityWarning:
+      severity = "warning";
+      break;
+    case lldb::eSeverityInfo:
+      severity = "note";
+      break;
+    }
+    detail_up->AddItem("severity", make_string(severity));
+    detail_up->AddItem("message", make_string(diag.message));
+    detail_up->AddItem("rendered", make_string(diag.rendered));
+    array_up->AddItem(std::move(detail_up));
+  }
+  dict_up->AddItem("details", std::move(array_up));
+  if (auto stream_sp = m_err_stream.GetStreamAtIndex(eStreamStringIndex)) {
+    auto text = std::static_pointer_cast<StreamString>(stream_sp)->GetString();
+    if (!text.empty())
+      dict_up->AddItem("text", make_string(text));
+  }
+  return dict_up;
 }
 
 // Similar to AppendError, but do not prepend 'Status: ' to message, and don't
@@ -146,6 +228,7 @@ void CommandReturnObject::Clear() {
   stream_sp = m_err_stream.GetStreamAtIndex(eStreamStringIndex);
   if (stream_sp)
     static_cast<StreamString *>(stream_sp.get())->Clear();
+  m_diagnostics.clear();
   m_status = eReturnStatusStarted;
   m_did_change_process_state = false;
   m_suppress_immediate_output = false;
diff --git a/lldb/source/Interpreter/Options.cpp b/lldb/source/Interpreter/Options.cpp
index 3888a5812628cd611a2f4cd8b1a74958d2ec763d..6a90b2cc9b9896d40628937afde86407cc082d1d 100644
--- a/lldb/source/Interpreter/Options.cpp
+++ b/lldb/source/Interpreter/Options.cpp
@@ -19,12 +19,47 @@
 #include "lldb/Interpreter/CommandObject.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Target/Target.h"
+#include "lldb/Utility/DiagnosticsRendering.h"
 #include "lldb/Utility/StreamString.h"
 #include "llvm/ADT/STLExtras.h"
 
 using namespace lldb;
 using namespace lldb_private;
 
+namespace lldb_private {
+
+/// An llvm::Error that represents an option parsing diagnostic.
+class OptionParseError
+    : public llvm::ErrorInfo<OptionParseError, DiagnosticError> {
+  std::vector<DiagnosticDetail> m_details;
+
+public:
+  using llvm::ErrorInfo<OptionParseError, DiagnosticError>::ErrorInfo;
+  OptionParseError(DiagnosticDetail detail)
+      : ErrorInfo(std::error_code(EINVAL, std::generic_category())),
+        m_details({detail}) {}
+  OptionParseError(const Args::ArgEntry &arg, std::string msg)
+      : ErrorInfo(std::error_code(EINVAL, std::generic_category())) {
+    DiagnosticDetail::SourceLocation sloc;
+    if (auto pos = arg.GetPos()) {
+      uint16_t len = arg.GetLength();
+      sloc = {FileSpec{}, 1, *pos, len, false, true};
+    }
+    m_details.push_back(DiagnosticDetail{sloc, lldb::eSeverityError, msg, msg});
+  }
+  std::unique_ptr<CloneableError> Clone() const override {
+    return std::make_unique<OptionParseError>(m_details[0]);
+  }
+  llvm::ArrayRef<DiagnosticDetail> GetDetails() const override {
+    return m_details;
+  }
+  static char ID;
+};
+
+char OptionParseError::ID;
+
+} // namespace lldb_private
+
 // Options
 Options::Options() { BuildValidOptionSets(); }
 
@@ -1254,13 +1289,13 @@ llvm::Expected<Args> Options::Parse(const Args &args,
 
   std::string short_options = BuildShortOptions(long_options);
   std::vector<char *> argv = GetArgvForParsing(args);
+
   std::unique_lock<std::mutex> lock;
   OptionParser::Prepare(lock);
-  int val;
   while (true) {
     int long_options_index = -1;
-    val = OptionParser::Parse(argv, short_options, long_options,
-                              &long_options_index);
+    int val = OptionParser::Parse(argv, short_options, long_options,
+                                  &long_options_index);
 
     if (val == ':') {
       error = Status::FromErrorString("last option requires an argument");
@@ -1272,7 +1307,14 @@ llvm::Expected<Args> Options::Parse(const Args &args,
 
     // Did we get an error?
     if (val == '?') {
-      error = Status::FromErrorString("unknown or ambiguous option");
+      // Account for "argv[0]" and that it points to the next option.
+      int idx = OptionParser::GetOptionIndex() - 2;
+      if (idx >= 0 && (size_t)idx < args.GetArgumentCount())
+        error = Status::FromError(llvm::make_error<OptionParseError>(
+            args[idx], "unknown or ambiguous option"));
+      else
+        error = Status("unknown or ambiguous option");
+
       break;
     }
     // The option auto-set itself
diff --git a/lldb/source/Utility/Args.cpp b/lldb/source/Utility/Args.cpp
index 13b993bc74c9f4d84a194a92a71507646008ee4f..8ba40bae4d67e5d8a24086d3e805d2aa1245fc2c 100644
--- a/lldb/source/Utility/Args.cpp
+++ b/lldb/source/Utility/Args.cpp
@@ -66,9 +66,11 @@ static size_t ArgvToArgc(const char **argv) {
 
 // Trims all whitespace that can separate command line arguments from the left
 // side of the string.
-static llvm::StringRef ltrimForArgs(llvm::StringRef str) {
+static llvm::StringRef ltrimForArgs(llvm::StringRef str, size_t &shift) {
   static const char *k_space_separators = " \t";
-  return str.ltrim(k_space_separators);
+  llvm::StringRef result = str.ltrim(k_space_separators);
+  shift = result.data() - str.data();
+  return result;
 }
 
 // A helper function for SetCommandString. Parses a single argument from the
@@ -156,7 +158,9 @@ ParseSingleArgument(llvm::StringRef command) {
   return std::make_tuple(arg, first_quote_char, command);
 }
 
-Args::ArgEntry::ArgEntry(llvm::StringRef str, char quote) : quote(quote) {
+Args::ArgEntry::ArgEntry(llvm::StringRef str, char quote,
+                         std::optional<uint16_t> column)
+    : quote(quote), column(column) {
   size_t size = str.size();
   ptr.reset(new char[size + 1]);
 
@@ -185,7 +189,7 @@ Args &Args::operator=(const Args &rhs) {
   m_argv.clear();
   m_entries.clear();
   for (auto &entry : rhs.m_entries) {
-    m_entries.emplace_back(entry.ref(), entry.quote);
+    m_entries.emplace_back(entry.ref(), entry.quote, entry.column);
     m_argv.push_back(m_entries.back().data());
   }
   m_argv.push_back(nullptr);
@@ -248,14 +252,20 @@ void Args::SetCommandString(llvm::StringRef command) {
   Clear();
   m_argv.clear();
 
-  command = ltrimForArgs(command);
+  uint16_t column = 1;
+  size_t shift = 0;
+  command = ltrimForArgs(command, shift);
+  column += shift;
   std::string arg;
   char quote;
   while (!command.empty()) {
+    const char *prev = command.data();
     std::tie(arg, quote, command) = ParseSingleArgument(command);
-    m_entries.emplace_back(arg, quote);
+    m_entries.emplace_back(arg, quote, column);
     m_argv.push_back(m_entries.back().data());
-    command = ltrimForArgs(command);
+    command = ltrimForArgs(command, shift);
+    column += shift;
+    column += command.data() - prev;
   }
   m_argv.push_back(nullptr);
 }
@@ -299,7 +309,7 @@ void Args::AppendArguments(const Args &rhs) {
   assert(m_argv.back() == nullptr);
   m_argv.pop_back();
   for (auto &entry : rhs.m_entries) {
-    m_entries.emplace_back(entry.ref(), entry.quote);
+    m_entries.emplace_back(entry.ref(), entry.quote, entry.column);
     m_argv.push_back(m_entries.back().data());
   }
   m_argv.push_back(nullptr);
@@ -312,7 +322,7 @@ void Args::AppendArguments(const char **argv) {
   assert(m_argv.back() == nullptr);
   m_argv.pop_back();
   for (auto arg : llvm::ArrayRef(argv, argc)) {
-    m_entries.emplace_back(arg, '\0');
+    m_entries.emplace_back(arg, '\0', std::nullopt);
     m_argv.push_back(m_entries.back().data());
   }
 
@@ -330,7 +340,7 @@ void Args::InsertArgumentAtIndex(size_t idx, llvm::StringRef arg_str,
 
   if (idx > m_entries.size())
     return;
-  m_entries.emplace(m_entries.begin() + idx, arg_str, quote_char);
+  m_entries.emplace(m_entries.begin() + idx, arg_str, quote_char, std::nullopt);
   m_argv.insert(m_argv.begin() + idx, m_entries[idx].data());
 }
 
@@ -342,7 +352,7 @@ void Args::ReplaceArgumentAtIndex(size_t idx, llvm::StringRef arg_str,
   if (idx >= m_entries.size())
     return;
 
-  m_entries[idx] = ArgEntry(arg_str, quote_char);
+  m_entries[idx] = ArgEntry(arg_str, quote_char, std::nullopt);
   m_argv[idx] = m_entries[idx].data();
 }
 
@@ -366,7 +376,7 @@ void Args::SetArguments(size_t argc, const char **argv) {
             ? args[i][0]
             : '\0';
 
-    m_entries[i] = ArgEntry(args[i], quote);
+    m_entries[i] = ArgEntry(args[i], quote, std::nullopt);
     m_argv[i] = m_entries[i].data();
   }
 }
@@ -635,7 +645,8 @@ OptionsWithRaw::OptionsWithRaw(llvm::StringRef arg_string) {
 void OptionsWithRaw::SetFromString(llvm::StringRef arg_string) {
   const llvm::StringRef original_args = arg_string;
 
-  arg_string = ltrimForArgs(arg_string);
+  size_t shift;
+  arg_string = ltrimForArgs(arg_string, shift);
   std::string arg;
   char quote;
 
@@ -656,7 +667,7 @@ void OptionsWithRaw::SetFromString(llvm::StringRef arg_string) {
 
     // If we get an unquoted '--' argument, then we reached the suffix part
     // of the command.
-    Args::ArgEntry entry(arg, quote);
+    Args::ArgEntry entry(arg, quote, std::nullopt);
     if (!entry.IsQuoted() && arg == "--") {
       // The remaining line is the raw suffix, and the line we parsed so far
       // needs to be interpreted as arguments.
@@ -681,7 +692,7 @@ void OptionsWithRaw::SetFromString(llvm::StringRef arg_string) {
       break;
     }
 
-    arg_string = ltrimForArgs(arg_string);
+    arg_string = ltrimForArgs(arg_string, shift);
   }
 
   // If we didn't find a suffix delimiter, the whole string is the raw suffix.
diff --git a/lldb/source/Utility/CMakeLists.txt b/lldb/source/Utility/CMakeLists.txt
index 397db0e897602344a2bf86cc449ac65c78f55760..6954a2508ffe1c5058f77fe4017d39e2fb29edc6 100644
--- a/lldb/source/Utility/CMakeLists.txt
+++ b/lldb/source/Utility/CMakeLists.txt
@@ -38,6 +38,7 @@ add_lldb_library(lldbUtility NO_INTERNAL_DEPENDENCIES
   DataEncoder.cpp
   DataExtractor.cpp
   Diagnostics.cpp
+  DiagnosticsRendering.cpp
   Environment.cpp
   ErrorMessages.cpp
   Event.cpp
diff --git a/lldb/source/Commands/DiagnosticRendering.h b/lldb/source/Utility/DiagnosticsRendering.cpp
similarity index 70%
rename from lldb/source/Commands/DiagnosticRendering.h
rename to lldb/source/Utility/DiagnosticsRendering.cpp
index 5fdd090253a8270af2e611bb96b9f972c80f88bb..96caf934cc23155a50f7af2ef4b94383934ab485 100644
--- a/lldb/source/Commands/DiagnosticRendering.h
+++ b/lldb/source/Utility/DiagnosticsRendering.cpp
@@ -1,4 +1,4 @@
-//===-- DiagnosticRendering.h -----------------------------------*- C++ -*-===//
+//===-- DiagnosticsRendering.cpp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,15 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_SOURCE_COMMANDS_DIAGNOSTICRENDERING_H
-#define LLDB_SOURCE_COMMANDS_DIAGNOSTICRENDERING_H
+#include "lldb/Utility/DiagnosticsRendering.h"
 
-#include "lldb/Expression/DiagnosticManager.h"
-#include "lldb/Utility/Stream.h"
-#include "llvm/Support/WithColor.h"
+using namespace lldb_private;
+using namespace lldb;
 
 namespace lldb_private {
 
+char DiagnosticError::ID;
+
+lldb::ErrorType DiagnosticError::GetErrorType() const {
+  return lldb::eErrorTypeExpression;
+}
+
 static llvm::raw_ostream &PrintSeverity(Stream &stream,
                                         lldb::Severity severity) {
   llvm::HighlightColor color;
@@ -36,12 +40,11 @@ static llvm::raw_ostream &PrintSeverity(Stream &stream,
   return llvm::WithColor(stream.AsRawOstream(), color, llvm::ColorMode::Enable)
          << text;
 }
-  
-// Public for unittesting.
-static void RenderDiagnosticDetails(Stream &stream,
-                                    std::optional<uint16_t> offset_in_command,
-                                    bool show_inline,
-                                    llvm::ArrayRef<DiagnosticDetail> details) {
+
+void RenderDiagnosticDetails(Stream &stream,
+                             std::optional<uint16_t> offset_in_command,
+                             bool show_inline,
+                             llvm::ArrayRef<DiagnosticDetail> details) {
   if (details.empty())
     return;
 
@@ -53,6 +56,27 @@ static void RenderDiagnosticDetails(Stream &stream,
     return;
   }
 
+  // Since there is no other way to find this out, use the color
+  // attribute as a proxy for whether the terminal supports Unicode
+  // characters.  In the future it might make sense to move this into
+  // Host so it can be customized for a specific platform.
+  llvm::StringRef cursor, underline, vbar, joint, hbar, spacer;
+  if (stream.AsRawOstream().colors_enabled()) {
+    cursor = "˄";
+    underline = "˜";
+    vbar = "│";
+    joint = "╰";
+    hbar = "─";
+    spacer = " ";
+  } else {
+    cursor = "^";
+    underline = "~";
+    vbar = "|";
+    joint = "";
+    hbar = "";
+    spacer = "";
+  }
+
   // Print a line with caret indicator(s) below the lldb prompt + command.
   const size_t padding = *offset_in_command;
   stream << std::string(padding, ' ');
@@ -78,9 +102,9 @@ static void RenderDiagnosticDetails(Stream &stream,
     remaining_details.push_back(detail);
     if (offset > loc.column)
       continue;
-    stream << std::string(loc.column - offset, ' ') << '^';
-    if (loc.length > 1)
-      stream << std::string(loc.length - 1, '~');
+    stream << std::string(loc.column - offset, ' ') << cursor;
+    for (unsigned i = 0; i + 1 < loc.length; ++i)
+      stream << underline;
     offset = loc.column + 1;
   }
   stream << '\n';
@@ -97,14 +121,14 @@ static void RenderDiagnosticDetails(Stream &stream,
     for (auto &remaining_detail :
          llvm::ArrayRef(remaining_details).drop_back(1)) {
       uint16_t column = remaining_detail.source_location->column;
-      stream << std::string(column - offset, ' ') << "│";
+      stream << std::string(column - offset, ' ') << vbar;
       offset = column + 1;
     }
 
     // Print the line connecting the ^ with the error message.
     uint16_t column = detail->source_location->column;
     if (offset <= column)
-      stream << std::string(column - offset, ' ') << "╰─ ";
+      stream << std::string(column - offset, ' ') << joint << hbar << spacer;
 
     // Print a colorized string based on the message's severity type.
     PrintSeverity(stream, detail->severity);
@@ -130,4 +154,3 @@ static void RenderDiagnosticDetails(Stream &stream,
 }
 
 } // namespace lldb_private
-#endif
diff --git a/lldb/source/Utility/Status.cpp b/lldb/source/Utility/Status.cpp
index cf3772bc480ba36ec5547fcfc2e7e73c4de83467..1d171c6b6c374649a1a7ef9f7f38d481b7c8f01d 100644
--- a/lldb/source/Utility/Status.cpp
+++ b/lldb/source/Utility/Status.cpp
@@ -43,7 +43,6 @@ char CloneableError::ID;
 char CloneableECError::ID;
 char MachKernelError::ID;
 char Win32Error::ID;
-char ExpressionErrorBase::ID;
 
 namespace {
 /// A std::error_code category for eErrorTypeGeneric.
@@ -253,10 +252,6 @@ lldb::ErrorType Win32Error::GetErrorType() const {
   return lldb::eErrorTypeWin32;
 }
 
-lldb::ErrorType ExpressionErrorBase::GetErrorType() const {
-  return lldb::eErrorTypeExpression;
-}
-
 ErrorType Status::GetType() const {
   ErrorType result = eErrorTypeInvalid;
   llvm::visitErrors(m_error, [&](const llvm::ErrorInfoBase &error) {
@@ -286,29 +281,3 @@ void llvm::format_provider<lldb_private::Status>::format(
   llvm::format_provider<llvm::StringRef>::format(error.AsCString(), OS,
                                                  Options);
 }
-
-const char *lldb_private::ExpressionResultAsCString(ExpressionResults result) {
-  switch (result) {
-  case eExpressionCompleted:
-    return "eExpressionCompleted";
-  case eExpressionDiscarded:
-    return "eExpressionDiscarded";
-  case eExpressionInterrupted:
-    return "eExpressionInterrupted";
-  case eExpressionHitBreakpoint:
-    return "eExpressionHitBreakpoint";
-  case eExpressionSetupError:
-    return "eExpressionSetupError";
-  case eExpressionParseError:
-    return "eExpressionParseError";
-  case eExpressionResultUnavailable:
-    return "eExpressionResultUnavailable";
-  case eExpressionTimedOut:
-    return "eExpressionTimedOut";
-  case eExpressionStoppedForDebug:
-    return "eExpressionStoppedForDebug";
-  case eExpressionThreadVanished:
-    return "eExpressionThreadVanished";
-  }
-  return "<unknown>";
-}
diff --git a/lldb/test/API/CMakeLists.txt b/lldb/test/API/CMakeLists.txt
index 896ce2c8c2772461ba1b0f3d007d905766a4c58e..2548cab5cfb5b8553187f7d674ea84b2db7149b7 100644
--- a/lldb/test/API/CMakeLists.txt
+++ b/lldb/test/API/CMakeLists.txt
@@ -96,14 +96,6 @@ if ( CMAKE_SYSTEM_NAME MATCHES "Windows" )
   endif()
 endif()
 
-if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows|Darwin")
-  list(APPEND LLDB_TEST_COMMON_ARGS_VAR
-    --env ARCHIVER=${CMAKE_AR} --env OBJCOPY=${CMAKE_OBJCOPY})
-else()
-  list(APPEND LLDB_TEST_COMMON_ARGS_VAR
-    --env OBJCOPY=${LLVM_TOOLS_BINARY_DIR}/llvm-objcopy${CMAKE_EXECUTABLE_SUFFIX})
-endif()
-
 if (NOT "${LLDB_LIT_TOOLS_DIR}" STREQUAL "")
   if (NOT EXISTS "${LLDB_LIT_TOOLS_DIR}")
     message(WARNING "LLDB_LIT_TOOLS_DIR ${LLDB_LIT_TOOLS_DIR} does not exist.")
diff --git a/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py b/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
index 1687b617350d927535ea939447367312818d5d0d..fac562edf9ece039f42b63b683ae41c7d929ffa6 100644
--- a/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
+++ b/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
@@ -184,53 +184,44 @@ note: candidate function not viable: requires single argument 'x', but 2 argumen
         # the first argument are probably stable enough that this test can check for them.
         self.assertIn("void NSLog(NSString *format", value.GetError().GetCString())
 
-    def test_command_expr_formatting(self):
-        """Test that the source and caret positions LLDB prints are correct"""
+    def test_command_expr_sbdata(self):
+        """Test the structured diagnostics data"""
         self.build()
 
         (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "// Break here", self.main_source_spec
         )
-        frame = thread.GetFrameAtIndex(0)
-        self.expect("settings set show-inline-diagnostics true")
-
-        def check(input_ref):
-            self.expect(input_ref[0], error=True, substrs=input_ref[1:])
-
-        check(
-            [
-                "expression -- a+b",
-                "              ^ ^",
-                "              │ ╰─ error: use of undeclared identifier 'b'",
-                "              ╰─ error: use of undeclared identifier 'a'",
-            ]
-        )
-
-        check(
-            [
-                "expr -- a",
-                "        ^",
-                "        ╰─ error: use of undeclared identifier 'a'",
-            ]
-        )
-        check(
-            [
-                "expr -i 0 -o 0 -- a",
-                "                  ^",
-                "                  ╰─ error: use of undeclared identifier 'a'",
-            ]
-        )
-
-        self.expect(
-            "expression --top-level -- template<typename T> T FOO(T x) { return x/2;}"
-        )
-        check(
-            [
-                'expression -- FOO("")',
-                "              ^",
-                "              ╰─ note: in instantiation of function template specialization 'FOO<const char *>' requested here",
-                "error: <user expression",
-                "invalid operands to binary expression",
-            ]
-        )
-        check(["expression --\na\n+\nb", "error: <user", "a", "error: <user", "b"])
+        interp = self.dbg.GetCommandInterpreter()
+        cro = lldb.SBCommandReturnObject()
+        interp.HandleCommand("expression -- a+b", cro)
+
+        diags = cro.GetErrorData()
+        # Version.
+        version = diags.GetValueForKey("version")
+        self.assertEqual(version.GetIntegerValue(), 1)
+
+        details = diags.GetValueForKey("details")
+
+        # Detail 1/2: undeclared 'a'
+        diag = details.GetItemAtIndex(0)
+
+        severity = diag.GetValueForKey("severity")
+        message = diag.GetValueForKey("message")
+        rendered = diag.GetValueForKey("rendered")
+        sloc = diag.GetValueForKey("source_location")
+        filename = sloc.GetValueForKey("file")
+        hidden = sloc.GetValueForKey("hidden")
+        in_user_input = sloc.GetValueForKey("in_user_input")
+
+        self.assertEqual(str(severity), "error")
+        self.assertIn("undeclared identifier 'a'", str(message))
+        # The rendered string should contain the source file.
+        self.assertIn("user expression", str(rendered))
+        self.assertIn("user expression", str(filename))
+        self.assertFalse(hidden.GetBooleanValue())
+        self.assertTrue(in_user_input.GetBooleanValue())
+
+        # Detail 1/2: undeclared 'b'
+        diag = details.GetItemAtIndex(1)
+        message = diag.GetValueForKey("message")
+        self.assertIn("undeclared identifier 'b'", str(message))
diff --git a/lldb/test/API/functionalities/completion/Makefile b/lldb/test/API/functionalities/completion/Makefile
index f46742243bd96d1fe358611d76d62db38d117eb8..e457693f62ca44929bb64413cda246f2d87ab4ef 100644
--- a/lldb/test/API/functionalities/completion/Makefile
+++ b/lldb/test/API/functionalities/completion/Makefile
@@ -4,7 +4,7 @@ USE_LIBDL := 1
 a.out: lib_shared
 
 lib_shared:
-	$(MAKE) -f $(MAKEFILE_RULES) \
+	"$(MAKE)" -f $(MAKEFILE_RULES) \
 		DYLIB_ONLY=YES DYLIB_CXX_SOURCES=shared.cpp DYLIB_NAME=shared
 
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
index 4818dde4f3b838dc6b91cba588151f5031c4759a..808de687e6ea2e74141e4a2e7e0257ac5006a433 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
+++ b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
@@ -117,14 +117,14 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
             expected_number_of_threads = process.GetNumThreads()
             expected_threads = []
             stacks_to_sp_map = {}
-            stakcs_to_registers_map = {}
+            stacks_to_registers_map = {}
 
             for thread_idx in range(process.GetNumThreads()):
                 thread = process.GetThreadAtIndex(thread_idx)
                 thread_id = thread.GetThreadID()
                 expected_threads.append(thread_id)
                 stacks_to_sp_map[thread_id] = thread.GetFrameAtIndex(0).GetSP()
-                stakcs_to_registers_map[thread_id] = thread.GetFrameAtIndex(
+                stacks_to_registers_map[thread_id] = thread.GetFrameAtIndex(
                     0
                 ).GetRegisters()
 
@@ -138,7 +138,7 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
                 expected_modules,
                 expected_threads,
                 stacks_to_sp_map,
-                stakcs_to_registers_map,
+                stacks_to_registers_map,
             )
 
             self.runCmd(base_command + " --style=modified-memory '%s'" % (core_dirty))
@@ -149,7 +149,7 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
                 expected_modules,
                 expected_threads,
                 stacks_to_sp_map,
-                stakcs_to_registers_map,
+                stacks_to_registers_map,
             )
 
             self.runCmd(base_command + " --style=full '%s'" % (core_full))
@@ -160,7 +160,7 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
                 expected_modules,
                 expected_threads,
                 stacks_to_sp_map,
-                stakcs_to_registers_map,
+                stacks_to_registers_map,
             )
 
             options = lldb.SBSaveCoreOptions()
@@ -178,7 +178,7 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
                 expected_modules,
                 expected_threads,
                 stacks_to_sp_map,
-                stakcs_to_registers_map,
+                stacks_to_registers_map,
             )
 
             options = lldb.SBSaveCoreOptions()
@@ -195,7 +195,7 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
                 expected_modules,
                 expected_threads,
                 stacks_to_sp_map,
-                stakcs_to_registers_map,
+                stacks_to_registers_map,
             )
 
             # Minidump can now save full core files, but they will be huge and
@@ -214,7 +214,7 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
                 expected_modules,
                 expected_threads,
                 stacks_to_sp_map,
-                stakcs_to_registers_map,
+                stacks_to_registers_map,
             )
 
             self.assertSuccess(process.Kill())
diff --git a/lldb/test/API/lang/cpp/stl/Makefile b/lldb/test/API/lang/cpp/stl/Makefile
index 99998b20bcb0502181cbf488a2034823e5f66d22..8534fa9b00209e7571120ae8df618e9a8ddb3d28 100644
--- a/lldb/test/API/lang/cpp/stl/Makefile
+++ b/lldb/test/API/lang/cpp/stl/Makefile
@@ -1,3 +1,5 @@
 CXX_SOURCES := main.cpp
 
+USE_SYSTEM_STDLIB := 1
+
 include Makefile.rules
diff --git a/lldb/test/API/lang/cpp/stl/TestStdCXXDisassembly.py b/lldb/test/API/lang/cpp/stl/TestStdCXXDisassembly.py
index 1e1b0a4d621f040be79a8b4957e7bcbb6a8120ac..8676ee16d83c04355c3806ea7e862bcb1e453f59 100644
--- a/lldb/test/API/lang/cpp/stl/TestStdCXXDisassembly.py
+++ b/lldb/test/API/lang/cpp/stl/TestStdCXXDisassembly.py
@@ -43,9 +43,7 @@ class StdCXXDisassembleTestCase(TestBase):
         # At this point, lib_stdcxx is the full path to the stdc++ library and
         # module is the corresponding SBModule.
 
-        self.expect(
-            lib_stdcxx, "Libraray StdC++ is located", exe=False, substrs=["lib"]
-        )
+        self.expect(lib_stdcxx, "Library StdC++ is located", exe=False, substrs=["lib"])
 
         self.runCmd("image dump symtab '%s'" % lib_stdcxx)
         raw_output = self.res.GetOutput()
diff --git a/lldb/test/API/macosx/ctf/Makefile b/lldb/test/API/macosx/ctf/Makefile
index 0857e234837e543811bb2c68f7ab82c59634b587..eeef9351e3ef138ca6f55f4021a6cbc17112e1d7 100644
--- a/lldb/test/API/macosx/ctf/Makefile
+++ b/lldb/test/API/macosx/ctf/Makefile
@@ -17,15 +17,19 @@ a.ctf: a.out.dSYM
 		-o a.ctf \
 		a.out.dSYM/Contents/Resources/DWARF/a.out
 	$(OBJCOPY) \
-		-R __DWARF,__debug_line \
-		-R __DWARF,__debug_aranges \
-		-R __DWARF,__debug_info \
-		-R __DWARF,__debug_abbrev \
-		-R __DWARF,__debug_str \
 		-R __DWARF,__apple_names \
 		-R __DWARF,__apple_namespac \
-		-R __DWARF,__apple_types \
 		-R __DWARF,__apple_objc \
+		-R __DWARF,__apple_types \
+		-R __DWARF,__debug_abbrev \
+		-R __DWARF,__debug_addr \
+		-R __DWARF,__debug_aranges \
+		-R __DWARF,__debug_info \
+		-R __DWARF,__debug_line \
+		-R __DWARF,__debug_line_str \
+		-R __DWARF,__debug_names \
+		-R __DWARF,__debug_str \
+		-R __DWARF,__debug_str_offs \
 		a.ctf a.ctf
 	rm -rf a.out.dSYM
 	rm -rf test.o
diff --git a/lldb/test/Shell/Commands/Inputs/multiline-expr.txt b/lldb/test/Shell/Commands/Inputs/multiline-expr.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4cea8eaeefbe7773e8c2b44d80d77eab24ddae01
--- /dev/null
+++ b/lldb/test/Shell/Commands/Inputs/multiline-expr.txt
@@ -0,0 +1,6 @@
+expression --
+a
++
+b
+
+quit
diff --git a/lldb/test/Shell/Commands/command-dwim-print.test b/lldb/test/Shell/Commands/command-dwim-print.test
new file mode 100644
index 0000000000000000000000000000000000000000..9153edbd2179138e727986b0edc390c274f5093e
--- /dev/null
+++ b/lldb/test/Shell/Commands/command-dwim-print.test
@@ -0,0 +1,16 @@
+# RUN: echo quit | %lldb -o "dwim-print a" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK1
+#            (lldb) dwim-print a 
+# CHECK1:{{^                  \^}}
+# CHECK1: {{^                  error: use of undeclared identifier 'a'}}
+# RUN: echo quit | %lldb -o "p a" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK2
+#            (lldb) p a 
+# CHECK2:{{^         \^}}
+# RUN: echo quit | %lldb -o "dwim-print -- a" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK3
+#            (lldb) dwim-print -- a 
+# CHECK3:{{^                     \^}}
+# RUN: echo quit | %lldb -o "settings set show-inline-diagnostics false" \
+# RUN:   -o "dwim-print a" 2>&1 | FileCheck %s --check-prefix=CHECK4
+# CHECK4: error: <user expression 0>:1:1: use of undeclared identifier
diff --git a/lldb/test/Shell/Commands/command-expr-diagnostics.test b/lldb/test/Shell/Commands/command-expr-diagnostics.test
new file mode 100644
index 0000000000000000000000000000000000000000..b242dba1980f0ff003aee0a6806e2820e3f62a23
--- /dev/null
+++ b/lldb/test/Shell/Commands/command-expr-diagnostics.test
@@ -0,0 +1,32 @@
+# RUN: echo quit | %lldb -o "expression a+b" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK1
+#            (lldb) expression a+b
+# CHECK1:{{^                  \^ \^}}
+# CHECK1: {{^                  | error: use of undeclared identifier 'b'}}
+# CHECK1: {{^                  error: use of undeclared identifier 'a'}}
+
+# RUN: echo quit | %lldb -o "expr a" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK2
+#            (lldb) expr a 
+# CHECK2:{{^            \^}}
+
+# RUN: echo quit | %lldb -o "expr -i 0 -o 0 -- a" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK3
+#            (lldb) expr -i 0 -o 0 -- a
+# CHECK3:{{^                         \^}}
+# CHECK3: {{^                         error: use of undeclared identifier 'a'}}
+
+# RUN: echo "int main(){return 0;}">%t.c
+# RUN: %clang_host %t.c -o %t.exe
+# RUN: echo quit | %lldb %t.exe -o "b main" -o r -o \
+# RUN: "expr --top-level -- template<typename T> T FOO(T x) { return x/2;}" -o \
+# RUN: "expression -- FOO(\"\")" 2>&1 | FileCheck %s --check-prefix=CHECK4
+#            (lldb) expression -- FOO("")
+# CHECK4:{{^                     \^}}
+# CHECK4: {{^                     note: in instantiation of function template}}
+# CHECK4: error: <user expression
+
+# RUN: echo expression --\na\n+\nb
+# RUN: cat %S/Inputs/multiline-expr.txt | %lldb 2>&1 | FileCheck %s --strict-whitespace --check-prefix=CHECK5
+# CHECK5: error: <user{{.*}}a
+# CHECK5: error: <user{{.*}}b
diff --git a/lldb/test/Shell/Commands/command-options.test b/lldb/test/Shell/Commands/command-options.test
new file mode 100644
index 0000000000000000000000000000000000000000..73aa374bde297ddca40d44bd7b0ccdc0c4d994ef
--- /dev/null
+++ b/lldb/test/Shell/Commands/command-options.test
@@ -0,0 +1,16 @@
+# RUN: echo quit | %lldb -O "log enable -x" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK1
+#            (lldb) log enable -x
+# CHECK1:{{^                  \^~}}
+# CHECK1: {{^                  error: unknown or ambiguous option}}
+
+# RUN: echo quit | %lldb -O "    log enable -xxxxxxx" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK2
+#            (lldb)     log enable -xxxxxxx
+# CHECK2:{{^                      \^~~~~~~~}}
+# CHECK2: {{^                      error: unknown or ambiguous option}}
+# RUN: echo quit | %lldb -O "log enable dwarf all -f dwarf.log -x" \
+# RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK3
+#            (lldb) log enable dwarf all -f dwarf.log -x
+# CHECK3:{{^                                         \^~}}
+# CHECK3: {{^                                         error: unknown or ambiguous option}}
diff --git a/lldb/test/Shell/SymbolFile/DWARF/no_unique_address-with-bitfields.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/no_unique_address-with-bitfields.cpp
similarity index 100%
rename from lldb/test/Shell/SymbolFile/DWARF/no_unique_address-with-bitfields.cpp
rename to lldb/test/Shell/SymbolFile/DWARF/x86/no_unique_address-with-bitfields.cpp
diff --git a/lldb/tools/debugserver/source/RNBServices.cpp b/lldb/tools/debugserver/source/RNBServices.cpp
index 6e4b55e3e6466da840faf80ae2f10251117a8698..bb57fb5ea027eb84d1dc4a2d0db0d7863bc031f5 100644
--- a/lldb/tools/debugserver/source/RNBServices.cpp
+++ b/lldb/tools/debugserver/source/RNBServices.cpp
@@ -208,8 +208,8 @@ int ListApplications(std::string &plist, bool opt_runningApps,
   GetProcesses(plistMutableArray.get(), all_users);
 #endif
 
-  CFReleaser<CFDataRef> plistData(
-      ::CFPropertyListCreateXMLData(alloc, plistMutableArray.get()));
+  CFReleaser<CFDataRef> plistData(::CFPropertyListCreateData(
+      alloc, plistMutableArray.get(), kCFPropertyListXMLFormat_v1_0, 0, NULL));
 
   // write plist to service port
   if (plistData.get() != NULL) {
diff --git a/lldb/tools/debugserver/source/debugserver.cpp b/lldb/tools/debugserver/source/debugserver.cpp
index cfc9646ed2d0811d0a78a9f5096aff45f7869335..f41a9e00ec9481fc7574263601de0e4de44af59a 100644
--- a/lldb/tools/debugserver/source/debugserver.cpp
+++ b/lldb/tools/debugserver/source/debugserver.cpp
@@ -759,35 +759,6 @@ static int ConnectRemote(RNBRemote *remote, const char *host, int port,
   return 1;
 }
 
-// ASL Logging callback that can be registered with DNBLogSetLogCallback
-void ASLLogCallback(void *baton, uint32_t flags, const char *format,
-                    va_list args) {
-  if (format == NULL)
-    return;
-  static aslmsg g_aslmsg = NULL;
-  if (g_aslmsg == NULL) {
-    g_aslmsg = ::asl_new(ASL_TYPE_MSG);
-    char asl_key_sender[PATH_MAX];
-    snprintf(asl_key_sender, sizeof(asl_key_sender), "com.apple.%s-%s",
-             DEBUGSERVER_PROGRAM_NAME, DEBUGSERVER_VERSION_STR);
-    ::asl_set(g_aslmsg, ASL_KEY_SENDER, asl_key_sender);
-  }
-
-  int asl_level;
-  if (flags & DNBLOG_FLAG_FATAL)
-    asl_level = ASL_LEVEL_CRIT;
-  else if (flags & DNBLOG_FLAG_ERROR)
-    asl_level = ASL_LEVEL_ERR;
-  else if (flags & DNBLOG_FLAG_WARNING)
-    asl_level = ASL_LEVEL_WARNING;
-  else if (flags & DNBLOG_FLAG_VERBOSE)
-    asl_level = ASL_LEVEL_WARNING; // ASL_LEVEL_INFO;
-  else
-    asl_level = ASL_LEVEL_WARNING; // ASL_LEVEL_DEBUG;
-
-  ::asl_vlog(NULL, g_aslmsg, asl_level, format, args);
-}
-
 // FILE based Logging callback that can be registered with
 // DNBLogSetLogCallback
 void FileLogCallback(void *baton, uint32_t flags, const char *format,
@@ -948,16 +919,8 @@ int main(int argc, char *argv[]) {
   // Set up DNB logging by default. If the user passes different log flags or a
   // log file, these settings will be modified after processing the command line
   // arguments.
-  auto log_callback = OsLogger::GetLogFunction();
-  if (log_callback) {
-    // if os_log() support is available, log through that.
+  if (auto log_callback = OsLogger::GetLogFunction())
     DNBLogSetLogCallback(log_callback, nullptr);
-    DNBLog("debugserver will use os_log for internal logging.");
-  } else {
-    // Fall back to ASL support.
-    DNBLogSetLogCallback(ASLLogCallback, nullptr);
-    DNBLog("debugserver will use ASL for internal logging.");
-  }
   DNBLogSetLogMask(/*log_flags*/ 0);
 
   g_remoteSP = std::make_shared<RNBRemote>();
diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp
index ce21e3d4eda058443bd2d6eff85d18e5b96ad021..50b85d4b51209c20087ca34fe3d69eb1aa3c1925 100644
--- a/lldb/tools/lldb-test/lldb-test.cpp
+++ b/lldb/tools/lldb-test/lldb-test.cpp
@@ -439,7 +439,7 @@ int opts::breakpoint::evaluateBreakpoints(Debugger &Dbg) {
     CommandReturnObject Result(/*colors*/ false);
     if (!Dbg.GetCommandInterpreter().HandleCommand(
             Command.c_str(), /*add_to_history*/ eLazyBoolNo, Result)) {
-      P.formatLine("Failed: {0}", Result.GetErrorData());
+      P.formatLine("Failed: {0}", Result.GetErrorString());
       HadErrors = 1;
       continue;
     }
@@ -1161,7 +1161,7 @@ int opts::irmemorymap::evaluateMemoryMapCommands(Debugger &Dbg) {
     return CI.HandleCommand(Cmd, eLazyBoolNo, Result);
   };
   if (!IssueCmd("b main") || !IssueCmd("run")) {
-    outs() << formatv("Failed: {0}\n", Result.GetErrorData());
+    outs() << formatv("Failed: {0}\n", Result.GetErrorString());
     exit(1);
   }
 
diff --git a/lldb/unittests/Interpreter/CMakeLists.txt b/lldb/unittests/Interpreter/CMakeLists.txt
index f7d639f50f5bf3e9323f22bd9c3977f3e50775f7..d4ba5b3d58334a125a66d407c0e2ecb1fef0b2f3 100644
--- a/lldb/unittests/Interpreter/CMakeLists.txt
+++ b/lldb/unittests/Interpreter/CMakeLists.txt
@@ -1,6 +1,5 @@
 add_lldb_unittest(InterpreterTests
   TestCommandPaths.cpp
-  TestCommandObjectExpression.cpp
   TestCompletion.cpp
   TestOptionArgParser.cpp
   TestOptions.cpp
diff --git a/lldb/unittests/Utility/CMakeLists.txt b/lldb/unittests/Utility/CMakeLists.txt
index 40e0959fc01d1427c8ff4299994aeca2429ee08e..3a73bdc3e9e1a2030d5c9cc3793dd79c43d496d5 100644
--- a/lldb/unittests/Utility/CMakeLists.txt
+++ b/lldb/unittests/Utility/CMakeLists.txt
@@ -10,6 +10,7 @@ add_lldb_unittest(UtilityTests
   DataBufferTest.cpp
   DataEncoderTest.cpp
   DataExtractorTest.cpp
+  DiagnosticsRenderingTest.cpp
   EnvironmentTest.cpp
   EventTest.cpp
   FileSpecListTest.cpp
diff --git a/lldb/unittests/Interpreter/TestCommandObjectExpression.cpp b/lldb/unittests/Utility/DiagnosticsRenderingTest.cpp
similarity index 84%
rename from lldb/unittests/Interpreter/TestCommandObjectExpression.cpp
rename to lldb/unittests/Utility/DiagnosticsRenderingTest.cpp
index 9e3417b5428923d6bb0e299761871c6ab0ce17fb..2bd80796b8074c21a5a41c6e0ae65332b9cb0105 100644
--- a/lldb/unittests/Interpreter/TestCommandObjectExpression.cpp
+++ b/lldb/unittests/Utility/DiagnosticsRenderingTest.cpp
@@ -1,4 +1,4 @@
-#include "../../source/Commands/DiagnosticRendering.h"
+#include "lldb/Utility/DiagnosticsRendering.h"
 #include "lldb/Utility/StreamString.h"
 #include "gtest/gtest.h"
 
@@ -7,13 +7,13 @@ using namespace lldb;
 using llvm::StringRef;
 namespace {
 class ErrorDisplayTest : public ::testing::Test {};
-} // namespace
 
-static std::string Render(std::vector<DiagnosticDetail> details) {
+std::string Render(std::vector<DiagnosticDetail> details) {
   StreamString stream;
   RenderDiagnosticDetails(stream, 0, true, details);
   return stream.GetData();
 }
+} // namespace
 
 TEST_F(ErrorDisplayTest, RenderStatus) {
   DiagnosticDetail::SourceLocation inline_loc;
diff --git a/lldb/utils/lldb-dotest/CMakeLists.txt b/lldb/utils/lldb-dotest/CMakeLists.txt
index bb17a2ce017d5d778ee88c0bebe9a28ff19c29e6..3b8c88b6dc78cb5e85e0c8c425cbbff93f5d5f12 100644
--- a/lldb/utils/lldb-dotest/CMakeLists.txt
+++ b/lldb/utils/lldb-dotest/CMakeLists.txt
@@ -34,6 +34,7 @@ set(vars
   LLDB_TEST_EXECUTABLE
   LLDB_TEST_COMPILER
   LLDB_TEST_DSYMUTIL
+  LLDB_TEST_MAKE
   LLDB_LIBS_DIR
   LLVM_TOOLS_DIR
   LIBCXX_LIBRARY_DIR
diff --git a/lldb/utils/lldb-dotest/lldb-dotest.in b/lldb/utils/lldb-dotest/lldb-dotest.in
index 0e9648a6e6dc834c59f551403e805fd015d8d8c9..f20859e87b1e0b750f1e9870c7a4b2ce842a7456 100755
--- a/lldb/utils/lldb-dotest/lldb-dotest.in
+++ b/lldb/utils/lldb-dotest/lldb-dotest.in
@@ -9,6 +9,7 @@ arch = '@LLDB_TEST_ARCH@'
 executable = '@LLDB_TEST_EXECUTABLE_CONFIGURED@'
 compiler = '@LLDB_TEST_COMPILER_CONFIGURED@'
 dsymutil = '@LLDB_TEST_DSYMUTIL_CONFIGURED@'
+make = '@LLDB_TEST_MAKE_CONFIGURED@'
 lldb_build_dir = '@LLDB_TEST_BUILD_DIRECTORY_CONFIGURED@'
 lldb_build_intel_pt = "@LLDB_BUILD_INTEL_PT@"
 lldb_framework_dir = "@LLDB_FRAMEWORK_DIR_CONFIGURED@"
@@ -35,6 +36,7 @@ if __name__ == '__main__':
     cmd.extend(['--executable', executable])
     cmd.extend(['--compiler', compiler])
     cmd.extend(['--dsymutil', dsymutil])
+    cmd.extend(['--make', make])
     cmd.extend(['--lldb-libs-dir', lldb_libs_dir])
     cmd.extend(['--llvm-tools-dir', llvm_tools_dir])
     if has_libcxx:
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index 0dc0ed71858f84d86c082f3bf5ccaeac1f22cda0..cb215fb8b2d99fe9236c2fc7f19ccc55af7b4c46 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -354,10 +354,10 @@ martin@martin.st (email), [mstrorsjo](https://github.com/mstrorsjo) (GitHub)
 Michael Spencer \
 bigcheesegs@gmail.com (email), [Bigcheese](https://github.com/Bigcheese) (GitHub)
 
-#### Sony PlayStation®4 support
+#### Sony PlayStation support
 
-Paul Robinson \
-paul.robinson@sony.com (email), [pogo59](https://github.com/pogo59) (GitHub)
+Jeremy Morse \
+jeremy.morse@sony.com (email), [jmorse](https://github.com/jmorse) (GitHub)
 
 #### Inline assembly
 
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 6ff3272422fe95796509925da38d93295147d235..aba39762861dd8882ad34a85ade7252f1eea4c90 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1369,6 +1369,12 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    sign-extended from the width of the underlying PC hardware register even on
                                                    processors where the s_getpc_b64 instruction returns a zero-extended value.
 
+  llvm.amdgcn.ballot                               Returns a bitfield(i32 or i64) containing the result of its i1 argument
+                                                   in all active lanes, and zero in all inactive lanes.
+                                                   Provides a way to convert i1 in LLVM IR to i32 or i64 lane mask - bitfield
+                                                   used by hardware to control active lanes when used in EXEC register.
+                                                   For example, ballot(i1 true) return EXEC mask.
+
   ==============================================   ==========================================================
 
 .. TODO::
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index 63df5af2523db6438977b9c02e2d7cad817ba85d..6e90efe0f111971379f6ffd2d306b98ff9983732 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -1713,6 +1713,17 @@ would help to avoid running into a "dangling else" situation.
     handleOtherDecl(D);
   }
 
+  // Use braces for the `else if` and `else` block to keep it uniform with the
+  // `if` block.
+  if (isa<FunctionDecl>(D)) {
+    verifyFunctionDecl(D);
+    handleFunctionDecl(D);
+  } else if (isa<GlobalVarDecl>(D)) {
+    handleGlobalVarDecl(D);
+  } else {
+    handleOtherDecl(D);
+  }
+
   // This should also omit braces.  The `for` loop contains only a single
   // statement, so it shouldn't have braces.  The `if` also only contains a
   // single simple statement (the `for` loop), so it also should omit braces.
diff --git a/llvm/docs/Coroutines.rst b/llvm/docs/Coroutines.rst
index 8794df65504fa2f778f2364e3572f03d3024c255..92e138b6893b264e6d92a0d57b65fd7785c96ddb 100644
--- a/llvm/docs/Coroutines.rst
+++ b/llvm/docs/Coroutines.rst
@@ -1090,7 +1090,7 @@ The first and second arguments are identical to those of the `coro.begin`
 intrinsic.
 
 The third argument is an i32 index of the generator list given to the
-`CoroSplit` pass specifying the custom ABI generator lor this coroutine.
+`CoroSplit` pass specifying the custom ABI generator for this coroutine.
 
 Semantics:
 """"""""""
diff --git a/llvm/docs/HowToAddABuilder.rst b/llvm/docs/HowToAddABuilder.rst
index 133375370cd049faee3e4d056fb69c98d6f6e299..f07640f6627cf1ee34eea523439e4e25c4bab703 100644
--- a/llvm/docs/HowToAddABuilder.rst
+++ b/llvm/docs/HowToAddABuilder.rst
@@ -276,4 +276,33 @@ Leave it on the staging buildmaster
   impacting the broader community.  The sponsoring organization simply
   has to take on the responsibility of all bisection and triage.
 
-  
+Managing a Worker From The Web Interface
+========================================
+
+Tasks such as clearing pending building requests can be done using
+the Buildbot web interface. To do this you must be recognised as an admin
+of the worker:
+
+* Set your public GitHub profile email to one that was included in the
+  ``admin`` information you set up on the worker. It does not matter if this
+  is your primary account email or a "verified email". To confirm this has been
+  done correctly, go to ``github.com/<your GitHub username>`` and you should
+  see the email address listed there.
+
+  A worker can have many admins, if they are listed in the form
+  ``First Last <first.last@example.com>, First2 Last2 <first2.last2@example.com>``.
+  You only need to have one of those addresses in your profile to be recognised
+  as an admin.
+
+  If you need to add an email address, you can edit the ``admin`` file and
+  restart the worker. You should see the new admin details in the web interface
+  shortly afterwards.
+
+* Connect GitHub to Buildbot by clicking on the "Anonymous" button on the
+  top right of the page, then "Login with GitHub" and authorise the app.
+
+Some tasks don't give immediate feedback, so if nothing happens within a short
+time, try again with the browser's web console open. Sometimes you will see
+403 errors and other messages that might indicate you don't have the correct
+details set up.
+
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index a330b804930326a527b4e53ada78b85c0bf704c0..566d0d4e4e81a39980b4a7668395826eb1506aeb 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1309,12 +1309,14 @@ Currently, only the following parameter attributes are defined:
     structure that is the return value of the function in the source
     program. This pointer must be guaranteed by the caller to be valid:
     loads and stores to the structure may be assumed by the callee not
-    to trap and to be properly aligned. This is not a valid attribute
-    for return values.
+    to trap and to be properly aligned.
 
     The sret type argument specifies the in memory type, which must be
     the same as the pointee type of the argument.
 
+    A function that accepts an ``sret`` argument must return ``void``.
+    A return value may not be ``sret``.
+
 .. _attr_elementtype:
 
 ``elementtype(<ty>)``
@@ -2082,6 +2084,12 @@ example:
     function call, use of ``longjmp``, or other means. It is a compiler hint that
     is used at module level to improve dataflow analysis, dropped during linking,
     and has no effect on functions defined in the current module.
+``nodivergencesource``
+    A call to this function is not a source of divergence. In uniformity
+    analysis, a *source of divergence* is an instruction that generates
+    divergence even if its inputs are uniform. A call with no further information
+    would normally be considered a source of divergence; setting this attribute
+    on a function means that a call to it is not a source of divergence.
 ``noduplicate``
     This attribute indicates that calls to the function cannot be
     duplicated. A call to a ``noduplicate`` function may be moved
@@ -2427,6 +2435,8 @@ example:
     function which has an ``ssp`` or ``sspstrong`` attribute, the calling
     function's attribute will be upgraded to ``sspreq``.
 
+.. _strictfp:
+
 ``strictfp``
     This attribute indicates that the function was called from a scope that
     requires strict floating-point semantics.  LLVM will not attempt any
@@ -3604,11 +3614,12 @@ status flags are not observable. Therefore, floating-point math operations do
 not have side effects and may be speculated freely. Results assume the
 round-to-nearest rounding mode, and subnormals are assumed to be preserved.
 
-Running LLVM code in an environment where these assumptions are not met can lead
-to undefined behavior. The ``strictfp`` and ``denormal-fp-math`` attributes as
-well as :ref:`Constrained Floating-Point Intrinsics <constrainedfp>` can be used
-to weaken LLVM's assumptions and ensure defined behavior in non-default
-floating-point environments; see their respective documentation for details.
+Running LLVM code in an environment where these assumptions are not met
+typically leads to undefined behavior. The ``strictfp`` and ``denormal-fp-math``
+attributes as well as :ref:`Constrained Floating-Point Intrinsics
+<constrainedfp>` can be used to weaken LLVM's assumptions and ensure defined
+behavior in non-default floating-point environments; see their respective
+documentation for details.
 
 .. _floatnan:
 
@@ -3630,10 +3641,11 @@ are not "floating-point math operations": ``fneg``, ``llvm.fabs``, and
 ``llvm.copysign``. These operations act directly on the underlying bit
 representation and never change anything except possibly for the sign bit.
 
-For floating-point math operations, unless specified otherwise, the following
-rules apply when a NaN value is returned: the result has a non-deterministic
-sign; the quiet bit and payload are non-deterministically chosen from the
-following set of options:
+Floating-point math operations that return a NaN are an exception from the
+general principle that LLVM implements IEEE-754 semantics. Unless specified
+otherwise, the following rules apply whenever the IEEE-754 semantics say that a
+NaN value is returned: the result has a non-deterministic sign; the quiet bit
+and payload are non-deterministically chosen from the following set of options:
 
 - The quiet bit is set and the payload is all-zero. ("Preferred NaN" case)
 - The quiet bit is set and the payload is copied from any input operand that is
@@ -3679,6 +3691,40 @@ specification on some architectures:
   LLVM does not correctly represent this. See `issue #60796
   <https://github.com/llvm/llvm-project/issues/60796>`_.
 
+.. _floatsem:
+
+Floating-Point Semantics
+------------------------
+
+This section defines the semantics for core floating-point operations on types
+that use a format specified by IEEE-745. These types are: ``half``, ``float``,
+``double``, and ``fp128``, which correspond to the binary16, binary32, binary64,
+and binary128 formats, respectively. The "core" operations are those defined in
+section 5 of IEEE-745, which all have corresponding LLVM operations.
+
+The value returned by those operations matches that of the corresponding
+IEEE-754 operation executed in the :ref:`default LLVM floating-point environment
+<floatenv>`, except that the behavior of NaN results is instead :ref:`as
+specified here <floatnan>`. In particular, such a floating-point instruction
+returning a non-NaN value is guaranteed to always return the same bit-identical
+result on all machines and optimization levels.
+
+This means that optimizations and backends may not change the observed bitwise
+result of these operations in any way (unless NaNs are returned), and frontends
+can rely on these operations providing correctly rounded results as described in
+the standard.
+
+(Note that this is only about the value returned by these operations; see the
+:ref:`floating-point environment section <floatenv>` regarding flags and
+exceptions.)
+
+Various flags, attributes, and metadata can alter the behavior of these
+operations and thus make them not bit-identical across machines and optimization
+levels any more: most notably, the :ref:`fast-math flags <fastmath>` as well as
+the :ref:`strictfp <strictfp>` and :ref:`denormal-fp-math <denormal_fp_math>`
+attributes and :ref:`!fpmath metadata <fpmath-metadata>`. See their
+corresponding documentation for details.
+
 .. _fastmath:
 
 Fast-Math Flags
@@ -3975,7 +4021,7 @@ Floating-Point Types
      - Description
 
    * - ``half``
-     - 16-bit floating-point value
+     - 16-bit floating-point value (IEEE-754 binary16)
 
    * - ``bfloat``
      - 16-bit "brain" floating-point value (7-bit significand).  Provides the
@@ -3984,13 +4030,13 @@ Floating-Point Types
        extensions and Arm's ARMv8.6-A extensions, among others.
 
    * - ``float``
-     - 32-bit floating-point value
+     - 32-bit floating-point value (IEEE-754 binary32)
 
    * - ``double``
-     - 64-bit floating-point value
+     - 64-bit floating-point value (IEEE-754 binary64)
 
    * - ``fp128``
-     - 128-bit floating-point value (113-bit significand)
+     - 128-bit floating-point value (IEEE-754 binary128)
 
    * - ``x86_fp80``
      -  80-bit floating-point value (X87)
@@ -3998,10 +4044,6 @@ Floating-Point Types
    * - ``ppc_fp128``
      - 128-bit floating-point value (two 64-bits)
 
-The binary format of half, float, double, and fp128 correspond to the
-IEEE-754-2008 specifications for binary16, binary32, binary64, and binary128
-respectively.
-
 X86_amx Type
 """"""""""""
 
@@ -6957,6 +6999,8 @@ For example,
     %2 = load float, ptr %c, align 4, !alias.scope !6
     store float %0, ptr %arrayidx.i, align 4, !noalias !7
 
+.. _fpmath-metadata:
+
 '``fpmath``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -12287,6 +12331,7 @@ Syntax:
 ::
 
       <result> = icmp <cond> <ty> <op1>, <op2>   ; yields i1 or <N x i1>:result
+      <result> = icmp samesign <cond> <ty> <op1>, <op2>   ; yields i1 or <N x i1>:result
 
 Overview:
 """""""""
@@ -12356,6 +12401,9 @@ If the operands are integer vectors, then they are compared element by
 element. The result is an ``i1`` vector with the same number of elements
 as the values being compared. Otherwise, the result is an ``i1``.
 
+If the ``samesign`` keyword is present and the operands are not of the
+same sign then the result is a :ref:`poison value <poisonvalues>`.
+
 Example:
 """"""""
 
diff --git a/llvm/examples/BrainF/BrainF.cpp b/llvm/examples/BrainF/BrainF.cpp
index ac01961735e1376b315ca4f03dacd588887b881e..e62cc7bd591a3f7a88df911c5a4b58f3b6b70513 100644
--- a/llvm/examples/BrainF/BrainF.cpp
+++ b/llvm/examples/BrainF/BrainF.cpp
@@ -67,8 +67,8 @@ void BrainF::header(LLVMContext& C) {
 
   //declare void @llvm.memset.p0i8.i32(i8 *, i8, i32, i1)
   Type *Tys[] = {PointerType::getUnqual(C), Type::getInt32Ty(C)};
-  Function *memset_func = Intrinsic::getDeclaration(module, Intrinsic::memset,
-                                                    Tys);
+  Function *memset_func =
+      Intrinsic::getOrInsertDeclaration(module, Intrinsic::memset, Tys);
 
   //declare i32 @getchar()
   getchar_func =
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
index fd0e081ff2b4bd067764a2040ee1be4c518a9247..fee2d26e5d9254f889cb983346837be11d6ab01a 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/ExecutionEngine/Orc/IRPartitionLayer.h"
 #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
@@ -48,6 +49,7 @@ private:
   RTDyldObjectLinkingLayer ObjectLayer;
   IRCompileLayer CompileLayer;
   IRTransformLayer OptimizeLayer;
+  IRPartitionLayer IPLayer;
   CompileOnDemandLayer CODLayer;
 
   JITDylib &MainJD;
@@ -68,8 +70,8 @@ public:
         CompileLayer(*this->ES, ObjectLayer,
                      std::make_unique<ConcurrentIRCompiler>(std::move(JTMB))),
         OptimizeLayer(*this->ES, CompileLayer, optimizeModule),
-        CODLayer(*this->ES, OptimizeLayer,
-                 this->EPCIU->getLazyCallThroughManager(),
+        IPLayer(*this->ES, OptimizeLayer),
+        CODLayer(*this->ES, IPLayer, this->EPCIU->getLazyCallThroughManager(),
                  [this] { return this->EPCIU->createIndirectStubsManager(); }),
         MainJD(this->ES->createBareJITDylib("<main>")) {
     MainJD.addGenerator(
diff --git a/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp b/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
index 1659e5c5c8b40bf0b8b41ea4696f5d340f563e50..24ae953eca7a537f92401e2770948c236dfa5aec 100644
--- a/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
+++ b/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
@@ -3,6 +3,7 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/ExecutionEngine/Orc/IRPartitionLayer.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
@@ -109,13 +110,14 @@ private:
       IndirectStubsManagerBuilderFunction ISMBuilder,
       std::unique_ptr<DynamicLibrarySearchGenerator> ProcessSymbolsGenerator)
       : ES(std::move(ES)), DL(std::move(DL)),
-        MainJD(this->ES->createBareJITDylib("<main>")), LCTMgr(std::move(LCTMgr)),
+        MainJD(this->ES->createBareJITDylib("<main>")),
+        LCTMgr(std::move(LCTMgr)),
         CompileLayer(*this->ES, ObjLayer,
                      std::make_unique<ConcurrentIRCompiler>(std::move(JTMB))),
         S(Imps, *this->ES),
         SpeculateLayer(*this->ES, CompileLayer, S, Mangle, BlockFreqQuery()),
-        CODLayer(*this->ES, SpeculateLayer, *this->LCTMgr,
-                 std::move(ISMBuilder)) {
+        IPLayer(*this->ES, SpeculateLayer),
+        CODLayer(*this->ES, IPLayer, *this->LCTMgr, std::move(ISMBuilder)) {
     MainJD.addGenerator(std::move(ProcessSymbolsGenerator));
     this->CODLayer.setImplMap(&Imps);
     ExitOnErr(S.addSpeculationRuntime(MainJD, Mangle));
@@ -141,6 +143,7 @@ private:
   Speculator S;
   RTDyldObjectLinkingLayer ObjLayer{*ES, createMemMgr};
   IRSpeculationLayer SpeculateLayer;
+  IRPartitionLayer IPLayer;
   CompileOnDemandLayer CODLayer;
 };
 
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 28dc270ca368d2171a46afce74be2ec52952fdfa..55649d89a6b8f41694f0a6df93081c706f1be763 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -2807,10 +2807,10 @@ unsigned LLVMLookupIntrinsicID(const char *Name, size_t NameLen);
 unsigned LLVMGetIntrinsicID(LLVMValueRef Fn);
 
 /**
- * Create or insert the declaration of an intrinsic.  For overloaded intrinsics,
+ * Get or insert the declaration of an intrinsic.  For overloaded intrinsics,
  * parameter types must be provided to uniquely identify an overload.
  *
- * @see llvm::Intrinsic::getDeclaration()
+ * @see llvm::Intrinsic::getOrInsertDeclaration()
  */
 LLVMValueRef LLVMGetIntrinsicDeclaration(LLVMModuleRef Mod,
                                          unsigned ID,
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index 6f1e24e5da33a96755e2510ed6be161e4b56670c..97547fb577e0ec18350933cc353fc154010507bd 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -1046,7 +1046,10 @@ public:
   ///
   /// \param Negative True iff the number should be negative.
   static APFloat getOne(const fltSemantics &Sem, bool Negative = false) {
-    return APFloat(Sem, Negative ? -1 : 1);
+    APFloat Val(Sem, 1U);
+    if (Negative)
+      Val.changeSign();
+    return Val;
   }
 
   /// Factory for Positive and Negative Infinity.
diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index a42dae8887392d387d3fd9f953f9959694800e95..63a138527b32e1277d4993444e3c63a220326f26 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -112,14 +112,21 @@ public:
         bool implicitTrunc = true)
       : BitWidth(numBits) {
     if (!implicitTrunc) {
-      if (BitWidth == 0) {
-        assert(val == 0 && "Value must be zero for 0-bit APInt");
-      } else if (isSigned) {
-        assert(llvm::isIntN(BitWidth, val) &&
-               "Value is not an N-bit signed value");
+      if (isSigned) {
+        if (BitWidth == 0) {
+          assert((val == 0 || val == uint64_t(-1)) &&
+                 "Value must be 0 or -1 for signed 0-bit APInt");
+        } else {
+          assert(llvm::isIntN(BitWidth, val) &&
+                 "Value is not an N-bit signed value");
+        }
       } else {
-        assert(llvm::isUIntN(BitWidth, val) &&
-               "Value is not an N-bit unsigned value");
+        if (BitWidth == 0) {
+          assert(val == 0 && "Value must be zero for unsigned 0-bit APInt");
+        } else {
+          assert(llvm::isUIntN(BitWidth, val) &&
+                 "Value is not an N-bit unsigned value");
+        }
       }
     }
     if (isSingleWord()) {
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 871a6e97861e297ff0f2a47638fb9370c40d1aca..700c3b0f18b8d58c6c0c539b05ccb6df4971aee0 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -371,7 +371,8 @@ getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM,
 /// using that cost, so we won't do so from this function. Return std::nullopt
 /// if inlining should not be attempted.
 std::optional<InlineCost>
-shouldInline(CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost,
+shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI,
+             function_ref<InlineCost(CallBase &CB)> GetInlineCost,
              OptimizationRemarkEmitter &ORE, bool EnableDeferral = true);
 
 /// Emit ORE message.
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index c5978ce54fc18b256d4942e1b1237d1211702e25..22e0b1bc901a43aad7b2c08b73c715330e7923fc 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -47,7 +47,6 @@ const int OptAggressiveThreshold = 250;
 int getInstrCost();
 const int IndirectCallThreshold = 100;
 const int LoopPenalty = 25;
-const int LastCallToStaticBonus = 15000;
 const int ColdccPenalty = 2000;
 /// Do not inline functions which allocate this many bytes on the stack
 /// when the caller is recursive.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 179a2c38d9d3c2b6753b1ac29d07595562acac7a..4b8cb3a39a86db2d8ad596302a9330d4ed711527 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -2168,6 +2168,9 @@ private:
   bool isGuaranteedToTransferExecutionTo(const Instruction *A,
                                          const Instruction *B);
 
+  /// Returns true if \p Op is guaranteed to not be poison.
+  static bool isGuaranteedNotToBePoison(const SCEV *Op);
+
   /// Return true if the SCEV corresponding to \p I is never poison.  Proving
   /// this is more complex than proving that just \p I is never poison, since
   /// SCEV commons expressions across control flow, and you can have cases
@@ -2376,6 +2379,10 @@ public:
   /// Get the (predicated) symbolic max backedge count for the analyzed loop.
   const SCEV *getSymbolicMaxBackedgeTakenCount();
 
+  /// Returns the upper bound of the loop trip count as a normal unsigned
+  /// value, or 0 if the trip count is unknown.
+  unsigned getSmallConstantMaxTripCount();
+
   /// Adds a new predicate.
   void addPredicate(const SCEVPredicate &Pred);
 
@@ -2447,6 +2454,9 @@ private:
 
   /// The symbolic backedge taken count.
   const SCEV *SymbolicMaxBackedgeCount = nullptr;
+
+  /// The constant max trip count for the loop.
+  std::optional<unsigned> SmallConstantMaxTripCount;
 };
 
 template <> struct DenseMapInfo<ScalarEvolution::FoldID> {
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index fd884f2a2f55b025f98b0e615d5ce271016597e8..6eb1aca1cf76adb00650b8571c8a6c1592ed93d1 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -511,8 +511,9 @@ public:
 /// This node is the base class for sequential/in-order min/max selections.
 /// Note that their fundamental difference from SCEVMinMaxExpr's is that they
 /// are early-returning upon reaching saturation point.
-/// I.e. given `0 umin_seq poison`, the result will be `0`,
-/// while the result of `0 umin poison` is `poison`.
+/// I.e. given `0 umin_seq poison`, the result will be `0`, while the result of
+/// `0 umin poison` is `poison`. When returning early, later expressions are not
+/// executed, so `0 umin_seq (%x u/ 0)` does not result in undefined behavior.
 class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
   friend class ScalarEvolution;
 
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 3c17e2da9ac98999f408cf9bc1695e0fec8a6f8e..93cc1f1539fb92a9ce1f7a830ddd90d3db49ccb5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -353,6 +353,9 @@ public:
   unsigned getInliningCostBenefitAnalysisSavingsMultiplier() const;
   unsigned getInliningCostBenefitAnalysisProfitableMultiplier() const;
 
+  /// \returns The bonus of inlining the last call to a static function.
+  int getInliningLastCallToStaticBonus() const;
+
   /// \returns A value to be added to the inlining threshold.
   unsigned adjustInliningThreshold(const CallBase *CB) const;
 
@@ -801,6 +804,12 @@ public:
   /// Return true if the target supports strided load.
   bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const;
 
+  /// Return true is the target supports interleaved access for the given vector
+  /// type \p VTy, interleave factor \p Factor, alignment \p Alignment and
+  /// address space \p AddrSpace.
+  bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+                                    Align Alignment, unsigned AddrSpace) const;
+
   // Return true if the target supports masked vector histograms.
   bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const;
 
@@ -1856,6 +1865,7 @@ public:
   virtual unsigned getInliningCostBenefitAnalysisSavingsMultiplier() const = 0;
   virtual unsigned
   getInliningCostBenefitAnalysisProfitableMultiplier() const = 0;
+  virtual int getInliningLastCallToStaticBonus() const = 0;
   virtual unsigned adjustInliningThreshold(const CallBase *CB) = 0;
   virtual int getInlinerVectorBonusPercent() const = 0;
   virtual unsigned getCallerAllocaCost(const CallBase *CB,
@@ -1946,6 +1956,10 @@ public:
   virtual bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalStridedLoadStore(Type *DataType, Align Alignment) = 0;
+  virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+                                            Align Alignment,
+                                            unsigned AddrSpace) = 0;
+
   virtual bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) = 0;
   virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
                                unsigned Opcode1,
@@ -2275,6 +2289,9 @@ public:
   unsigned getInliningCostBenefitAnalysisProfitableMultiplier() const override {
     return Impl.getInliningCostBenefitAnalysisProfitableMultiplier();
   }
+  int getInliningLastCallToStaticBonus() const override {
+    return Impl.getInliningLastCallToStaticBonus();
+  }
   int getInlinerVectorBonusPercent() const override {
     return Impl.getInlinerVectorBonusPercent();
   }
@@ -2474,6 +2491,11 @@ public:
   bool isLegalStridedLoadStore(Type *DataType, Align Alignment) override {
     return Impl.isLegalStridedLoadStore(DataType, Alignment);
   }
+  bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+                                    Align Alignment,
+                                    unsigned AddrSpace) override {
+    return Impl.isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace);
+  }
   bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) override {
     return Impl.isLegalMaskedVectorHistogram(AddrType, DataType);
   }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 1bca5685de61cd719f89fc9f143c215ee3b9fd2c..139a0c0aa9e39b53e0a4810a6e0b870194de3120 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -75,6 +75,11 @@ public:
   unsigned getInliningCostBenefitAnalysisProfitableMultiplier() const {
     return 8;
   }
+  int getInliningLastCallToStaticBonus() const {
+    // This is the value of InlineConstants::LastCallToStaticBonus before it was
+    // removed along with the introduction of this function.
+    return 15000;
+  }
   unsigned adjustInliningThreshold(const CallBase *CB) const { return 0; }
   unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const {
     return 0;
@@ -328,6 +333,11 @@ public:
     return false;
   }
 
+  bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+                                    Align Alignment, unsigned AddrSpace) {
+    return false;
+  }
+
   bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const {
     return false;
   }
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 5749a34d511dd7acc06138fafd85077b1c86fa07..0c6ebf87b2923214681325770bd1ec9bd2d191cd 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -791,6 +791,9 @@ bool onlyUsedByLifetimeMarkers(const Value *V);
 /// droppable instructions.
 bool onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V);
 
+/// Return true if the instruction doesn't potentially cross vector lanes.
+bool isNotCrossLaneOperation(const Instruction *I);
+
 /// Return true if the instruction does not have any effects besides
 /// calculating the result and does not have undefined behavior.
 ///
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index 19029842a572a4a7c1cf609744a7be2c07242901..178c911120b4ce933ef17329d74fe95e2aa07fb1 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -114,6 +114,7 @@ enum Kind {
   kw_disjoint,
   kw_inbounds,
   kw_nneg,
+  kw_samesign,
   kw_inrange,
   kw_addrspace,
   kw_section,
diff --git a/llvm/include/llvm/BinaryFormat/COFF.h b/llvm/include/llvm/BinaryFormat/COFF.h
index 3fc543f73c49dbddc9130607dc22272b9f4ae3b7..bbc5264d17872abf66095bda7a7c1e96e88f0484 100644
--- a/llvm/include/llvm/BinaryFormat/COFF.h
+++ b/llvm/include/llvm/BinaryFormat/COFF.h
@@ -417,6 +417,24 @@ enum RelocationTypesARM64 : unsigned {
   IMAGE_REL_ARM64_REL32 = 0x0011,
 };
 
+enum RelocationTypesMips : unsigned {
+  IMAGE_REL_MIPS_ABSOLUTE = 0x0000,
+  IMAGE_REL_MIPS_REFHALF = 0x0001,
+  IMAGE_REL_MIPS_REFWORD = 0x0002,
+  IMAGE_REL_MIPS_JMPADDR = 0x0003,
+  IMAGE_REL_MIPS_REFHI = 0x0004,
+  IMAGE_REL_MIPS_REFLO = 0x0005,
+  IMAGE_REL_MIPS_GPREL = 0x0006,
+  IMAGE_REL_MIPS_LITERAL = 0x0007,
+  IMAGE_REL_MIPS_SECTION = 0x000A,
+  IMAGE_REL_MIPS_SECREL = 0x000B,
+  IMAGE_REL_MIPS_SECRELLO = 0x000C,
+  IMAGE_REL_MIPS_SECRELHI = 0x000D,
+  IMAGE_REL_MIPS_JMPADDR16 = 0x0010,
+  IMAGE_REL_MIPS_REFWORDNB = 0x0022,
+  IMAGE_REL_MIPS_PAIR = 0x0025,
+};
+
 enum DynamicRelocationType : unsigned {
   IMAGE_DYNAMIC_RELOCATION_GUARD_RF_PROLOGUE = 1,
   IMAGE_DYNAMIC_RELOCATION_GUARD_RF_EPILOGUE = 2,
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index ba2efee9414218fa32cfc5b19e61c61f9786c241..08574cc356e514493b1d091fee8cbb8fe2e46aff 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -540,6 +540,10 @@ enum GetElementPtrOptionalFlags {
   GEP_NUW = 2,
 };
 
+/// ICmpOptionalFlags - Flags for serializing
+/// ICmpOptionalFlags's SubclassOptionalData contents.
+enum ICmpOptionalFlags { ICMP_SAME_SIGN = 0 };
+
 /// Encoded AtomicOrdering values.
 enum AtomicOrderingCodes {
   ORDERING_NOTATOMIC = 0,
@@ -767,6 +771,7 @@ enum AttributeKindCodes {
   ATTR_KIND_SANITIZE_REALTIME_UNSAFE = 97,
   ATTR_KIND_CORO_ELIDE_SAFE = 98,
   ATTR_KIND_NO_EXT = 99,
+  ATTR_KIND_NO_DIVERGENCE_SOURCE = 100,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h
index 7fd9d99ded69952c91db3a99dd8c5992695abc05..4d93213de5e07012247efd24aed91c725bc1bbed 100644
--- a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h
+++ b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h
@@ -37,9 +37,7 @@ class TargetRegisterClass;
 class TargetSubtargetInfo;
 
 struct VRegInfo {
-  enum uint8_t {
-    UNKNOWN, NORMAL, GENERIC, REGBANK
-  } Kind = UNKNOWN;
+  enum : uint8_t { UNKNOWN, NORMAL, GENERIC, REGBANK } Kind = UNKNOWN;
   bool Explicit = false; ///< VReg was explicitly specified in the .mir file.
   union {
     const TargetRegisterClass *RC;
@@ -47,6 +45,7 @@ struct VRegInfo {
   } D;
   Register VReg;
   Register PreferredReg;
+  std::vector<uint8_t> Flags;
 };
 
 using Name2RegClassMap = StringMap<const TargetRegisterClass *>;
@@ -150,6 +149,8 @@ public:
   /// Return null if the name isn't a register bank.
   const RegisterBank *getRegBank(StringRef Name);
 
+  bool getVRegFlagValue(StringRef FlagName, uint8_t &FlagValue) const;
+
   PerTargetMIParsingState(const TargetSubtargetInfo &STI)
     : Subtarget(STI) {
     initNames2RegClasses();
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index c5bf6971df4535adc33f67de82486ebb4d89e241..09a6ca936fe1f4a1ced988eeb32f01fad8d7abd1 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -191,6 +191,7 @@ struct VirtualRegisterDefinition {
   UnsignedValue ID;
   StringValue Class;
   StringValue PreferredRegister;
+  std::vector<FlowStringValue> RegisterFlags;
 
   // TODO: Serialize the target specific register hints.
 
@@ -206,6 +207,8 @@ template <> struct MappingTraits<VirtualRegisterDefinition> {
     YamlIO.mapRequired("class", Reg.Class);
     YamlIO.mapOptional("preferred-register", Reg.PreferredRegister,
                        StringValue()); // Don't print out when it's empty.
+    YamlIO.mapOptional("flags", Reg.RegisterFlags,
+                       std::vector<FlowStringValue>());
   }
 
   static const bool flow = true;
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index ddb2012cd2bffcc2929184264e7c6f98ea4a934f..99421bdf769ffa2b43282314d2877457a11875c5 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -285,7 +285,7 @@ namespace llvm {
 
   /// StackSlotColoring - This pass performs stack coloring and merging.
   /// It merges disjoint allocas to reduce the stack size.
-  extern char &StackColoringID;
+  extern char &StackColoringLegacyID;
 
   /// StackFramePrinter - This pass prints the stack frame layout and variable
   /// mappings.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 6067b3b29ea1815ce91b3eeb2529d585a7390ede..639e9311977502d6618a8607c119d961ee69edc3 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1678,6 +1678,7 @@ class ConstantSDNode : public SDNode {
       : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
                VTs),
         Value(val) {
+    assert(!isa<VectorType>(val->getType()) && "Unexpected vector type!");
     ConstantSDNodeBits.IsOpaque = isOpaque;
   }
 
@@ -1730,7 +1731,9 @@ class ConstantFPSDNode : public SDNode {
   ConstantFPSDNode(bool isTarget, const ConstantFP *val, SDVTList VTs)
       : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
                DebugLoc(), VTs),
-        Value(val) {}
+        Value(val) {
+    assert(!isa<VectorType>(val->getType()) && "Unexpected vector type!");
+  }
 
 public:
   const APFloat& getValueAPF() const { return Value->getValueAPF(); }
diff --git a/llvm/include/llvm/CodeGen/StackColoring.h b/llvm/include/llvm/CodeGen/StackColoring.h
new file mode 100644
index 0000000000000000000000000000000000000000..65f8916f42a91c5d76a39559ff739e084d4f0e30
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/StackColoring.h
@@ -0,0 +1,24 @@
+//===- llvm/CodeGen/StackColoring.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_STACKCOLORINGPASS_H
+#define LLVM_CODEGEN_STACKCOLORINGPASS_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class StackColoringPass : public PassInfoMixin<StackColoringPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_STACKCOLORINGPASS_H
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 9ea0fba1144b1388093ca4d4fcc743f00dd1119e..292fa3c94969be93fc32f7c62d2be7d7e96c9c1a 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -1213,6 +1213,15 @@ public:
   virtual bool isNonallocatableRegisterCalleeSave(MCRegister Reg) const {
     return false;
   }
+
+  virtual std::optional<uint8_t> getVRegFlagValue(StringRef Name) const {
+    return {};
+  }
+
+  virtual SmallVector<StringLiteral>
+  getVRegFlagsOfReg(Register Reg, const MachineFunction &MF) const {
+    return {};
+  }
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
index 41f2882c576e136a7ff5a3e5f799d74e0b2568e5..d492dfe29fba205538d01ff32ff16ba838439dde 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
@@ -53,37 +53,15 @@ namespace llvm {
 namespace orc {
 
 class CompileOnDemandLayer : public IRLayer {
-  friend class PartitioningIRMaterializationUnit;
-
 public:
   /// Builder for IndirectStubsManagers.
   using IndirectStubsManagerBuilder =
       std::function<std::unique_ptr<IndirectStubsManager>()>;
 
-  using GlobalValueSet = std::set<const GlobalValue *>;
-
-  /// Partitioning function.
-  using PartitionFunction =
-      std::function<std::optional<GlobalValueSet>(GlobalValueSet Requested)>;
-
-  /// Off-the-shelf partitioning which compiles all requested symbols (usually
-  /// a single function at a time).
-  static std::optional<GlobalValueSet>
-  compileRequested(GlobalValueSet Requested);
-
-  /// Off-the-shelf partitioning which compiles whole modules whenever any
-  /// symbol in them is requested.
-  static std::optional<GlobalValueSet>
-  compileWholeModule(GlobalValueSet Requested);
-
   /// Construct a CompileOnDemandLayer.
   CompileOnDemandLayer(ExecutionSession &ES, IRLayer &BaseLayer,
-                        LazyCallThroughManager &LCTMgr,
-                        IndirectStubsManagerBuilder BuildIndirectStubsManager);
-
-  /// Sets the partition function.
-  void setPartitionFunction(PartitionFunction Partition);
-
+                       LazyCallThroughManager &LCTMgr,
+                       IndirectStubsManagerBuilder BuildIndirectStubsManager);
   /// Sets the ImplSymbolMap
   void setImplMap(ImplSymbolMap *Imp);
 
@@ -110,22 +88,12 @@ private:
 
   PerDylibResources &getPerDylibResources(JITDylib &TargetD);
 
-  void cleanUpModule(Module &M);
-
-  void expandPartition(GlobalValueSet &Partition);
-
-  void emitPartition(std::unique_ptr<MaterializationResponsibility> R,
-                     ThreadSafeModule TSM,
-                     IRMaterializationUnit::SymbolNameToDefinitionMap Defs);
-
   mutable std::mutex CODLayerMutex;
 
   IRLayer &BaseLayer;
   LazyCallThroughManager &LCTMgr;
   IndirectStubsManagerBuilder BuildIndirectStubsManager;
   PerDylibResourcesMap DylibResources;
-  PartitionFunction Partition = compileRequested;
-  SymbolLinkagePromoter PromoteSymbols;
   ImplSymbolMap *AliaseeImpls = nullptr;
 };
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IRPartitionLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/IRPartitionLayer.h
new file mode 100644
index 0000000000000000000000000000000000000000..a2981fea61d3ead1d2d93f0eb10a3360464f74dc
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/IRPartitionLayer.h
@@ -0,0 +1,85 @@
+//===- IRPartitionLayer.h - Partition IR module on lookup -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// JIT layer for breaking up modules into smaller submodules that only contains
+// looked up symbols.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_IRPARTITIONLAYER_H
+#define LLVM_EXECUTIONENGINE_ORC_IRPARTITIONLAYER_H
+
+#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
+#include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalAlias.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Mangler.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+
+namespace llvm {
+namespace orc {
+
+/// A layer that breaks up IR modules into smaller submodules that only contains
+/// looked up symbols.
+class IRPartitionLayer : public IRLayer {
+  friend class PartitioningIRMaterializationUnit;
+
+public:
+  using GlobalValueSet = std::set<const GlobalValue *>;
+
+  /// Partitioning function.
+  using PartitionFunction =
+      std::function<std::optional<GlobalValueSet>(GlobalValueSet Requested)>;
+
+  /// Construct a IRPartitionLayer.
+  IRPartitionLayer(ExecutionSession &ES, IRLayer &BaseLayer);
+
+  /// Off-the-shelf partitioning which compiles all requested symbols (usually
+  /// a single function at a time).
+  static std::optional<GlobalValueSet>
+  compileRequested(GlobalValueSet Requested);
+
+  /// Off-the-shelf partitioning which compiles whole modules whenever any
+  /// symbol in them is requested.
+  static std::optional<GlobalValueSet>
+  compileWholeModule(GlobalValueSet Requested);
+
+  /// Sets the partition function.
+  void setPartitionFunction(PartitionFunction Partition);
+
+  /// Emits the given module. This should not be called by clients: it will be
+  /// called by the JIT when a definition added via the add method is requested.
+  void emit(std::unique_ptr<MaterializationResponsibility> R,
+            ThreadSafeModule TSM) override;
+
+private:
+  void cleanUpModule(Module &M);
+
+  void expandPartition(GlobalValueSet &Partition);
+
+  void emitPartition(std::unique_ptr<MaterializationResponsibility> R,
+                     ThreadSafeModule TSM,
+                     IRMaterializationUnit::SymbolNameToDefinitionMap Defs);
+
+  IRLayer &BaseLayer;
+  PartitionFunction Partition = compileRequested;
+  SymbolLinkagePromoter PromoteSymbols;
+};
+
+} // namespace orc
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
new file mode 100644
index 0000000000000000000000000000000000000000..52f284c89bdade56b0c6d605011af5be5d8b207a
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
@@ -0,0 +1,107 @@
+//===- JITLinkRedirectableSymbolManager.h - JITLink redirection -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Redirectable Symbol Manager implementation using JITLink
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKREDIRECABLEMANAGER_H
+#define LLVM_EXECUTIONENGINE_ORC_JITLINKREDIRECABLEMANAGER_H
+
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
+#include "llvm/Support/StringSaver.h"
+
+namespace llvm {
+namespace orc {
+
+class JITLinkRedirectableSymbolManager : public RedirectableSymbolManager,
+                                         public ResourceManager {
+public:
+  /// Create redirection manager that uses JITLink based implementaion.
+  static Expected<std::unique_ptr<RedirectableSymbolManager>>
+  Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &JD) {
+    Error Err = Error::success();
+    auto RM = std::unique_ptr<RedirectableSymbolManager>(
+        new JITLinkRedirectableSymbolManager(ObjLinkingLayer, JD, Err));
+    if (Err)
+      return Err;
+    return std::move(RM);
+  }
+
+  void emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> R,
+                               const SymbolAddrMap &InitialDests) override;
+
+  Error redirect(JITDylib &TargetJD, const SymbolAddrMap &NewDests) override;
+
+  Error handleRemoveResources(JITDylib &TargetJD, ResourceKey K) override;
+
+  void handleTransferResources(JITDylib &TargetJD, ResourceKey DstK,
+                               ResourceKey SrcK) override;
+
+private:
+  using StubHandle = unsigned;
+  constexpr static unsigned StubBlockSize = 256;
+  constexpr static StringRef JumpStubPrefix = "$__IND_JUMP_STUBS";
+  constexpr static StringRef StubPtrPrefix = "$IND_JUMP_PTR_";
+  constexpr static StringRef JumpStubTableName = "$IND_JUMP_";
+  constexpr static StringRef StubPtrTableName = "$__IND_JUMP_PTRS";
+
+  JITLinkRedirectableSymbolManager(ObjectLinkingLayer &ObjLinkingLayer,
+                                   JITDylib &JD, Error &Err)
+      : ObjLinkingLayer(ObjLinkingLayer), JD(JD),
+        AnonymousPtrCreator(jitlink::getAnonymousPointerCreator(
+            ObjLinkingLayer.getExecutionSession().getTargetTriple())),
+        PtrJumpStubCreator(jitlink::getPointerJumpStubCreator(
+            ObjLinkingLayer.getExecutionSession().getTargetTriple())) {
+    if (!AnonymousPtrCreator || !PtrJumpStubCreator)
+      Err = make_error<StringError>("Architecture not supported",
+                                    inconvertibleErrorCode());
+    if (Err)
+      return;
+    ObjLinkingLayer.getExecutionSession().registerResourceManager(*this);
+  }
+
+  ~JITLinkRedirectableSymbolManager() {
+    ObjLinkingLayer.getExecutionSession().deregisterResourceManager(*this);
+  }
+
+  StringRef JumpStubSymbolName(unsigned I) {
+    return *ObjLinkingLayer.getExecutionSession().intern(
+        (JumpStubPrefix + Twine(I)).str());
+  }
+
+  StringRef StubPtrSymbolName(unsigned I) {
+    return *ObjLinkingLayer.getExecutionSession().intern(
+        (StubPtrPrefix + Twine(I)).str());
+  }
+
+  unsigned GetNumAvailableStubs() const { return AvailableStubs.size(); }
+
+  Error redirectInner(JITDylib &TargetJD, const SymbolAddrMap &NewDests);
+  Error grow(unsigned Need);
+
+  ObjectLinkingLayer &ObjLinkingLayer;
+  JITDylib &JD;
+  jitlink::AnonymousPointerCreator AnonymousPtrCreator;
+  jitlink::PointerJumpStubCreator PtrJumpStubCreator;
+
+  std::vector<StubHandle> AvailableStubs;
+  using SymbolToStubMap = DenseMap<SymbolStringPtr, StubHandle>;
+  DenseMap<JITDylib *, SymbolToStubMap> SymbolToStubs;
+  std::vector<ExecutorSymbolDef> JumpStubs;
+  std::vector<ExecutorSymbolDef> StubPointers;
+  DenseMap<ResourceKey, std::vector<SymbolStringPtr>> TrackedResources;
+
+  std::mutex Mutex;
+};
+
+} // namespace orc
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
index 2660b9f74f405a9db7121978a92ef40c9a69bbeb..a2364b4515f01b41b5fe100eca27df6cb7e74bbe 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
@@ -18,6 +18,7 @@
 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/ExecutionEngine/Orc/IRPartitionLayer.h"
 #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
@@ -271,9 +272,8 @@ class LLLazyJIT : public LLJIT {
 public:
 
   /// Sets the partition function.
-  void
-  setPartitionFunction(CompileOnDemandLayer::PartitionFunction Partition) {
-    CODLayer->setPartitionFunction(std::move(Partition));
+  void setPartitionFunction(IRPartitionLayer::PartitionFunction Partition) {
+    IPLayer->setPartitionFunction(std::move(Partition));
   }
 
   /// Returns a reference to the on-demand layer.
@@ -293,6 +293,7 @@ private:
   LLLazyJIT(LLLazyJITBuilderState &S, Error &Err);
 
   std::unique_ptr<LazyCallThroughManager> LCTMgr;
+  std::unique_ptr<IRPartitionLayer> IPLayer;
   std::unique_ptr<CompileOnDemandLayer> CODLayer;
 };
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
new file mode 100644
index 0000000000000000000000000000000000000000..4adc3efad557307c4a15493dcd7330c7dfcae320
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ReOptimizeLayer.h
@@ -0,0 +1,181 @@
+//===- ReOptimizeLayer.h - Re-optimization layer interface ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Re-optimization layer interface.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_EXECUTIONENGINE_ORC_REOPTIMIZELAYER_H
+#define LLVM_EXECUTIONENGINE_ORC_REOPTIMIZELAYER_H
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/Mangling.h"
+#include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
+#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+
+namespace llvm {
+namespace orc {
+
+class ReOptimizeLayer : public IRLayer, public ResourceManager {
+public:
+  using ReOptMaterializationUnitID = uint64_t;
+
+  /// AddProfilerFunc will be called when ReOptimizeLayer emits the first
+  /// version of a materialization unit in order to inject profiling code and
+  /// reoptimization request code.
+  using AddProfilerFunc = unique_function<Error(
+      ReOptimizeLayer &Parent, ReOptMaterializationUnitID MUID,
+      unsigned CurVersion, ThreadSafeModule &TSM)>;
+
+  /// ReOptimizeFunc will be called when ReOptimizeLayer reoptimization of a
+  /// materialization unit was requested in order to reoptimize the IR module
+  /// based on profile data. OldRT is the ResourceTracker that tracks the old
+  /// function definitions. The OldRT must be kept alive until it can be
+  /// guaranteed that every invocation of the old function definitions has been
+  /// terminated.
+  using ReOptimizeFunc = unique_function<Error(
+      ReOptimizeLayer &Parent, ReOptMaterializationUnitID MUID,
+      unsigned CurVersion, ResourceTrackerSP OldRT, ThreadSafeModule &TSM)>;
+
+  ReOptimizeLayer(ExecutionSession &ES, DataLayout &DL, IRLayer &BaseLayer,
+                  RedirectableSymbolManager &RM)
+      : IRLayer(ES, BaseLayer.getManglingOptions()), ES(ES), Mangle(ES, DL),
+        BaseLayer(BaseLayer), RSManager(RM), ReOptFunc(identity),
+        ProfilerFunc(reoptimizeIfCallFrequent) {}
+
+  void setReoptimizeFunc(ReOptimizeFunc ReOptFunc) {
+    this->ReOptFunc = std::move(ReOptFunc);
+  }
+
+  void setAddProfilerFunc(AddProfilerFunc ProfilerFunc) {
+    this->ProfilerFunc = std::move(ProfilerFunc);
+  }
+
+  /// Registers reoptimize runtime dispatch handlers to given PlatformJD. The
+  /// reoptimization request will not be handled if dispatch handler is not
+  /// registered by using this function.
+  Error reigsterRuntimeFunctions(JITDylib &PlatformJD);
+
+  /// Emits the given module. This should not be called by clients: it will be
+  /// called by the JIT when a definition added via the add method is requested.
+  void emit(std::unique_ptr<MaterializationResponsibility> R,
+            ThreadSafeModule TSM) override;
+
+  static const uint64_t CallCountThreshold = 10;
+
+  /// Basic AddProfilerFunc that reoptimizes the function when the call count
+  /// exceeds CallCountThreshold.
+  static Error reoptimizeIfCallFrequent(ReOptimizeLayer &Parent,
+                                        ReOptMaterializationUnitID MUID,
+                                        unsigned CurVersion,
+                                        ThreadSafeModule &TSM);
+
+  static Error identity(ReOptimizeLayer &Parent,
+                        ReOptMaterializationUnitID MUID, unsigned CurVersion,
+                        ResourceTrackerSP OldRT, ThreadSafeModule &TSM) {
+    return Error::success();
+  }
+
+  // Create IR reoptimize request fucntion call.
+  static void createReoptimizeCall(Module &M, Instruction &IP,
+                                   GlobalVariable *ArgBuffer);
+
+  Error handleRemoveResources(JITDylib &JD, ResourceKey K) override;
+  void handleTransferResources(JITDylib &JD, ResourceKey DstK,
+                               ResourceKey SrcK) override;
+
+private:
+  class ReOptMaterializationUnitState {
+  public:
+    ReOptMaterializationUnitState() = default;
+    ReOptMaterializationUnitState(ReOptMaterializationUnitID ID,
+                                  ThreadSafeModule TSM)
+        : ID(ID), TSM(std::move(TSM)) {}
+    ReOptMaterializationUnitState(ReOptMaterializationUnitState &&Other)
+        : ID(Other.ID), TSM(std::move(Other.TSM)), RT(std::move(Other.RT)),
+          Reoptimizing(std::move(Other.Reoptimizing)),
+          CurVersion(Other.CurVersion) {}
+
+    ReOptMaterializationUnitID getID() { return ID; }
+
+    const ThreadSafeModule &getThreadSafeModule() { return TSM; }
+
+    ResourceTrackerSP getResourceTracker() {
+      std::unique_lock<std::mutex> Lock(Mutex);
+      return RT;
+    }
+
+    void setResourceTracker(ResourceTrackerSP RT) {
+      std::unique_lock<std::mutex> Lock(Mutex);
+      this->RT = RT;
+    }
+
+    uint32_t getCurVersion() {
+      std::unique_lock<std::mutex> Lock(Mutex);
+      return CurVersion;
+    }
+
+    bool tryStartReoptimize();
+    void reoptimizeSucceeded();
+    void reoptimizeFailed();
+
+  private:
+    std::mutex Mutex;
+    ReOptMaterializationUnitID ID;
+    ThreadSafeModule TSM;
+    ResourceTrackerSP RT;
+    bool Reoptimizing = false;
+    uint32_t CurVersion = 0;
+  };
+
+  using SPSReoptimizeArgList =
+      shared::SPSArgList<ReOptMaterializationUnitID, uint32_t>;
+  using SendErrorFn = unique_function<void(Error)>;
+
+  Expected<SymbolMap> emitMUImplSymbols(ReOptMaterializationUnitState &MUState,
+                                        uint32_t Version, JITDylib &JD,
+                                        ThreadSafeModule TSM);
+
+  void rt_reoptimize(SendErrorFn SendResult, ReOptMaterializationUnitID MUID,
+                     uint32_t CurVersion);
+
+  static Expected<Constant *>
+  createReoptimizeArgBuffer(Module &M, ReOptMaterializationUnitID MUID,
+                            uint32_t CurVersion);
+
+  ReOptMaterializationUnitState &
+  createMaterializationUnitState(const ThreadSafeModule &TSM);
+
+  void
+  registerMaterializationUnitResource(ResourceKey Key,
+                                      ReOptMaterializationUnitState &State);
+
+  ReOptMaterializationUnitState &
+  getMaterializationUnitState(ReOptMaterializationUnitID MUID);
+
+  ExecutionSession &ES;
+  MangleAndInterner Mangle;
+  IRLayer &BaseLayer;
+  RedirectableSymbolManager &RSManager;
+
+  ReOptimizeFunc ReOptFunc;
+  AddProfilerFunc ProfilerFunc;
+
+  std::mutex Mutex;
+  std::map<ReOptMaterializationUnitID, ReOptMaterializationUnitState> MUStates;
+  DenseMap<ResourceKey, DenseSet<ReOptMaterializationUnitID>> MUResources;
+  ReOptMaterializationUnitID NextID = 1;
+};
+
+} // namespace orc
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
new file mode 100644
index 0000000000000000000000000000000000000000..4004c42d91468436bdf68f11aa4b56357b134153
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
@@ -0,0 +1,103 @@
+//===- RedirectionManager.h - Redirection manager interface -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Redirection manager interface that redirects a call to symbol to another.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_REDIRECTIONMANAGER_H
+#define LLVM_EXECUTIONENGINE_ORC_REDIRECTIONMANAGER_H
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
+
+namespace llvm {
+namespace orc {
+
+/// Base class for performing redirection of call to symbol to another symbol in
+/// runtime.
+class RedirectionManager {
+public:
+  /// Symbol name to symbol definition map.
+  using SymbolAddrMap = DenseMap<SymbolStringPtr, ExecutorSymbolDef>;
+
+  virtual ~RedirectionManager() = default;
+  /// Change the redirection destination of given symbols to new destination
+  /// symbols.
+  virtual Error redirect(JITDylib &JD, const SymbolAddrMap &NewDests) = 0;
+
+  /// Change the redirection destination of given symbol to new destination
+  /// symbol.
+  virtual Error redirect(JITDylib &JD, SymbolStringPtr Symbol,
+                         ExecutorSymbolDef NewDest) {
+    return redirect(JD, {{Symbol, NewDest}});
+  }
+
+private:
+  virtual void anchor();
+};
+
+/// Base class for managing redirectable symbols in which a call
+/// gets redirected to another symbol in runtime.
+class RedirectableSymbolManager : public RedirectionManager {
+public:
+  /// Create redirectable symbols with given symbol names and initial
+  /// desitnation symbol addresses.
+  Error createRedirectableSymbols(ResourceTrackerSP RT,
+                                  const SymbolMap &InitialDests);
+
+  /// Create a single redirectable symbol with given symbol name and initial
+  /// desitnation symbol address.
+  Error createRedirectableSymbol(ResourceTrackerSP RT, SymbolStringPtr Symbol,
+                                 ExecutorSymbolDef InitialDest) {
+    return createRedirectableSymbols(RT, {{Symbol, InitialDest}});
+  }
+
+  /// Emit redirectable symbol
+  virtual void
+  emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> MR,
+                          const SymbolMap &InitialDests) = 0;
+};
+
+/// RedirectableMaterializationUnit materializes redirectable symbol
+/// by invoking RedirectableSymbolManager::emitRedirectableSymbols
+class RedirectableMaterializationUnit : public MaterializationUnit {
+public:
+  RedirectableMaterializationUnit(RedirectableSymbolManager &RM,
+                                  const SymbolMap &InitialDests)
+      : MaterializationUnit(convertToFlags(InitialDests)), RM(RM),
+        InitialDests(InitialDests) {}
+
+  StringRef getName() const override {
+    return "RedirectableSymbolMaterializationUnit";
+  }
+
+  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
+    RM.emitRedirectableSymbols(std::move(R), std::move(InitialDests));
+  }
+
+  void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
+    InitialDests.erase(Name);
+  }
+
+private:
+  static MaterializationUnit::Interface
+  convertToFlags(const SymbolMap &InitialDests) {
+    SymbolFlagsMap Flags;
+    for (auto [K, V] : InitialDests)
+      Flags[K] = V.getFlags();
+    return MaterializationUnit::Interface(Flags, {});
+  }
+
+  RedirectableSymbolManager &RM;
+  SymbolMap InitialDests;
+};
+
+} // namespace orc
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 1b8a6e47b3baf8ff3e1c7c80e2b13834e6463195..8834c3b1f50115f10b51b4d5367b03740882fdff 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -2875,16 +2875,16 @@ public:
   /// instructions for passed in target arguments where neccessary
   /// \param Dependencies A vector of DependData objects that carry
   // dependency information as passed in the depend clause
-  InsertPointTy
-  createTarget(const LocationDescription &Loc, bool IsOffloadEntry,
-               OpenMPIRBuilder::InsertPointTy AllocaIP,
-               OpenMPIRBuilder::InsertPointTy CodeGenIP,
-               TargetRegionEntryInfo &EntryInfo, ArrayRef<int32_t> NumTeams,
-               ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Inputs,
-               GenMapInfoCallbackTy GenMapInfoCB,
-               TargetBodyGenCallbackTy BodyGenCB,
-               TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
-               SmallVector<DependData> Dependencies = {});
+  // \param HasNowait Whether the target construct has a `nowait` clause or not.
+  InsertPointTy createTarget(
+      const LocationDescription &Loc, bool IsOffloadEntry,
+      OpenMPIRBuilder::InsertPointTy AllocaIP,
+      OpenMPIRBuilder::InsertPointTy CodeGenIP,
+      TargetRegionEntryInfo &EntryInfo, ArrayRef<int32_t> NumTeams,
+      ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Inputs,
+      GenMapInfoCallbackTy GenMapInfoCB, TargetBodyGenCallbackTy BodyGenCB,
+      TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
+      SmallVector<DependData> Dependencies = {}, bool HasNowait = false);
 
   /// Returns __kmpc_for_static_init_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned. Will create a distribute call
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index d05a6ca92aaba01102958d79a1cadfa3bafb5b5b..b6d36a5f7ae4fb36db031a84ef460020e328819a 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -183,6 +183,9 @@ def NoCallback : EnumAttr<"nocallback", IntersectAnd, [FnAttr]>;
 /// Function creates no aliases of pointer.
 def NoCapture : EnumAttr<"nocapture", IntersectAnd, [ParamAttr]>;
 
+/// Function is not a source of divergence.
+def NoDivergenceSource : EnumAttr<"nodivergencesource", IntersectAnd, [FnAttr]>;
+
 /// Call cannot be duplicated.
 def NoDuplicate : EnumAttr<"noduplicate", IntersectPreserve, [FnAttr]>;
 
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 46796bf44e2ac7e11070e72b277d043a82e4d10c..63d93ba5f5bc077c2b120c948dedc6f8306fadb2 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -1166,6 +1166,8 @@ class ICmpInst: public CmpInst {
            "Invalid operand types for ICmp instruction");
   }
 
+  enum { SameSign = (1 << 0) };
+
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
@@ -1225,6 +1227,15 @@ public:
   /// Return the unsigned version of the predicate.
   static Predicate getUnsignedPredicate(Predicate pred);
 
+  void setSameSign(bool B = true) {
+    SubclassOptionalData = (SubclassOptionalData & ~SameSign) | (B * SameSign);
+  }
+
+  /// An icmp instruction, which can be marked as "samesign", indicating that
+  /// the two operands have the same sign. This means that we can convert
+  /// "slt" to "ult" and vice versa, which enables more optimizations.
+  bool hasSameSign() const { return SubclassOptionalData & SameSign; }
+
   /// Return true if this predicate is either EQ or NE.  This also
   /// tests for commutativity.
   static bool isEquality(Predicate P) {
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index f7332f0741522cf4cfe61efb7fe068a18be50564..bf5a653dbf71432cca8b39e1dfb18166173a95b3 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -568,9 +568,9 @@ public:
   /// \brief Declares a llvm.vp.* intrinsic in \p M that matches the parameters
   /// \p Params. Additionally, the load and gather intrinsics require
   /// \p ReturnType to be specified.
-  static Function *getDeclarationForParams(Module *M, Intrinsic::ID,
-                                           Type *ReturnType,
-                                           ArrayRef<Value *> Params);
+  static Function *getOrInsertDeclarationForParams(Module *M, Intrinsic::ID,
+                                                   Type *ReturnType,
+                                                   ArrayRef<Value *> Params);
 
   static std::optional<unsigned> getMaskParamPos(Intrinsic::ID IntrinsicID);
   static std::optional<unsigned> getVectorLengthParamPos(
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index b251036247c5c003e261ba47692cdf1930376906..49f4fe4c5c3d7f1a3407c3ff0a32b69a5ec283c0 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -87,15 +87,21 @@ namespace Intrinsic {
   /// Return the attributes for an intrinsic.
   AttributeList getAttributes(LLVMContext &C, ID id);
 
-  /// Create or insert an LLVM Function declaration for an intrinsic, and return
-  /// it.
+  /// Look up the Function declaration of the intrinsic \p id in the Module
+  /// \p M. If it does not exist, add a declaration and return it. Otherwise,
+  /// return the existing declaration.
   ///
-  /// The Tys parameter is for intrinsics with overloaded types (e.g., those
+  /// The \p Tys parameter is for intrinsics with overloaded types (e.g., those
   /// using iAny, fAny, vAny, or iPTRAny).  For a declaration of an overloaded
   /// intrinsic, Tys must provide exactly one type for each overloaded type in
   /// the intrinsic.
-  Function *getDeclaration(Module *M, ID id, ArrayRef<Type *> Tys = {});
+  Function *getOrInsertDeclaration(Module *M, ID id, ArrayRef<Type *> Tys = {});
 
+  LLVM_DEPRECATED("Use getOrInsertDeclaration instead",
+                  "getOrInsertDeclaration")
+  inline Function *getDeclaration(Module *M, ID id, ArrayRef<Type *> Tys = {}) {
+    return getOrInsertDeclaration(M, id, Tys);
+  }
   /// Looks up Name in NameTable via binary search. NameTable must be sorted
   /// and all entries must start with "llvm.".  If NameTable contains an exact
   /// match for Name or a prefix of Name followed by a dot, its index in
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 2738eb77b675ab2360bfb4e94a370c74fb70d15c..715f2cc917e21c9f33d344b21755ccbfe98bab81 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2086,6 +2086,8 @@ def int_amdgcn_fcmp :
             [IntrNoMem, IntrConvergent,
              ImmArg<ArgIndex<2>>, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
 
+// Returns a bitfield(i32 or i64) containing the result of its i1 argument
+// in all active lanes, and zero in all inactive lanes.
 def int_amdgcn_ballot :
   Intrinsic<[llvm_anyint_ty], [llvm_i1_ty],
             [IntrNoMem, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
diff --git a/llvm/include/llvm/IR/MatrixBuilder.h b/llvm/include/llvm/IR/MatrixBuilder.h
index dbf2cfb7c5e966629fe2af61e831397294c8341d..3a04ca87f2b558bcfa89e3aca7e9f01936c8d3a9 100644
--- a/llvm/include/llvm/IR/MatrixBuilder.h
+++ b/llvm/include/llvm/IR/MatrixBuilder.h
@@ -72,7 +72,7 @@ public:
                     B.getInt32(Columns)};
     Type *OverloadedTypes[] = {RetType, Stride->getType()};
 
-    Function *TheFn = Intrinsic::getDeclaration(
+    Function *TheFn = Intrinsic::getOrInsertDeclaration(
         getModule(), Intrinsic::matrix_column_major_load, OverloadedTypes);
 
     CallInst *Call = B.CreateCall(TheFn->getFunctionType(), TheFn, Ops, Name);
@@ -95,7 +95,7 @@ public:
                     B.getInt32(Rows), B.getInt32(Columns)};
     Type *OverloadedTypes[] = {Matrix->getType(), Stride->getType()};
 
-    Function *TheFn = Intrinsic::getDeclaration(
+    Function *TheFn = Intrinsic::getOrInsertDeclaration(
         getModule(), Intrinsic::matrix_column_major_store, OverloadedTypes);
 
     CallInst *Call = B.CreateCall(TheFn->getFunctionType(), TheFn, Ops, Name);
@@ -115,7 +115,7 @@ public:
 
     Type *OverloadedTypes[] = {ReturnType};
     Value *Ops[] = {Matrix, B.getInt32(Rows), B.getInt32(Columns)};
-    Function *TheFn = Intrinsic::getDeclaration(
+    Function *TheFn = Intrinsic::getOrInsertDeclaration(
         getModule(), Intrinsic::matrix_transpose, OverloadedTypes);
 
     return B.CreateCall(TheFn->getFunctionType(), TheFn, Ops, Name);
@@ -136,7 +136,7 @@ public:
                     B.getInt32(RHSColumns)};
     Type *OverloadedTypes[] = {ReturnType, LHSType, RHSType};
 
-    Function *TheFn = Intrinsic::getDeclaration(
+    Function *TheFn = Intrinsic::getOrInsertDeclaration(
         getModule(), Intrinsic::matrix_multiply, OverloadedTypes);
     return B.CreateCall(TheFn->getFunctionType(), TheFn, Ops, Name);
   }
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 288aba16702f5621291a0a958201b59a69ade687..1cfe7c15f97dbca9cedbecf1b56d81e9224edbda 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -200,7 +200,12 @@ struct ValueInfo {
     return getRef()->second.SummaryList;
   }
 
+  // Even if the index is built with GVs available, we may not have one for
+  // summary entries synthesized for profiled indirect call targets.
+  bool hasName() const { return !haveGVs() || getValue(); }
+
   StringRef name() const {
+    assert(!haveGVs() || getRef()->second.U.GV);
     return haveGVs() ? getRef()->second.U.GV->getName()
                      : getRef()->second.U.Name;
   }
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index a96b507e07b383e264cff2ee0811861f4f6bbe78..a8d7d3d3a84216971133376099a568f767711acd 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -288,7 +288,7 @@ void initializeSjLjEHPreparePass(PassRegistry &);
 void initializeSlotIndexesWrapperPassPass(PassRegistry &);
 void initializeSpeculativeExecutionLegacyPassPass(PassRegistry &);
 void initializeSpillPlacementPass(PassRegistry &);
-void initializeStackColoringPass(PassRegistry &);
+void initializeStackColoringLegacyPass(PassRegistry &);
 void initializeStackFrameLayoutAnalysisPassPass(PassRegistry &);
 void initializeStackMapLivenessPass(PassRegistry &);
 void initializeStackProtectorPass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index 13bc4700d87029f77186491965a2af21a17be7bb..0d45df08cb0ca7b9d4faf093b419b6e4b002351c 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -56,6 +56,7 @@
 #include "llvm/CodeGen/SelectOptimize.h"
 #include "llvm/CodeGen/ShadowStackGCLowering.h"
 #include "llvm/CodeGen/SjLjEHPrepare.h"
+#include "llvm/CodeGen/StackColoring.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 6ae80a42792b04a446aa57f9625a2f0196e2227b..2aa5f4fc176aba56ec3df375944fa753b5a3e320 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -152,6 +152,7 @@ MACHINE_FUNCTION_PASS("print<machine-post-dom-tree>",
 MACHINE_FUNCTION_PASS("print<slot-indexes>", SlotIndexesPrinterPass(dbgs()))
 MACHINE_FUNCTION_PASS("require-all-machine-function-properties",
                       RequireAllMachineFunctionPropertiesPass())
+MACHINE_FUNCTION_PASS("stack-coloring", StackColoringPass())
 MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass())
 MACHINE_FUNCTION_PASS("two-address-instruction", TwoAddressInstructionPass())
 MACHINE_FUNCTION_PASS("verify", MachineVerifierPass())
@@ -255,7 +256,6 @@ DUMMY_MACHINE_FUNCTION_PASS("rename-independent-subregs", RenameIndependentSubre
 DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass)
 DUMMY_MACHINE_FUNCTION_PASS("shrink-wrap", ShrinkWrapPass)
 DUMMY_MACHINE_FUNCTION_PASS("simple-register-coalescing", RegisterCoalescerPass)
-DUMMY_MACHINE_FUNCTION_PASS("stack-coloring", StackColoringPass)
 DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass)
 DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
 DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
index f05b90114d75a682cea08c07361a295fe741573d..886b4d3d6894dcede186c1d0af99930a15abc8f9 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
@@ -180,6 +180,7 @@ public:
   };
 
   using FuncRecordsStorage = std::unique_ptr<MemoryBuffer>;
+  using CoverageMapCopyStorage = std::unique_ptr<MemoryBuffer>;
 
 private:
   std::vector<std::string> Filenames;
@@ -195,9 +196,16 @@ private:
   // D69471, which can split up function records into multiple sections on ELF.
   FuncRecordsStorage FuncRecords;
 
+  // Used to tie the lifetimes of an optional copy of the coverage mapping data
+  // to the lifetime of this BinaryCoverageReader instance. Needed to support
+  // Wasm object format, which might require realignment of section contents.
+  CoverageMapCopyStorage CoverageMapCopy;
+
   BinaryCoverageReader(std::unique_ptr<InstrProfSymtab> Symtab,
-                       FuncRecordsStorage &&FuncRecords)
-      : ProfileNames(std::move(Symtab)), FuncRecords(std::move(FuncRecords)) {}
+                       FuncRecordsStorage &&FuncRecords,
+                       CoverageMapCopyStorage &&CoverageMapCopy)
+      : ProfileNames(std::move(Symtab)), FuncRecords(std::move(FuncRecords)),
+        CoverageMapCopy(std::move(CoverageMapCopy)) {}
 
 public:
   BinaryCoverageReader(const BinaryCoverageReader &) = delete;
@@ -212,6 +220,7 @@ public:
   static Expected<std::unique_ptr<BinaryCoverageReader>>
   createCoverageReaderFromBuffer(
       StringRef Coverage, FuncRecordsStorage &&FuncRecords,
+      CoverageMapCopyStorage &&CoverageMap,
       std::unique_ptr<InstrProfSymtab> ProfileNamesPtr, uint8_t BytesInAddress,
       llvm::endianness Endian, StringRef CompilationDir = "");
 
diff --git a/llvm/include/llvm/SandboxIR/Context.h b/llvm/include/llvm/SandboxIR/Context.h
index 77924fbcd5acea4361b082f9657090886ac1dfa8..1285598a1c0282218da73c5417c722239859bf59 100644
--- a/llvm/include/llvm/SandboxIR/Context.h
+++ b/llvm/include/llvm/SandboxIR/Context.h
@@ -68,6 +68,7 @@ protected:
   }
   /// Get or create a sandboxir::Constant from an existing LLVM IR \p LLVMC.
   Constant *getOrCreateConstant(llvm::Constant *LLVMC);
+  friend class Utils; // For getMemoryBase
 
   // Friends for getOrCreateConstant().
 #define DEF_CONST(ID, CLASS) friend class CLASS;
diff --git a/llvm/include/llvm/SandboxIR/Utils.h b/llvm/include/llvm/SandboxIR/Utils.h
index 4ff4509b7086c50c1e2b3532d33ceee099fbca49..a73498adea1d59418bdfeded883e9cc3ea632d04 100644
--- a/llvm/include/llvm/SandboxIR/Utils.h
+++ b/llvm/include/llvm/SandboxIR/Utils.h
@@ -50,6 +50,16 @@ public:
     return const_cast<Instruction *>(I);
   }
 
+  /// \Returns the base Value for load or store instruction \p LSI.
+  template <typename LoadOrStoreT>
+  static Value *getMemInstructionBase(const LoadOrStoreT *LSI) {
+    static_assert(std::is_same_v<LoadOrStoreT, LoadInst> ||
+                      std::is_same_v<LoadOrStoreT, StoreInst>,
+                  "Expected sandboxir::Load or sandboxir::Store!");
+    return LSI->Ctx.getOrCreateValue(
+        getUnderlyingObject(LSI->getPointerOperand()->Val));
+  }
+
   /// \Returns the number of bits required to represent the operands or return
   /// value of \p V in \p DL.
   static unsigned getNumBits(Value *V, const DataLayout &DL) {
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def
index 7480d45807f23ce81ff4300a517e69bd4774a763..c5cd1b1bc63765b0eeed9668862b9f1f27f80659 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.def
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def
@@ -342,6 +342,7 @@ ARM_CPU_NAME("cortex-m4", ARMV7EM, FK_FPV4_SP_D16, true, ARM::AEK_NONE)
 ARM_CPU_NAME("cortex-m7", ARMV7EM, FK_FPV5_D16, false, ARM::AEK_NONE)
 ARM_CPU_NAME("cortex-m23", ARMV8MBaseline, FK_NONE, false, ARM::AEK_NONE)
 ARM_CPU_NAME("cortex-m33", ARMV8MMainline, FK_FPV5_SP_D16, false, ARM::AEK_DSP)
+ARM_CPU_NAME("star-mc1", ARMV8MMainline, FK_FPV5_SP_D16, false, ARM::AEK_DSP)
 ARM_CPU_NAME("cortex-m35p", ARMV8MMainline, FK_FPV5_SP_D16, false, ARM::AEK_DSP)
 ARM_CPU_NAME("cortex-m55", ARMV8_1MMainline, FK_FP_ARMV8_FULLFP16_D16, false,
              (ARM::AEK_DSP | ARM::AEK_SIMD | ARM::AEK_FP | ARM::AEK_FP16))
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 280bd7a9febc1f4b6453c2450e8c18169298b36d..779bd9017d7f12721323688e24bebcb17dda8bfd 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -244,10 +244,13 @@ public:
     UnknownEnvironment,
 
     GNU,
+    GNUT64,
     GNUABIN32,
     GNUABI64,
     GNUEABI,
+    GNUEABIT64,
     GNUEABIHF,
+    GNUEABIHFT64,
     GNUF32,
     GNUF64,
     GNUSF,
@@ -608,11 +611,12 @@ public:
 
   bool isGNUEnvironment() const {
     EnvironmentType Env = getEnvironment();
-    return Env == Triple::GNU || Env == Triple::GNUABIN32 ||
-           Env == Triple::GNUABI64 || Env == Triple::GNUEABI ||
-           Env == Triple::GNUEABIHF || Env == Triple::GNUF32 ||
-           Env == Triple::GNUF64 || Env == Triple::GNUSF ||
-           Env == Triple::GNUX32;
+    return Env == Triple::GNU || Env == Triple::GNUT64 ||
+           Env == Triple::GNUABIN32 || Env == Triple::GNUABI64 ||
+           Env == Triple::GNUEABI || Env == Triple::GNUEABIT64 ||
+           Env == Triple::GNUEABIHF || Env == Triple::GNUEABIHFT64 ||
+           Env == Triple::GNUF32 || Env == Triple::GNUF64 ||
+           Env == Triple::GNUSF || Env == Triple::GNUX32;
   }
 
   /// Tests whether the OS is Haiku.
@@ -873,9 +877,11 @@ public:
     return (isARM() || isThumb()) &&
            (getEnvironment() == Triple::EABI ||
             getEnvironment() == Triple::GNUEABI ||
+            getEnvironment() == Triple::GNUEABIT64 ||
             getEnvironment() == Triple::MuslEABI ||
             getEnvironment() == Triple::EABIHF ||
             getEnvironment() == Triple::GNUEABIHF ||
+            getEnvironment() == Triple::GNUEABIHFT64 ||
             getEnvironment() == Triple::OpenHOS ||
             getEnvironment() == Triple::MuslEABIHF || isAndroid()) &&
            isOSBinFormatELF();
@@ -1059,6 +1065,22 @@ public:
     return getArch() == Triple::bpfel || getArch() == Triple::bpfeb;
   }
 
+  /// Tests if the target forces 64-bit time_t on a 32-bit architecture.
+  bool isTime64ABI() const {
+    EnvironmentType Env = getEnvironment();
+    return Env == Triple::GNUT64 || Env == Triple::GNUEABIT64 ||
+           Env == Triple::GNUEABIHFT64;
+  }
+
+  /// Tests if the target forces hardfloat.
+  bool isHardFloatABI() const {
+    EnvironmentType Env = getEnvironment();
+    return Env == llvm::Triple::GNUEABIHF ||
+           Env == llvm::Triple::GNUEABIHFT64 ||
+           Env == llvm::Triple::MuslEABIHF ||
+           Env == llvm::Triple::EABIHF;
+  }
+
   /// Tests whether the target supports comdat
   bool supportsCOMDAT() const {
     return !(isOSBinFormatMachO() || isOSBinFormatXCOFF() ||
diff --git a/llvm/include/llvm/Transforms/IPO/MemProfContextDisambiguation.h b/llvm/include/llvm/Transforms/IPO/MemProfContextDisambiguation.h
index f5dce01c34e7aa54f8f53aa7038a9a5371da72bf..f2de08353f2a61477ac07a5e2968eb69f235f460 100644
--- a/llvm/include/llvm/Transforms/IPO/MemProfContextDisambiguation.h
+++ b/llvm/include/llvm/Transforms/IPO/MemProfContextDisambiguation.h
@@ -15,9 +15,11 @@
 #ifndef LLVM_TRANSFORMS_IPO_MEMPROF_CONTEXT_DISAMBIGUATION_H
 #define LLVM_TRANSFORMS_IPO_MEMPROF_CONTEXT_DISAMBIGUATION_H
 
+#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
 #include <functional>
 
 namespace llvm {
@@ -36,6 +38,37 @@ class MemProfContextDisambiguation
   /// the IR.
   bool applyImport(Module &M);
 
+  // Builds the symtab and analysis used for ICP during ThinLTO backends.
+  bool initializeIndirectCallPromotionInfo(Module &M);
+
+  // Data structure for saving indirect call profile info for use in ICP with
+  // cloning.
+  struct ICallAnalysisData {
+    CallBase *CB;
+    std::vector<InstrProfValueData> CandidateProfileData;
+    uint32_t NumCandidates;
+    uint64_t TotalCount;
+    size_t CallsiteInfoStartIndex;
+  };
+
+  // Record information needed for ICP of an indirect call, depending on its
+  // profile information and the clone information recorded in the corresponding
+  // CallsiteInfo records. The SI iterator point to the current iteration point
+  // through AllCallsites in this function, and will be updated in this method
+  // as we iterate through profiled targets. The number of clones recorded for
+  // this indirect call is returned. The necessary information is recorded in
+  // the ICallAnalysisInfo list for later ICP.
+  unsigned recordICPInfo(CallBase *CB, ArrayRef<CallsiteInfo> AllCallsites,
+                         ArrayRef<CallsiteInfo>::iterator &SI,
+                         SmallVector<ICallAnalysisData> &ICallAnalysisInfo);
+
+  // Actually performs any needed ICP in the function, using the information
+  // recorded in the ICallAnalysisInfo list.
+  void performICP(Module &M, ArrayRef<CallsiteInfo> AllCallsites,
+                  ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps,
+                  ArrayRef<ICallAnalysisData> ICallAnalysisInfo,
+                  OptimizationRemarkEmitter &ORE);
+
   /// Import summary containing cloning decisions for the ThinLTO backend.
   const ModuleSummaryIndex *ImportSummary;
 
@@ -43,8 +76,18 @@ class MemProfContextDisambiguation
   // ThinLTO backend via opt (to simulate distributed ThinLTO).
   std::unique_ptr<ModuleSummaryIndex> ImportSummaryForTesting;
 
+  // Whether we are building with SamplePGO. This is needed for correctly
+  // updating profile metadata on speculatively promoted calls.
+  bool isSamplePGO;
+
+  // Used when performing indirect call analysis and promotion when cloning in
+  // the ThinLTO backend during applyImport.
+  std::unique_ptr<InstrProfSymtab> Symtab;
+  std::unique_ptr<ICallPromotionAnalysis> ICallAnalysis;
+
 public:
-  MemProfContextDisambiguation(const ModuleSummaryIndex *Summary = nullptr);
+  MemProfContextDisambiguation(const ModuleSummaryIndex *Summary = nullptr,
+                               bool isSamplePGO = false);
 
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 2806326593649eadda1fdfb608446e284f6654a0..5697d983c9ad4801a54a2c959847c203c91e9530 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -47,6 +47,7 @@ struct PoisonFlags {
   unsigned Exact : 1;
   unsigned Disjoint : 1;
   unsigned NNeg : 1;
+  unsigned SameSign : 1;
   GEPNoWrapFlags GEPNW;
 
   PoisonFlags(const Instruction *I);
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
index 5fa57efc1462e88cb285b76bc856ca48a9a581c8..ae3ceed447c40b99f705362f43044b50796d400a 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
@@ -96,9 +96,14 @@ protected:
   // TODO: Use a PointerIntPair for SubclassID and I.
   /// For isa/dyn_cast etc.
   DGNodeID SubclassID;
+  /// The number of unscheduled successors.
+  unsigned UnscheduledSuccs = 0;
+  /// This is true if this node has been scheduled.
+  bool Scheduled = false;
 
   DGNode(Instruction *I, DGNodeID ID) : I(I), SubclassID(ID) {}
-  friend class MemDGNode; // For constructor.
+  friend class MemDGNode;       // For constructor.
+  friend class DependencyGraph; // For UnscheduledSuccs
 
 public:
   DGNode(Instruction *I) : I(I), SubclassID(DGNodeID::DGNode) {
@@ -106,6 +111,10 @@ public:
   }
   DGNode(const DGNode &Other) = delete;
   virtual ~DGNode() = default;
+  /// \Returns the number of unscheduled successors.
+  unsigned getNumUnscheduledSuccs() const { return UnscheduledSuccs; }
+  /// \Returns true if this node has been scheduled.
+  bool scheduled() const { return Scheduled; }
   /// \Returns true if this is before \p Other in program order.
   bool comesBefore(const DGNode *Other) { return I->comesBefore(Other->I); }
   using iterator = PredIterator;
@@ -215,8 +224,16 @@ public:
   MemDGNode *getPrevNode() const { return PrevMemN; }
   /// \Returns the next Mem DGNode in instruction order.
   MemDGNode *getNextNode() const { return NextMemN; }
-  /// Adds the mem dependency edge PredN->this.
-  void addMemPred(MemDGNode *PredN) { MemPreds.insert(PredN); }
+  /// Adds the mem dependency edge PredN->this. This also increments the
+  /// UnscheduledSuccs counter of the predecessor if this node has not been
+  /// scheduled.
+  void addMemPred(MemDGNode *PredN) {
+    [[maybe_unused]] auto Inserted = MemPreds.insert(PredN).second;
+    assert(Inserted && "PredN already exists!");
+    if (!Scheduled) {
+      ++PredN->UnscheduledSuccs;
+    }
+  }
   /// \Returns true if there is a memory dependency N->this.
   bool hasMemPred(DGNode *N) const {
     if (auto *MN = dyn_cast<MemDGNode>(N))
@@ -284,6 +301,14 @@ private:
   /// \p DstN.
   void scanAndAddDeps(MemDGNode &DstN, const Interval<MemDGNode> &SrcScanRange);
 
+  /// Sets the UnscheduledSuccs of all DGNodes in \p NewInterval based on
+  /// def-use edges.
+  void setDefUseUnscheduledSuccs(const Interval<Instruction> &NewInterval);
+
+  /// Create DAG nodes for instrs in \p NewInterval and update the MemNode
+  /// chain.
+  void createNewNodes(const Interval<Instruction> &NewInterval);
+
 public:
   DependencyGraph(AAResults &AA)
       : BatchAA(std::make_unique<BatchAAResults>(AA)) {}
@@ -309,8 +334,10 @@ public:
     return It->second.get();
   }
   /// Build/extend the dependency graph such that it includes \p Instrs. Returns
-  /// the interval spanning \p Instrs.
+  /// the range of instructions added to the DAG.
   Interval<Instruction> extend(ArrayRef<Instruction *> Instrs);
+  /// \Returns the range of instructions included in the DAG.
+  Interval<Instruction> getInterval() const { return DAGInterval; }
 #ifndef NDEBUG
   void print(raw_ostream &OS) const;
   LLVM_DUMP_METHOD void dump() const;
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
index e0c581f1d50b4066859a8f99692eea30ba6771a9..e2d0b82489ddc7cc02476e414628bfede205291d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
@@ -74,66 +74,72 @@ public:
 };
 
 template <typename T> class Interval {
-  T *From;
-  T *To;
+  T *Top;
+  T *Bottom;
 
 public:
-  Interval() : From(nullptr), To(nullptr) {}
-  Interval(T *From, T *To) : From(From), To(To) {
-    assert((From == To || From->comesBefore(To)) &&
-           "From should come before From!");
+  Interval() : Top(nullptr), Bottom(nullptr) {}
+  Interval(T *Top, T *Bottom) : Top(Top), Bottom(Bottom) {
+    assert((Top == Bottom || Top->comesBefore(Bottom)) &&
+           "Top should come before Bottom!");
   }
   Interval(ArrayRef<T *> Elems) {
     assert(!Elems.empty() && "Expected non-empty Elems!");
-    From = Elems[0];
-    To = Elems[0];
+    Top = Elems[0];
+    Bottom = Elems[0];
     for (auto *I : drop_begin(Elems)) {
-      if (I->comesBefore(From))
-        From = I;
-      else if (To->comesBefore(I))
-        To = I;
+      if (I->comesBefore(Top))
+        Top = I;
+      else if (Bottom->comesBefore(I))
+        Bottom = I;
     }
   }
   bool empty() const {
-    assert(((From == nullptr && To == nullptr) ||
-            (From != nullptr && To != nullptr)) &&
+    assert(((Top == nullptr && Bottom == nullptr) ||
+            (Top != nullptr && Bottom != nullptr)) &&
            "Either none or both should be null");
-    return From == nullptr;
+    return Top == nullptr;
   }
   bool contains(T *I) const {
     if (empty())
       return false;
-    return (From == I || From->comesBefore(I)) &&
-           (I == To || I->comesBefore(To));
+    return (Top == I || Top->comesBefore(I)) &&
+           (I == Bottom || I->comesBefore(Bottom));
   }
-  T *top() const { return From; }
-  T *bottom() const { return To; }
+  T *top() const { return Top; }
+  T *bottom() const { return Bottom; }
 
   using iterator = IntervalIterator<T, Interval>;
-  iterator begin() { return iterator(From, *this); }
+  iterator begin() { return iterator(Top, *this); }
   iterator end() {
-    return iterator(To != nullptr ? To->getNextNode() : nullptr, *this);
+    return iterator(Bottom != nullptr ? Bottom->getNextNode() : nullptr, *this);
   }
   iterator begin() const {
-    return iterator(From, const_cast<Interval &>(*this));
+    return iterator(Top, const_cast<Interval &>(*this));
   }
   iterator end() const {
-    return iterator(To != nullptr ? To->getNextNode() : nullptr,
+    return iterator(Bottom != nullptr ? Bottom->getNextNode() : nullptr,
                     const_cast<Interval &>(*this));
   }
   /// Equality.
   bool operator==(const Interval &Other) const {
-    return From == Other.From && To == Other.To;
+    return Top == Other.Top && Bottom == Other.Bottom;
   }
   /// Inequality.
   bool operator!=(const Interval &Other) const { return !(*this == Other); }
+  /// \Returns true if this interval comes before \p Other in program order.
+  /// This expects disjoint intervals.
+  bool comesBefore(const Interval &Other) const {
+    assert(disjoint(Other) && "Expect disjoint intervals!");
+    return bottom()->comesBefore(Other.top());
+  }
   /// \Returns true if this and \p Other have nothing in common.
   bool disjoint(const Interval &Other) const {
     if (Other.empty())
       return true;
     if (empty())
       return true;
-    return Other.To->comesBefore(From) || To->comesBefore(Other.From);
+    return Other.Bottom->comesBefore(Top) || Bottom->comesBefore(Other.Top);
   }
   /// \Returns the intersection between this and \p Other.
   // Example:
@@ -148,14 +154,14 @@ public:
     // 1. No overlap
     // A---B      this
     //       C--D Other
-    if (To->comesBefore(Other.From) || Other.To->comesBefore(From))
+    if (Bottom->comesBefore(Other.Top) || Other.Bottom->comesBefore(Top))
       return Interval();
     // 2. Overlap.
     // A---B   this
     //   C--D  Other
-    auto NewFromI = From->comesBefore(Other.From) ? Other.From : From;
-    auto NewToI = To->comesBefore(Other.To) ? To : Other.To;
-    return Interval(NewFromI, NewToI);
+    auto NewTopI = Top->comesBefore(Other.Top) ? Other.Top : Top;
+    auto NewBottomI = Bottom->comesBefore(Other.Bottom) ? Bottom : Other.Bottom;
+    return Interval(NewTopI, NewBottomI);
   }
   /// Difference operation. This returns up to two intervals.
   // Example:
@@ -172,11 +178,11 @@ public:
     Interval Intersection = intersection(Other);
     SmallVector<Interval, 2> Result;
     // Part 1, skip if empty.
-    if (From != Intersection.From)
-      Result.emplace_back(From, Intersection.From->getPrevNode());
+    if (Top != Intersection.Top)
+      Result.emplace_back(Top, Intersection.Top->getPrevNode());
     // Part 2, skip if empty.
-    if (Intersection.To != To)
-      Result.emplace_back(Intersection.To->getNextNode(), To);
+    if (Intersection.Bottom != Bottom)
+      Result.emplace_back(Intersection.Bottom->getNextNode(), Bottom);
     return Result;
   }
   /// \Returns the interval difference `this - Other`. This will crash in Debug
@@ -196,9 +202,9 @@ public:
       return Other;
     if (Other.empty())
       return *this;
-    auto *NewFrom = From->comesBefore(Other.From) ? From : Other.From;
-    auto *NewTo = To->comesBefore(Other.To) ? Other.To : To;
-    return {NewFrom, NewTo};
+    auto *NewTop = Top->comesBefore(Other.Top) ? Top : Other.Top;
+    auto *NewBottom = Bottom->comesBefore(Other.Bottom) ? Other.Bottom : Bottom;
+    return {NewTop, NewBottom};
   }
 
 #ifndef NDEBUG
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
index 619c2147f2e5c4c7643ff21e7aa3d7f7bfe9a688..626fcb5afcb90b6b3517033bfaee98f3bfeecabd 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
@@ -54,6 +54,8 @@ public:
     NumUnusedBits += Utils::getNumBits(I);
   }
 
+  virtual void insert(Instruction *I, ScalarEvolution &SE) = 0;
+
   unsigned getFirstUnusedElementIdx() const {
     for (unsigned ElmIdx : seq<unsigned>(0, Seeds.size()))
       if (!isUsed(ElmIdx))
@@ -96,6 +98,9 @@ public:
   MutableArrayRef<Instruction *>
   getSlice(unsigned StartIdx, unsigned MaxVecRegBits, bool ForcePowOf2);
 
+  /// \Returns the number of seed elements in the bundle.
+  std::size_t size() const { return Seeds.size(); }
+
 protected:
   SmallVector<Instruction *> Seeds;
   /// The lanes that we have already vectorized.
@@ -148,7 +153,7 @@ public:
                   "Expected LoadInst or StoreInst!");
     assert(isa<LoadOrStoreT>(MemI) && "Expected Load or Store!");
   }
-  void insert(sandboxir::Instruction *I, ScalarEvolution &SE) {
+  void insert(sandboxir::Instruction *I, ScalarEvolution &SE) override {
     assert(isa<LoadOrStoreT>(I) && "Expected a Store or a Load!");
     auto Cmp = [&SE](Instruction *I0, Instruction *I1) {
       return Utils::atLowerAddress(cast<LoadOrStoreT>(I0),
@@ -162,5 +167,123 @@ public:
 using StoreSeedBundle = MemSeedBundle<sandboxir::StoreInst>;
 using LoadSeedBundle = MemSeedBundle<sandboxir::LoadInst>;
 
+/// Class to conveniently track Seeds within SeedBundles. Saves newly collected
+/// seeds in the proper bundle. Supports constant-time removal, as seeds and
+/// entire bundles are vectorized and marked used to signify removal. Iterators
+/// skip bundles that are completely used.
+class SeedContainer {
+  // Use the same key for different seeds if they are the same type and
+  // reference the same pointer, even if at different offsets. This directs
+  // potentially vectorizable seeds into the same bundle.
+  using KeyT = std::tuple<Value *, Type *, Instruction::Opcode>;
+  // Trying to vectorize too many seeds at once is expensive in
+  // compilation-time. Use a vector of bundles (all with the same key) to
+  // partition the candidate set into more manageable units. Each bundle is
+  // size-limited by sbvec-seed-bundle-size-limit.  TODO: There might be a
+  // better way to divide these than by simple insertion order.
+  using ValT = SmallVector<std::unique_ptr<SeedBundle>>;
+  using BundleMapT = MapVector<KeyT, ValT>;
+  // Map from {pointer, Type, Opcode} to a vector of bundles.
+  BundleMapT Bundles;
+  // Allows finding a particular Instruction's bundle.
+  DenseMap<Instruction *, SeedBundle *> SeedLookupMap;
+
+  ScalarEvolution &SE;
+
+  template <typename LoadOrStoreT> KeyT getKey(LoadOrStoreT *LSI) const;
+
+public:
+  SeedContainer(ScalarEvolution &SE) : SE(SE) {}
+
+  class iterator {
+    BundleMapT *Map = nullptr;
+    BundleMapT::iterator MapIt;
+    ValT *Vec = nullptr;
+    size_t VecIdx;
+
+  public:
+    using difference_type = std::ptrdiff_t;
+    using value_type = SeedBundle;
+    using pointer = value_type *;
+    using reference = value_type &;
+    using iterator_category = std::input_iterator_tag;
+
+    /// Iterates over the \p Map of SeedBundle Vectors, starting at \p MapIt,
+    /// and \p Vec at \p VecIdx, skipping vectors that are completely
+    /// used. Iteration order over the keys {Pointer, Type, Opcode} follows
+    /// DenseMap iteration order. For a given key, the vectors of
+    /// SeedBundles will be returned in insertion order. As in the
+    /// pseudo code below:
+    ///
+    /// for Key,Value in Bundles
+    ///   for SeedBundleVector in Value
+    ///     for SeedBundle in SeedBundleVector
+    ///        if !SeedBundle.allUsed() ...
+    ///
+    /// Note that the bundles themselves may have additional ordering, created
+    /// by the subclasses by insertAt. The bundles themselves may also have used
+    /// instructions.
+    iterator(BundleMapT &Map, BundleMapT::iterator MapIt, ValT *Vec, int VecIdx)
+        : Map(&Map), MapIt(MapIt), Vec(Vec), VecIdx(VecIdx) {}
+    value_type &operator*() {
+      assert(Vec != nullptr && "Already at end!");
+      return *(*Vec)[VecIdx];
+    }
+    // Skip completely used bundles by repeatedly calling operator++().
+    void skipUsed() {
+      while (Vec && VecIdx < Vec->size() && this->operator*().allUsed())
+        ++(*this);
+    }
+    // Iterators iterate over the bundles
+    iterator &operator++() {
+      assert(VecIdx >= 0 && "Already at end!");
+      ++VecIdx;
+      if (VecIdx >= Vec->size()) {
+        assert(MapIt != Map->end() && "Already at end!");
+        VecIdx = 0;
+        ++MapIt;
+        if (MapIt != Map->end())
+          Vec = &MapIt->second;
+        else {
+          Vec = nullptr;
+        }
+      }
+      skipUsed();
+      return *this;
+    }
+    iterator operator++(int) {
+      auto Copy = *this;
+      ++(*this);
+      return Copy;
+    }
+    bool operator==(const iterator &Other) const {
+      assert(Map == Other.Map && "Iterator of different objects!");
+      return MapIt == Other.MapIt && VecIdx == Other.VecIdx;
+    }
+    bool operator!=(const iterator &Other) const { return !(*this == Other); }
+  };
+  using const_iterator = BundleMapT::const_iterator;
+  template <typename LoadOrStoreT> void insert(LoadOrStoreT *LSI);
+  // To support constant-time erase, these just mark the element used, rather
+  // than actually removing them from the bundle.
+  bool erase(Instruction *I);
+  bool erase(const KeyT &Key) { return Bundles.erase(Key); }
+  iterator begin() {
+    if (Bundles.empty())
+      return end();
+    auto BeginIt =
+        iterator(Bundles, Bundles.begin(), &Bundles.begin()->second, 0);
+    BeginIt.skipUsed();
+    return BeginIt;
+  }
+  iterator end() { return iterator(Bundles, Bundles.end(), nullptr, 0); }
+  unsigned size() const { return Bundles.size(); }
+
+#ifndef NDEBUG
+  LLVM_DUMP_METHOD void dump() const;
+#endif // NDEBUG
+};
+
 } // namespace llvm::sandboxir
+
 #endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_SEEDCOLLECTOR_H
diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp
index 9c86ef13f6df038e2d8dfe58a6a03135c75369b2..a0e57ab741dfa8701d0148bb926dd52b943013be 100644
--- a/llvm/lib/Analysis/AssumptionCache.cpp
+++ b/llvm/lib/Analysis/AssumptionCache.cpp
@@ -50,9 +50,7 @@ AssumptionCache::getOrInsertAffectedValues(Value *V) {
   if (AVI != AffectedValues.end())
     return AVI->second;
 
-  auto AVIP = AffectedValues.insert(
-      {AffectedValueCallbackVH(V, this), SmallVector<ResultElem, 1>()});
-  return AVIP.first->second;
+  return AffectedValues[AffectedValueCallbackVH(V, this)];
 }
 
 static void
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index c6907cb128bb47ed73c3aef028372c9bd9e11d76..45702fa25d8b14245e4b6265422137fb84d242e0 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -151,9 +151,9 @@ std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
     return FAM.getResult<TargetLibraryAnalysis>(F);
   };
 
+  Function &Callee = *CB.getCalledFunction();
+  auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
   auto GetInlineCost = [&](CallBase &CB) {
-    Function &Callee = *CB.getCalledFunction();
-    auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
     bool RemarksEnabled =
         Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
             DEBUG_TYPE);
@@ -161,7 +161,7 @@ std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
                          GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
   };
   return llvm::shouldInline(
-      CB, GetInlineCost, ORE,
+      CB, CalleeTTI, GetInlineCost, ORE,
       Params.EnableDeferral.value_or(EnableInlineDeferral));
 }
 
@@ -247,7 +247,8 @@ bool InlineAdvisorAnalysis::Result::tryCreate(
 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the
 /// caller if \p CB is suppressed for inlining.
 static bool
-shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
+shouldBeDeferred(Function *Caller, TargetTransformInfo &CalleeTTI,
+                 InlineCost IC, int &TotalSecondaryCost,
                  function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
   // For now we only handle local or inline functions.
   if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
@@ -320,7 +321,7 @@ shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
   // be removed entirely.  We did not account for this above unless there
   // is only one caller of Caller.
   if (ApplyLastCallBonus)
-    TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
+    TotalSecondaryCost -= CalleeTTI.getInliningLastCallToStaticBonus();
 
   // If InlineDeferralScale is negative, then ignore the cost of primary
   // inlining -- IC.getCost() multiplied by the number of callers to Caller.
@@ -374,7 +375,7 @@ void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
 /// using that cost, so we won't do so from this function. Return std::nullopt
 /// if inlining should not be attempted.
 std::optional<InlineCost>
-llvm::shouldInline(CallBase &CB,
+llvm::shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI,
                    function_ref<InlineCost(CallBase &CB)> GetInlineCost,
                    OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
   using namespace ore;
@@ -413,8 +414,8 @@ llvm::shouldInline(CallBase &CB,
   }
 
   int TotalSecondaryCost = 0;
-  if (EnableDeferral &&
-      shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
+  if (EnableDeferral && shouldBeDeferred(Caller, CalleeTTI, IC,
+                                         TotalSecondaryCost, GetInlineCost)) {
     LLVM_DEBUG(dbgs() << "    NOT Inlining: " << CB
                       << " Cost = " << IC.getCost()
                       << ", outer Cost = " << TotalSecondaryCost << '\n');
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index d2c329ba748e586b0d67e7d80af6e6972d5f11ec..dd9ac910456ade0ba1f88e02b121c939c73f44a4 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1943,7 +1943,7 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) {
   // and the callsite.
   int SingleBBBonusPercent = 50;
   int VectorBonusPercent = TTI.getInlinerVectorBonusPercent();
-  int LastCallToStaticBonus = InlineConstants::LastCallToStaticBonus;
+  int LastCallToStaticBonus = TTI.getInliningLastCallToStaticBonus();
 
   // Lambda to set all the above bonus and bonus percentages to 0.
   auto DisallowAllBonuses = [&]() {
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 513f5bcf847b3756ddc31cecd48488940af5ccff..efa3e5eb4dea1f6855ec0e3def2ffb7e330cd10c 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4343,13 +4343,10 @@ static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
   if (isa<PHINode>(I))
     return nullptr;
 
-  if (Op->getType()->isVectorTy()) {
-    // For vector types, the simplification must hold per-lane, so forbid
-    // potentially cross-lane operations like shufflevector.
-    if (!I->getType()->isVectorTy() || isa<ShuffleVectorInst>(I) ||
-        isa<CallBase>(I) || isa<BitCastInst>(I))
-      return nullptr;
-  }
+  // For vector types, the simplification must hold per-lane, so forbid
+  // potentially cross-lane operations like shufflevector.
+  if (Op->getType()->isVectorTy() && !isNotCrossLaneOperation(I))
+    return nullptr;
 
   // Don't fold away llvm.is.constant checks based on assumptions.
   if (match(I, m_Intrinsic<Intrinsic::is_constant>()))
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 2d4961dade9db1faec4b2534e92bcf0f4c585147..1bd9ee651d2b0b713c43c2c452ac8af72c6dc8bf 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -81,6 +81,11 @@ static cl::opt<std::string> ModuleSummaryDotFile(
     "module-summary-dot-file", cl::Hidden, cl::value_desc("filename"),
     cl::desc("File to emit dot graph of new summary into"));
 
+static cl::opt<bool> EnableMemProfIndirectCallSupport(
+    "enable-memprof-indirect-call-support", cl::init(true), cl::Hidden,
+    cl::desc(
+        "Enable MemProf support for summarizing and cloning indirect calls"));
+
 extern cl::opt<bool> ScalePartialSampleProfileWorkingSetSize;
 
 extern cl::opt<unsigned> MaxNumVTableAnnotations;
@@ -404,6 +409,11 @@ static void computeFunctionSummary(
       if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
         HasInlineAsmMaybeReferencingInternal = true;
 
+      // Compute this once per indirect call.
+      uint32_t NumCandidates = 0;
+      uint64_t TotalCount = 0;
+      MutableArrayRef<InstrProfValueData> CandidateProfileData;
+
       auto *CalledValue = CB->getCalledOperand();
       auto *CalledFunction = CB->getCalledFunction();
       if (CalledValue && !CalledFunction) {
@@ -481,9 +491,7 @@ static void computeFunctionSummary(
           }
         }
 
-        uint32_t NumCandidates;
-        uint64_t TotalCount;
-        auto CandidateProfileData =
+        CandidateProfileData =
             ICallAnalysis.getPromotionCandidatesForInstruction(&I, TotalCount,
                                                                NumCandidates);
         for (const auto &Candidate : CandidateProfileData)
@@ -495,16 +503,6 @@ static void computeFunctionSummary(
       if (!IsThinLTO)
         continue;
 
-      // TODO: Skip indirect calls for now. Need to handle these better, likely
-      // by creating multiple Callsites, one per target, then speculatively
-      // devirtualize while applying clone info in the ThinLTO backends. This
-      // will also be important because we will have a different set of clone
-      // versions per target. This handling needs to match that in the ThinLTO
-      // backend so we handle things consistently for matching of callsite
-      // summaries to instructions.
-      if (!CalledFunction)
-        continue;
-
       // Ensure we keep this analysis in sync with the handling in the ThinLTO
       // backend (see MemProfContextDisambiguation::applyImport). Save this call
       // so that we can skip it in checking the reverse case later.
@@ -555,13 +553,24 @@ static void computeFunctionSummary(
         SmallVector<unsigned> StackIdIndices;
         for (auto StackId : InstCallsite)
           StackIdIndices.push_back(Index.addOrGetStackIdIndex(StackId));
-        // Use the original CalledValue, in case it was an alias. We want
-        // to record the call edge to the alias in that case. Eventually
-        // an alias summary will be created to associate the alias and
-        // aliasee.
-        auto CalleeValueInfo =
-            Index.getOrInsertValueInfo(cast<GlobalValue>(CalledValue));
-        Callsites.push_back({CalleeValueInfo, StackIdIndices});
+        if (CalledFunction) {
+          // Use the original CalledValue, in case it was an alias. We want
+          // to record the call edge to the alias in that case. Eventually
+          // an alias summary will be created to associate the alias and
+          // aliasee.
+          auto CalleeValueInfo =
+              Index.getOrInsertValueInfo(cast<GlobalValue>(CalledValue));
+          Callsites.push_back({CalleeValueInfo, StackIdIndices});
+        } else if (EnableMemProfIndirectCallSupport) {
+          // For indirect callsites, create multiple Callsites, one per target.
+          // This enables having a different set of clone versions per target,
+          // and we will apply the cloning decisions while speculatively
+          // devirtualizing in the ThinLTO backends.
+          for (const auto &Candidate : CandidateProfileData) {
+            auto CalleeValueInfo = Index.getOrInsertValueInfo(Candidate.Value);
+            Callsites.push_back({CalleeValueInfo, StackIdIndices});
+          }
+        }
       }
     }
   }
@@ -1214,9 +1223,13 @@ bool llvm::mayHaveMemprofSummary(const CallBase *CB) {
     if (CI && CalledFunction->isIntrinsic())
       return false;
   } else {
-    // TODO: For now skip indirect calls. See comments in
-    // computeFunctionSummary for what is needed to handle this.
-    return false;
+    // Skip inline assembly calls.
+    if (CI && CI->isInlineAsm())
+      return false;
+    // Skip direct calls via Constant.
+    if (!CalledValue || isa<Constant>(CalledValue))
+      return false;
+    return true;
   }
   return true;
 }
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 3d890f05c8ca210df2fa71a633b97aef439928fb..97ea405a5267aebfdf821578205165763afbb9b7 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -4200,7 +4200,7 @@ bool ScalarEvolution::canReuseInstruction(
 
     // Either the value can't be poison, or the S would also be poison if it
     // is.
-    if (PoisonVals.contains(V) || isGuaranteedNotToBePoison(V))
+    if (PoisonVals.contains(V) || ::isGuaranteedNotToBePoison(V))
       continue;
 
     auto *I = dyn_cast<Instruction>(V);
@@ -4303,6 +4303,16 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
   }
 
   for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
+    bool MayBeUB = SCEVExprContains(Ops[i], [this](const SCEV *S) {
+      auto *UDiv = dyn_cast<SCEVUDivExpr>(S);
+      // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
+      // is a non-zero constant, we have to assume the UDiv may be UB.
+      return UDiv && (!isKnownNonZero(UDiv->getOperand(1)) ||
+                      !isGuaranteedNotToBePoison(UDiv->getOperand(1)));
+    });
+
+    if (MayBeUB)
+      continue;
     // We can replace %x umin_seq %y with %x umin %y if either:
     //  * %y being poison implies %x is also poison.
     //  * %x cannot be the saturating value (e.g. zero for umin).
@@ -7298,6 +7308,11 @@ bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
   return false;
 }
 
+bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
+  SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
+  visitAll(Op, PC);
+  return PC.MaybePoison.empty();
+}
 
 bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
   // Only proceed if we can prove that I does not yield poison.
@@ -15050,6 +15065,16 @@ const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
   return SymbolicMaxBackedgeCount;
 }
 
+unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
+  if (!SmallConstantMaxTripCount) {
+    SmallVector<const SCEVPredicate *, 4> Preds;
+    SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(&L, &Preds);
+    for (const auto *P : Preds)
+      addPredicate(*P);
+  }
+  return *SmallConstantMaxTripCount;
+}
+
 void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
   if (Preds->implies(&Pred))
     return;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index f7491acd1cc3bcc8858134b1946c8261df2a25c7..950dd414d191c175e6c1701085d47af50443f356 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -228,6 +228,10 @@ TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier()
   return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier();
 }
 
+int TargetTransformInfo::getInliningLastCallToStaticBonus() const {
+  return TTIImpl->getInliningLastCallToStaticBonus();
+}
+
 unsigned
 TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
   return TTIImpl->adjustInliningThreshold(CB);
@@ -288,6 +292,10 @@ bool TargetTransformInfo::hasBranchDivergence(const Function *F) const {
 }
 
 bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
+  if (const auto *Call = dyn_cast<CallBase>(V)) {
+    if (Call->hasFnAttr(Attribute::NoDivergenceSource))
+      return false;
+  }
   return TTIImpl->isSourceOfDivergence(V);
 }
 
@@ -517,6 +525,13 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
   return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
 }
 
+bool TargetTransformInfo::isLegalInterleavedAccessType(
+    VectorType *VTy, unsigned Factor, Align Alignment,
+    unsigned AddrSpace) const {
+  return TTIImpl->isLegalInterleavedAccessType(VTy, Factor, Alignment,
+                                               AddrSpace);
+}
+
 bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
                                                        Type *DataType) const {
   return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 0a9678670430e13dc3cfa5701ee430fc552b3f25..0a7679639dabdfc795d65153ddcf536426b44d2b 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -6955,6 +6955,14 @@ bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
       V, /* AllowLifetime */ true, /* AllowDroppable */ true);
 }
 
+bool llvm::isNotCrossLaneOperation(const Instruction *I) {
+  if (auto *II = dyn_cast<IntrinsicInst>(I))
+    return isTriviallyVectorizable(II->getIntrinsicID());
+  auto *Shuffle = dyn_cast<ShuffleVectorInst>(I);
+  return (!Shuffle || Shuffle->isSelect()) &&
+         !isa<CallBase, BitCastInst, ExtractElementInst>(I);
+}
+
 bool llvm::isSafeToSpeculativelyExecute(const Instruction *Inst,
                                         const Instruction *CtxI,
                                         AssumptionCache *AC,
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index f71cbe1b4b1e887bd8525bcf002a88d3299c1b4b..759db6db60774c2f685bd5ac96b3cfb985f48615 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -583,6 +583,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(disjoint);
   KEYWORD(inbounds);
   KEYWORD(nneg);
+  KEYWORD(samesign);
   KEYWORD(inrange);
   KEYWORD(addrspace);
   KEYWORD(section);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index de41f3c71c85166387c7b57083b30d798fe7ded3..5c6111d0f21be2bcf3b410a7651e21c968aba171 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -360,7 +360,7 @@ bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
                                               OverloadTys))
           return error(Info.second, "invalid intrinsic signature");
 
-        U.set(Intrinsic::getDeclaration(M, IID, OverloadTys));
+        U.set(Intrinsic::getOrInsertDeclaration(M, IID, OverloadTys));
       }
 
       Info.first->eraseFromParent();
@@ -6952,8 +6952,14 @@ int LLParser::parseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_and:
   case lltok::kw_xor:
     return parseLogical(Inst, PFS, KeywordVal);
-  case lltok::kw_icmp:
-    return parseCompare(Inst, PFS, KeywordVal);
+  case lltok::kw_icmp: {
+    bool SameSign = EatIfPresent(lltok::kw_samesign);
+    if (parseCompare(Inst, PFS, KeywordVal))
+      return true;
+    if (SameSign)
+      cast<ICmpInst>(Inst)->setSameSign();
+    return false;
+  }
   case lltok::kw_fcmp: {
     FastMathFlags FMF = EatFastMathFlagsIfPresent();
     int Res = parseCompare(Inst, PFS, KeywordVal);
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 8ee93253bc2447188b1f332a2f4a6775bd0e1eca..e09532dc4735fb8cf7fb75182ac227260cccfe86 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2066,6 +2066,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::NoCallback;
   case bitc::ATTR_KIND_NO_CAPTURE:
     return Attribute::NoCapture;
+  case bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE:
+    return Attribute::NoDivergenceSource;
   case bitc::ATTR_KIND_NO_DUPLICATE:
     return Attribute::NoDuplicate;
   case bitc::ATTR_KIND_NOFREE:
@@ -5469,9 +5471,6 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       if (IsFP && Record.size() > OpNum+1)
         FMF = getDecodedFastMathFlags(Record[++OpNum]);
 
-      if (OpNum+1 != Record.size())
-        return error("Invalid record");
-
       if (IsFP) {
         if (!CmpInst::isFPPredicate(PredVal))
           return error("Invalid fcmp predicate");
@@ -5480,8 +5479,14 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
         if (!CmpInst::isIntPredicate(PredVal))
           return error("Invalid icmp predicate");
         I = new ICmpInst(PredVal, LHS, RHS);
+        if (Record.size() > OpNum + 1 &&
+            (Record[++OpNum] & (1 << bitc::ICMP_SAME_SIGN)))
+          cast<ICmpInst>(I)->setSameSign();
       }
 
+      if (OpNum + 1 != Record.size())
+        return error("Invalid record");
+
       ResTypeID = getVirtualTypeID(I->getType()->getScalarType());
       if (LHS->getType()->isVectorTy())
         ResTypeID = getVirtualTypeID(I->getType(), ResTypeID);
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index bec0caef58afa80b599421f3cbfab1269db5474b..d9002149fba55af8563f9e1bf336b9fd476ca022 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -763,6 +763,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_NO_CALLBACK;
   case Attribute::NoCapture:
     return bitc::ATTR_KIND_NO_CAPTURE;
+  case Attribute::NoDivergenceSource:
+    return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
   case Attribute::NoDuplicate:
     return bitc::ATTR_KIND_NO_DUPLICATE;
   case Attribute::NoFree:
@@ -1716,6 +1718,9 @@ static uint64_t getOptimizationFlags(const Value *V) {
       Flags |= 1 << bitc::GEP_NUSW;
     if (GEP->hasNoUnsignedWrap())
       Flags |= 1 << bitc::GEP_NUW;
+  } else if (const auto *ICmp = dyn_cast<ICmpInst>(V)) {
+    if (ICmp->hasSameSign())
+      Flags |= 1 << bitc::ICMP_SAME_SIGN;
   }
 
   return Flags;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 27931e118504b9df05b1350900b1391ad3cae6ff..10736305762d21abf9b07b736ae8183d461c4831 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -630,8 +630,7 @@ static void finishCallSiteParams(ValT Val, const DIExpression *Expr,
 static void addToFwdRegWorklist(FwdRegWorklist &Worklist, unsigned Reg,
                                 const DIExpression *Expr,
                                 ArrayRef<FwdRegParamInfo> ParamsToAdd) {
-  auto I = Worklist.insert({Reg, {}});
-  auto &ParamsForFwdReg = I.first->second;
+  auto &ParamsForFwdReg = Worklist[Reg];
   for (auto Param : ParamsToAdd) {
     assert(none_of(ParamsForFwdReg,
                    [Param](const FwdRegParamInfo &D) {
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 16b8d456748facc216b7d90f7c2b27d307ae8345..48cc21ee20f0af5990472a011132937dd4252eff 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -125,7 +125,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeShrinkWrapPass(Registry);
   initializeSjLjEHPreparePass(Registry);
   initializeSlotIndexesWrapperPassPass(Registry);
-  initializeStackColoringPass(Registry);
+  initializeStackColoringLegacyPass(Registry);
   initializeStackFrameLayoutAnalysisPassPass(Registry);
   initializeStackMapLivenessPass(Registry);
   initializeStackProtectorPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 3e09fbad6ab198246393c6f3f2cfb3a61713541d..86f28293ba9ff8728ee9a01c356baa0872e4211d 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -2111,6 +2111,31 @@ bool CodeGenPrepare::optimizeURem(Instruction *Rem) {
   return false;
 }
 
+/// Some targets have better codegen for `ctpop(X) u< 2` than `ctpop(X) == 1`.
+/// This function converts `ctpop(X) ==/!= 1` into `ctpop(X) u</u> 2/1` if the
+/// result cannot be zero.
+static bool adjustIsPower2Test(CmpInst *Cmp, const TargetLowering &TLI,
+                               const TargetTransformInfo &TTI,
+                               const DataLayout &DL) {
+  ICmpInst::Predicate Pred;
+  if (!match(Cmp, m_ICmp(Pred, m_Intrinsic<Intrinsic::ctpop>(), m_One())))
+    return false;
+  if (!ICmpInst::isEquality(Pred))
+    return false;
+  auto *II = cast<IntrinsicInst>(Cmp->getOperand(0));
+
+  if (isKnownNonZero(II, DL)) {
+    if (Pred == ICmpInst::ICMP_EQ) {
+      Cmp->setOperand(1, ConstantInt::get(II->getType(), 2));
+      Cmp->setPredicate(ICmpInst::ICMP_ULT);
+    } else {
+      Cmp->setPredicate(ICmpInst::ICMP_UGT);
+    }
+    return true;
+  }
+  return false;
+}
+
 bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {
   if (sinkCmpExpression(Cmp, *TLI))
     return true;
@@ -2130,6 +2155,9 @@ bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {
   if (foldFCmpToFPClassTest(Cmp, *TLI, *DL))
     return true;
 
+  if (adjustIsPower2Test(Cmp, *TLI, *TTI, *DL))
+    return true;
+
   return false;
 }
 
diff --git a/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp b/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp
index 11f123aa5bed85300e47390aad7c16a6264905c5..0a3d0cf8ec93006ccbdc6244d344666f1b0c1f6a 100644
--- a/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp
+++ b/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp
@@ -356,7 +356,7 @@ static void expandIToFP(Instruction *IToFP) {
   Entry->getTerminator()->eraseFromParent();
 
   Function *CTLZ =
-      Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, IntTy);
+      Intrinsic::getOrInsertDeclaration(F->getParent(), Intrinsic::ctlz, IntTy);
   ConstantInt *True = Builder.getTrue();
 
   // entry:
diff --git a/llvm/lib/CodeGen/ExpandMemCmp.cpp b/llvm/lib/CodeGen/ExpandMemCmp.cpp
index 04222d5b4afd4c15f4b1a60313811e259bfc449c..6d626de0b4e635e04a6b25d199ffc0157ac17e0e 100644
--- a/llvm/lib/CodeGen/ExpandMemCmp.cpp
+++ b/llvm/lib/CodeGen/ExpandMemCmp.cpp
@@ -355,7 +355,7 @@ MemCmpExpansion::LoadPair MemCmpExpansion::getLoadPair(Type *LoadSizeType,
 
   // Swap bytes if required.
   if (BSwapSizeType) {
-    Function *Bswap = Intrinsic::getDeclaration(
+    Function *Bswap = Intrinsic::getOrInsertDeclaration(
         CI->getModule(), Intrinsic::bswap, BSwapSizeType);
     Lhs = Builder.CreateCall(Bswap, Lhs);
     Rhs = Builder.CreateCall(Bswap, Rhs);
diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
index ffe879ff0496484b2b9230f77737769725e2b4ad..32ba3e91822ddb8981a6ab84f0a8c61fdf8c284a 100644
--- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp
+++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
@@ -237,7 +237,7 @@ Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
   if (ElemCount.isScalable()) {
     auto *M = Builder.GetInsertBlock()->getModule();
     Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);
-    Function *ActiveMaskFunc = Intrinsic::getDeclaration(
+    Function *ActiveMaskFunc = Intrinsic::getOrInsertDeclaration(
         M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()});
     // `get_active_lane_mask` performs an implicit less-than comparison.
     Value *ConstZero = Builder.getInt32(0);
@@ -299,7 +299,7 @@ Value *CachingVPExpander::expandPredicationToIntCall(
   case Intrinsic::umin: {
     Value *Op0 = VPI.getOperand(0);
     Value *Op1 = VPI.getOperand(1);
-    Function *Fn = Intrinsic::getDeclaration(
+    Function *Fn = Intrinsic::getOrInsertDeclaration(
         VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
     Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());
     replaceOperation(*NewOp, VPI);
@@ -308,7 +308,7 @@ Value *CachingVPExpander::expandPredicationToIntCall(
   case Intrinsic::bswap:
   case Intrinsic::bitreverse: {
     Value *Op = VPI.getOperand(0);
-    Function *Fn = Intrinsic::getDeclaration(
+    Function *Fn = Intrinsic::getOrInsertDeclaration(
         VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
     Value *NewOp = Builder.CreateCall(Fn, {Op}, VPI.getName());
     replaceOperation(*NewOp, VPI);
@@ -327,7 +327,7 @@ Value *CachingVPExpander::expandPredicationToFPCall(
   case Intrinsic::fabs:
   case Intrinsic::sqrt: {
     Value *Op0 = VPI.getOperand(0);
-    Function *Fn = Intrinsic::getDeclaration(
+    Function *Fn = Intrinsic::getOrInsertDeclaration(
         VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
     Value *NewOp = Builder.CreateCall(Fn, {Op0}, VPI.getName());
     replaceOperation(*NewOp, VPI);
@@ -337,7 +337,7 @@ Value *CachingVPExpander::expandPredicationToFPCall(
   case Intrinsic::minnum: {
     Value *Op0 = VPI.getOperand(0);
     Value *Op1 = VPI.getOperand(1);
-    Function *Fn = Intrinsic::getDeclaration(
+    Function *Fn = Intrinsic::getOrInsertDeclaration(
         VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
     Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());
     replaceOperation(*NewOp, VPI);
@@ -350,7 +350,7 @@ Value *CachingVPExpander::expandPredicationToFPCall(
     Value *Op0 = VPI.getOperand(0);
     Value *Op1 = VPI.getOperand(1);
     Value *Op2 = VPI.getOperand(2);
-    Function *Fn = Intrinsic::getDeclaration(
+    Function *Fn = Intrinsic::getOrInsertDeclaration(
         VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
     Value *NewOp;
     if (Intrinsic::isConstrainedFPIntrinsic(UnpredicatedIntrinsicID))
@@ -594,7 +594,7 @@ bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
     // TODO add caching
     auto *M = VPI.getModule();
     Function *VScaleFunc =
-        Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty);
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::vscale, Int32Ty);
     IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
     Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
     Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale");
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 35ee85b474d043f47be37010e6734128f9932009..2c98b129a1a892f05d1755e122fee1cfed337459 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -148,6 +148,17 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
   unsigned Opcode = MI.getOpcode();
   LLT DstTy = MRI.getType(R);
 
+#ifndef NDEBUG
+  if (DstTy.isFixedVector()) {
+    assert(
+        DstTy.getNumElements() == DemandedElts.getBitWidth() &&
+        "DemandedElt width should equal the fixed vector number of elements");
+  } else {
+    assert(DemandedElts.getBitWidth() == 1 && DemandedElts == APInt(1, 1) &&
+           "DemandedElt width should be 1 for scalars or scalable vectors");
+  }
+#endif
+
   // Handle the case where this is called on a register that does not have a
   // type constraint (i.e. it has a register class constraint instead). This is
   // unlikely to occur except by looking through copies but it is possible for
@@ -196,7 +207,7 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
       if (!DemandedElts[i])
         continue;
 
-      computeKnownBitsImpl(MI.getOperand(i + 1).getReg(), Known2, DemandedElts,
+      computeKnownBitsImpl(MI.getOperand(i + 1).getReg(), Known2, APInt(1, 1),
                            Depth + 1);
 
       // Known bits are the values that are shared by every demanded element.
@@ -503,15 +514,12 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
     break;
   }
   case TargetOpcode::G_UNMERGE_VALUES: {
-    if (DstTy.isVector())
-      break;
     unsigned NumOps = MI.getNumOperands();
     Register SrcReg = MI.getOperand(NumOps - 1).getReg();
-    if (MRI.getType(SrcReg).isVector())
-      return; // TODO: Handle vectors.
+    LLT SrcTy = MRI.getType(SrcReg);
 
-    KnownBits SrcOpKnown;
-    computeKnownBitsImpl(SrcReg, SrcOpKnown, DemandedElts, Depth + 1);
+    if (SrcTy.isVector() && SrcTy.getScalarType() != DstTy.getScalarType())
+      return; // TODO: Handle vector->subelement unmerges
 
     // Figure out the result operand index
     unsigned DstIdx = 0;
@@ -519,7 +527,20 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
          ++DstIdx)
       ;
 
-    Known = SrcOpKnown.extractBits(BitWidth, BitWidth * DstIdx);
+    APInt SubDemandedElts = DemandedElts;
+    if (SrcTy.isVector()) {
+      unsigned DstLanes = DstTy.isVector() ? DstTy.getNumElements() : 1;
+      SubDemandedElts =
+          DemandedElts.zext(SrcTy.getNumElements()).shl(DstIdx * DstLanes);
+    }
+
+    KnownBits SrcOpKnown;
+    computeKnownBitsImpl(SrcReg, SrcOpKnown, SubDemandedElts, Depth + 1);
+
+    if (SrcTy.isVector())
+      Known = SrcOpKnown;
+    else
+      Known = SrcOpKnown.extractBits(BitWidth, BitWidth * DstIdx);
     break;
   }
   case TargetOpcode::G_BSWAP: {
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 932287d1403021d3acd6fc9b43b0f5dd8d67c784..d0464670b292a4734e932048ea0ae50658594bbe 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3913,16 +3913,11 @@ bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
     return false;
   }
 
-  // On PS4/PS5, the "return address" must still be within the calling
-  // function, even if it's at the very end, so emit an explicit TRAP here.
-  // WebAssembly needs an unreachable instruction after a non-returning call,
-  // because the function return type can be different from __stack_chk_fail's
-  // return type (void).
-  const TargetMachine &TM = MF->getTarget();
-  if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
-    LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
-    return false;
-  }
+  // Emit a trap instruction if we are required to do so.
+  const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
+  if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
+    CurBuilder->buildInstr(TargetOpcode::G_TRAP);
+
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/HardwareLoops.cpp b/llvm/lib/CodeGen/HardwareLoops.cpp
index 9205eabcf5684ee8c1a56bbb9d14f5bca4bf39f8..c8a63304a3b63b9abbc5154b8da3ae8386a057e6 100644
--- a/llvm/lib/CodeGen/HardwareLoops.cpp
+++ b/llvm/lib/CodeGen/HardwareLoops.cpp
@@ -512,7 +512,7 @@ Value* HardwareLoop::InsertIterationSetup(Value *LoopCountInit) {
                                    : Intrinsic::test_set_loop_iterations)
                          : (UsePhi ? Intrinsic::start_loop_iterations
                                    : Intrinsic::set_loop_iterations);
-  Function *LoopIter = Intrinsic::getDeclaration(M, ID, Ty);
+  Function *LoopIter = Intrinsic::getOrInsertDeclaration(M, ID, Ty);
   Value *LoopSetup = Builder.CreateCall(LoopIter, LoopCountInit);
 
   // Use the return value of the intrinsic to control the entry of the loop.
@@ -541,9 +541,8 @@ void HardwareLoop::InsertLoopDec() {
           Attribute::StrictFP))
     CondBuilder.setIsFPConstrained(true);
 
-  Function *DecFunc =
-    Intrinsic::getDeclaration(M, Intrinsic::loop_decrement,
-                              LoopDecrement->getType());
+  Function *DecFunc = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::loop_decrement, LoopDecrement->getType());
   Value *Ops[] = { LoopDecrement };
   Value *NewCond = CondBuilder.CreateCall(DecFunc, Ops);
   Value *OldCond = ExitBranch->getCondition();
@@ -566,9 +565,8 @@ Instruction* HardwareLoop::InsertLoopRegDec(Value *EltsRem) {
           Attribute::StrictFP))
     CondBuilder.setIsFPConstrained(true);
 
-  Function *DecFunc =
-      Intrinsic::getDeclaration(M, Intrinsic::loop_decrement_reg,
-                                { EltsRem->getType() });
+  Function *DecFunc = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::loop_decrement_reg, {EltsRem->getType()});
   Value *Ops[] = { EltsRem, LoopDecrement };
   Value *Call = CondBuilder.CreateCall(DecFunc, Ops);
 
diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp
index 256c081b46e2626b728c303e3318ecd20b12697a..f799a8cfc1ba7e9cd0ffebc7364a7a55ce7092fe 100644
--- a/llvm/lib/CodeGen/IntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp
@@ -474,7 +474,7 @@ bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) {
 
   // Okay, we can do this xform, do so now.
   Module *M = CI->getModule();
-  Function *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty);
+  Function *Int = Intrinsic::getOrInsertDeclaration(M, Intrinsic::bswap, Ty);
 
   Value *Op = CI->getArgOperand(0);
   Op = CallInst::Create(Int, Op, CI->getName(), CI->getIterator());
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.cpp b/llvm/lib/CodeGen/LiveDebugVariables.cpp
index 822a1beb489592275362f7adab65e382829523fa..2ff346d3fd0223baf07be6d37d381cfcce1b7db0 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -1625,8 +1625,9 @@ findInsertLocation(MachineBasicBlock *MBB, SlotIndex Idx, LiveIntervals &LIS,
   }
 
   // Don't insert anything after the first terminator, though.
-  return MI->isTerminator() ? MBB->getFirstTerminator() :
-                              std::next(MachineBasicBlock::iterator(MI));
+  auto It = MI->isTerminator() ? MBB->getFirstTerminator()
+                               : std::next(MachineBasicBlock::iterator(MI));
+  return skipDebugInstructionsForward(It, MBB->end());
 }
 
 /// Find an iterator for inserting the next DBG_VALUE instruction
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 74f38e886a6b97f33adcb28fb9d7f19e53a54000..7aaa0f409d5ef9df5acc92ebc27c80fb7df71cf4 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -127,6 +127,16 @@ bool PerTargetMIParsingState::getRegisterByName(StringRef RegName,
   return false;
 }
 
+bool PerTargetMIParsingState::getVRegFlagValue(StringRef FlagName,
+                                               uint8_t &FlagValue) const {
+  const auto *TRI = Subtarget.getRegisterInfo();
+  std::optional<uint8_t> FV = TRI->getVRegFlagValue(FlagName);
+  if (!FV)
+    return true;
+  FlagValue = *FV;
+  return false;
+}
+
 void PerTargetMIParsingState::initNames2InstrOpCodes() {
   if (!Names2InstrOpCodes.empty())
     return;
@@ -1776,6 +1786,7 @@ bool MIParser::parseRegisterOperand(MachineOperand &Dest,
 
         MRI.setRegClassOrRegBank(Reg, static_cast<RegisterBank *>(nullptr));
         MRI.setType(Reg, Ty);
+        MRI.noteNewVirtualRegister(Reg);
       }
     }
   } else if (consumeIfPresent(MIToken::lparen)) {
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 30b1a717caaab992e156fa127b990bee3982e92f..10d3cdcf0c1ce10b5bb31c03b6ecb11f23390888 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -696,6 +696,16 @@ bool MIRParserImpl::parseRegisterInfo(PerFunctionMIParsingState &PFS,
                                  VReg.PreferredRegister.Value, Error))
         return error(Error, VReg.PreferredRegister.SourceRange);
     }
+
+    for (const auto &FlagStringValue : VReg.RegisterFlags) {
+      uint8_t FlagValue;
+      if (Target->getVRegFlagValue(FlagStringValue.Value, FlagValue))
+        return error(FlagStringValue.SourceRange.Start,
+                     Twine("use of undefined register flag '") +
+                         FlagStringValue.Value + "'");
+      Info.Flags.push_back(FlagValue);
+    }
+    RegInfo.noteNewVirtualRegister(Info.VReg);
   }
 
   // Parse the liveins.
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index d52c1d831267f6bff53975853849c88b260e391a..a015cd3c2a55f9b36ca0cd328c45cd606142dadd 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -113,7 +113,8 @@ public:
 
   void print(const MachineFunction &MF);
 
-  void convert(yaml::MachineFunction &MF, const MachineRegisterInfo &RegInfo,
+  void convert(yaml::MachineFunction &YamlMF, const MachineFunction &MF,
+               const MachineRegisterInfo &RegInfo,
                const TargetRegisterInfo *TRI);
   void convert(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI,
                const MachineFrameInfo &MFI);
@@ -231,7 +232,7 @@ void MIRPrinter::print(const MachineFunction &MF) {
   YamlMF.NoVRegs = MF.getProperties().hasProperty(
       MachineFunctionProperties::Property::NoVRegs);
 
-  convert(YamlMF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo());
+  convert(YamlMF, MF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo());
   MachineModuleSlotTracker MST(MMI, &MF);
   MST.incorporateFunction(MF.getFunction());
   convert(MST, YamlMF.FrameInfo, MF.getFrameInfo());
@@ -316,10 +317,21 @@ printStackObjectDbgInfo(const MachineFunction::VariableDbgInfo &DebugVar,
   }
 }
 
-void MIRPrinter::convert(yaml::MachineFunction &MF,
+static void printRegFlags(Register Reg,
+                          std::vector<yaml::FlowStringValue> &RegisterFlags,
+                          const MachineFunction &MF,
+                          const TargetRegisterInfo *TRI) {
+  auto FlagValues = TRI->getVRegFlagsOfReg(Reg, MF);
+  for (auto &Flag : FlagValues) {
+    RegisterFlags.push_back(yaml::FlowStringValue(Flag.str()));
+  }
+}
+
+void MIRPrinter::convert(yaml::MachineFunction &YamlMF,
+                         const MachineFunction &MF,
                          const MachineRegisterInfo &RegInfo,
                          const TargetRegisterInfo *TRI) {
-  MF.TracksRegLiveness = RegInfo.tracksLiveness();
+  YamlMF.TracksRegLiveness = RegInfo.tracksLiveness();
 
   // Print the virtual register definitions.
   for (unsigned I = 0, E = RegInfo.getNumVirtRegs(); I < E; ++I) {
@@ -332,7 +344,8 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
     Register PreferredReg = RegInfo.getSimpleHint(Reg);
     if (PreferredReg)
       printRegMIR(PreferredReg, VReg.PreferredRegister, TRI);
-    MF.VirtualRegisters.push_back(VReg);
+    printRegFlags(Reg, VReg.RegisterFlags, MF, TRI);
+    YamlMF.VirtualRegisters.push_back(VReg);
   }
 
   // Print the live ins.
@@ -341,7 +354,7 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
     printRegMIR(LI.first, LiveIn.Register, TRI);
     if (LI.second)
       printRegMIR(LI.second, LiveIn.VirtualRegister, TRI);
-    MF.LiveIns.push_back(LiveIn);
+    YamlMF.LiveIns.push_back(LiveIn);
   }
 
   // Prints the callee saved registers.
@@ -353,7 +366,7 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
       printRegMIR(*I, Reg, TRI);
       CalleeSavedRegisters.push_back(Reg);
     }
-    MF.CalleeSavedRegisters = CalleeSavedRegisters;
+    YamlMF.CalleeSavedRegisters = CalleeSavedRegisters;
   }
 }
 
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index b7218afdd3820667f67a06b562173e1342632f68..e2c09fe25d55cd4b190fdd2df26c67c48dbae874 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -77,8 +77,10 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ModRef.h"
+#include "llvm/Support/Mutex.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include <algorithm>
@@ -93,21 +95,31 @@ using namespace llvm;
 
 namespace {
 
+/// Used the by the ReportedErrors class to guarantee only one error is reported
+/// at one time.
+static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock;
+
 struct MachineVerifier {
   MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b,
-                  raw_ostream *OS)
-      : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b) {}
+                  raw_ostream *OS, bool AbortOnError = true)
+      : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b),
+        ReportedErrs(AbortOnError) {}
 
-  MachineVerifier(Pass *pass, const char *b, raw_ostream *OS)
-      : PASS(pass), OS(OS ? *OS : nulls()), Banner(b) {}
+  MachineVerifier(Pass *pass, const char *b, raw_ostream *OS,
+                  bool AbortOnError = true)
+      : PASS(pass), OS(OS ? *OS : nulls()), Banner(b),
+        ReportedErrs(AbortOnError) {}
 
   MachineVerifier(const char *b, LiveVariables *LiveVars,
                   LiveIntervals *LiveInts, LiveStacks *LiveStks,
-                  SlotIndexes *Indexes, raw_ostream *OS)
+                  SlotIndexes *Indexes, raw_ostream *OS,
+                  bool AbortOnError = true)
       : OS(OS ? *OS : nulls()), Banner(b), LiveVars(LiveVars),
-        LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes) {}
+        LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes),
+        ReportedErrs(AbortOnError) {}
 
-  unsigned verify(const MachineFunction &MF);
+  /// \returns true if no problems were found.
+  bool verify(const MachineFunction &MF);
 
   MachineFunctionAnalysisManager *MFAM = nullptr;
   Pass *const PASS = nullptr;
@@ -120,8 +132,6 @@ struct MachineVerifier {
   const MachineRegisterInfo *MRI = nullptr;
   const RegisterBankInfo *RBI = nullptr;
 
-  unsigned foundErrors = 0;
-
   // Avoid querying the MachineFunctionProperties for each operand.
   bool isFunctionRegBankSelected = false;
   bool isFunctionSelected = false;
@@ -231,6 +241,44 @@ struct MachineVerifier {
   LiveStacks *LiveStks = nullptr;
   SlotIndexes *Indexes = nullptr;
 
+  /// A class to track the number of reported error and to guarantee that only
+  /// one error is reported at one time.
+  class ReportedErrors {
+    unsigned NumReported = 0;
+    bool AbortOnError;
+
+  public:
+    /// \param AbortOnError -- If set, abort after printing the first error.
+    ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {}
+
+    ~ReportedErrors() {
+      if (!hasError())
+        return;
+      if (AbortOnError)
+        report_fatal_error("Found " + Twine(NumReported) +
+                           " machine code errors.");
+      // Since we haven't aborted, release the lock to allow other threads to
+      // report errors.
+      ReportedErrorsLock->unlock();
+    }
+
+    /// Increment the number of reported errors.
+    /// \returns true if this is the first reported error.
+    bool increment() {
+      // If this is the first error this thread has encountered, grab the lock
+      // to prevent other threads from reporting errors at the same time.
+      // Otherwise we assume we already have the lock.
+      if (!hasError())
+        ReportedErrorsLock->lock();
+      ++NumReported;
+      return NumReported == 1;
+    }
+
+    /// \returns true if an error was reported.
+    bool hasError() { return NumReported; }
+  };
+  ReportedErrors ReportedErrs;
+
   // This is calculated only when trying to verify convergence control tokens.
   // Similar to the LLVM IR verifier, we calculate this locally instead of
   // relying on the pass manager.
@@ -337,11 +385,7 @@ struct MachineVerifierLegacyPass : public MachineFunctionPass {
             MachineFunctionProperties::Property::FailsVerification))
       return false;
 
-    unsigned FoundErrors =
-        MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);
-    if (FoundErrors)
-      report_fatal_error("Found " + Twine(FoundErrors) +
-                         " machine code errors.");
+    MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);
     return false;
   }
 };
@@ -357,10 +401,7 @@ MachineVerifierPass::run(MachineFunction &MF,
   if (MF.getProperties().hasProperty(
           MachineFunctionProperties::Property::FailsVerification))
     return PreservedAnalyses::all();
-  unsigned FoundErrors =
-      MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);
-  if (FoundErrors)
-    report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors.");
+  MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);
   return PreservedAnalyses::all();
 }
 
@@ -380,31 +421,20 @@ void llvm::verifyMachineFunction(const std::string &Banner,
   // LiveIntervals *LiveInts;
   // LiveStacks *LiveStks;
   // SlotIndexes *Indexes;
-  unsigned FoundErrors =
-      MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);
-  if (FoundErrors)
-    report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors.");
+  MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);
 }
 
 bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,
-                             bool AbortOnErrors) const {
-  MachineFunction &MF = const_cast<MachineFunction&>(*this);
-  unsigned FoundErrors = MachineVerifier(p, Banner, OS).verify(MF);
-  if (AbortOnErrors && FoundErrors)
-    report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors.");
-  return FoundErrors == 0;
+                             bool AbortOnError) const {
+  return MachineVerifier(p, Banner, OS, AbortOnError).verify(*this);
 }
 
 bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
                              const char *Banner, raw_ostream *OS,
-                             bool AbortOnErrors) const {
-  MachineFunction &MF = const_cast<MachineFunction &>(*this);
-  unsigned FoundErrors =
-      MachineVerifier(Banner, nullptr, LiveInts, nullptr, Indexes, OS)
-          .verify(MF);
-  if (AbortOnErrors && FoundErrors)
-    report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors.");
-  return FoundErrors == 0;
+                             bool AbortOnError) const {
+  return MachineVerifier(Banner, /*LiveVars=*/nullptr, LiveInts,
+                         /*LiveStks=*/nullptr, Indexes, OS, AbortOnError)
+      .verify(*this);
 }
 
 void MachineVerifier::verifySlotIndexes() const {
@@ -430,9 +460,7 @@ void MachineVerifier::verifyProperties(const MachineFunction &MF) {
     report("Function has NoVRegs property but there are VReg operands", &MF);
 }
 
-unsigned MachineVerifier::verify(const MachineFunction &MF) {
-  foundErrors = 0;
-
+bool MachineVerifier::verify(const MachineFunction &MF) {
   this->MF = &MF;
   TM = &MF.getTarget();
   TII = MF.getSubtarget().getInstrInfo();
@@ -447,7 +475,7 @@ unsigned MachineVerifier::verify(const MachineFunction &MF) {
   // it's expected that the MIR is somewhat broken but that's ok since we'll
   // reset it and clear the FailedISel attribute in ResetMachineFunctions.
   if (isFunctionFailedISel)
-    return foundErrors;
+    return true;
 
   isFunctionRegBankSelected = MF.getProperties().hasProperty(
       MachineFunctionProperties::Property::RegBankSelected);
@@ -544,13 +572,13 @@ unsigned MachineVerifier::verify(const MachineFunction &MF) {
   regMasks.clear();
   MBBInfoMap.clear();
 
-  return foundErrors;
+  return !ReportedErrs.hasError();
 }
 
 void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
   assert(MF);
   OS << '\n';
-  if (!foundErrors++) {
+  if (ReportedErrs.increment()) {
     if (Banner)
       OS << "# " << Banner << '\n';
 
diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
index 9fbb7b461364b10fd37b8772ff03cb69a7fb2e5a..7f3c5cf6cb4436a210a26441b43e8fbe84db14b6 100644
--- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
+++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
@@ -100,20 +100,34 @@ static void replaceWithTLIFunction(IntrinsicInst *II, VFInfo &Info,
 static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
                                     IntrinsicInst *II) {
   assert(II != nullptr && "Intrinsic cannot be null");
+  Intrinsic::ID IID = II->getIntrinsicID();
+  Type *RetTy = II->getType();
+  Type *ScalarRetTy = RetTy->getScalarType();
   // At the moment VFABI assumes the return type is always widened unless it is
   // a void type.
-  auto *VTy = dyn_cast<VectorType>(II->getType());
+  auto *VTy = dyn_cast<VectorType>(RetTy);
   ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(0));
+
+  // OloadTys collects types used in scalar intrinsic overload name.
+  SmallVector<Type *, 3> OloadTys;
+  if (!RetTy->isVoidTy() && isVectorIntrinsicWithOverloadTypeAtArg(IID, -1))
+    OloadTys.push_back(ScalarRetTy);
+
   // Compute the argument types of the corresponding scalar call and check that
   // all vector operands match the previously found EC.
   SmallVector<Type *, 8> ScalarArgTypes;
-  Intrinsic::ID IID = II->getIntrinsicID();
   for (auto Arg : enumerate(II->args())) {
     auto *ArgTy = Arg.value()->getType();
+    bool IsOloadTy = isVectorIntrinsicWithOverloadTypeAtArg(IID, Arg.index());
     if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) {
       ScalarArgTypes.push_back(ArgTy);
+      if (IsOloadTy)
+        OloadTys.push_back(ArgTy);
     } else if (auto *VectorArgTy = dyn_cast<VectorType>(ArgTy)) {
-      ScalarArgTypes.push_back(VectorArgTy->getElementType());
+      auto *ScalarArgTy = VectorArgTy->getElementType();
+      ScalarArgTypes.push_back(ScalarArgTy);
+      if (IsOloadTy)
+        OloadTys.push_back(ScalarArgTy);
       // When return type is void, set EC to the first vector argument, and
       // disallow vector arguments with different ECs.
       if (EC.isZero())
@@ -129,7 +143,7 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
   // using scalar argument types.
   std::string ScalarName =
       Intrinsic::isOverloaded(IID)
-          ? Intrinsic::getName(IID, ScalarArgTypes, II->getModule())
+          ? Intrinsic::getName(IID, OloadTys, II->getModule())
           : Intrinsic::getName(IID).str();
 
   // Try to find the mapping for the scalar version of this intrinsic and the
@@ -146,7 +160,6 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
 
   // Replace the call to the intrinsic with a call to the vector library
   // function.
-  Type *ScalarRetTy = II->getType()->getScalarType();
   FunctionType *ScalarFTy =
       FunctionType::get(ScalarRetTy, ScalarArgTypes, /*isVarArg*/ false);
   const std::string MangledName = VD->getVectorFunctionABIVariantString();
@@ -193,6 +206,8 @@ static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
   for (auto &I : instructions(F)) {
     // Process only intrinsic calls that return void or a vector.
     if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+      if (II->getIntrinsicID() == Intrinsic::not_intrinsic)
+        continue;
       if (!II->getType()->isVectorTy() && !II->getType()->isVoidTy())
         continue;
 
diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp
index e41d1bfb0e530d054d5cfe3140fcbb6a91f5523a..a50909af8bfcfb8a8157447db021b650ce004c18 100644
--- a/llvm/lib/CodeGen/SafeStack.cpp
+++ b/llvm/lib/CodeGen/SafeStack.cpp
@@ -368,7 +368,8 @@ Value *SafeStack::getStackGuard(IRBuilder<> &IRB, Function &F) {
 
   if (!StackGuardVar) {
     TL.insertSSPDeclarations(*M);
-    return IRB.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::stackguard));
+    return IRB.CreateCall(
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::stackguard));
   }
 
   return IRB.CreateLoad(StackPtrTy, StackGuardVar, "StackGuard");
diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index 6a6bc19314d5b6ea5e7aeb29f73848a68e048a2c..374f23099a9080ef6543b70eaad887b6f1d1ea94 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -120,8 +120,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
     for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
       for (WinEHHandlerType &H : TBME.HandlerArray) {
         if (const AllocaInst *AI = H.CatchObj.Alloca)
-          CatchObjects.insert({AI, {}}).first->second.push_back(
-              &H.CatchObj.FrameIndex);
+          CatchObjects[AI].push_back(&H.CatchObj.FrameIndex);
         else
           H.CatchObj.FrameIndex = INT_MAX;
       }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 597ba5d42788351395da9920dc7e4401613d00e9..3cb6f3f56ec9ae6a7da9b1a5f8f1c800d0b8f663 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -5210,7 +5210,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
   if (N->getOpcode() == ISD::ZERO_EXTEND &&
       getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
       TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
-      WidenVT.getScalarSizeInBits()) {
+          WidenVT.getScalarSizeInBits()) {
     InOp = ZExtPromotedInteger(InOp);
     InVT = InOp.getValueType();
     if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
@@ -5227,7 +5227,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
     InVTEC = InVT.getVectorElementCount();
     if (InVTEC == WidenEC) {
       if (N->getNumOperands() == 1)
-        return DAG.getNode(Opcode, DL, WidenVT, InOp);
+        return DAG.getNode(Opcode, DL, WidenVT, InOp, Flags);
       if (N->getNumOperands() == 3) {
         assert(N->isVPOpcode() && "Expected VP opcode");
         SDValue Mask =
@@ -5263,7 +5263,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
       Ops[0] = InOp;
       SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
       if (N->getNumOperands() == 1)
-        return DAG.getNode(Opcode, DL, WidenVT, InVec);
+        return DAG.getNode(Opcode, DL, WidenVT, InVec, Flags);
       return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
     }
 
@@ -5272,7 +5272,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
                                   DAG.getVectorIdxConstant(0, DL));
       // Extract the input and convert the shorten input vector.
       if (N->getNumOperands() == 1)
-        return DAG.getNode(Opcode, DL, WidenVT, InVal);
+        return DAG.getNode(Opcode, DL, WidenVT, InVal, Flags);
       return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
     }
   }
@@ -5287,7 +5287,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
     SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
                               DAG.getVectorIdxConstant(i, DL));
     if (N->getNumOperands() == 1)
-      Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
+      Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, Flags);
     else
       Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 55a60c33691e6e22e11eba112c1e2e19fd77c5a9..d342a14b9775fe385cb34bf62359393d154209d9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1659,6 +1659,11 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
   EVT EltVT = VT.getScalarType();
   const ConstantInt *Elt = &Val;
 
+  // Vector splats are explicit within the DAG, with ConstantSDNode holding the
+  // to-be-splatted scalar ConstantInt.
+  if (isa<VectorType>(Elt->getType()))
+    Elt = ConstantInt::get(*getContext(), Elt->getValue());
+
   // In some cases the vector type is legal but the element type is illegal and
   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
   // inserted value (the type does not need to match the vector element type).
@@ -1812,6 +1817,12 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
 
   EVT EltVT = VT.getScalarType();
+  const ConstantFP *Elt = &V;
+
+  // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
+  // the to-be-splatted scalar ConstantFP.
+  if (isa<VectorType>(Elt->getType()))
+    Elt = ConstantFP::get(*getContext(), Elt->getValue());
 
   // Do the map lookup using the actual bit pattern for the floating point
   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
@@ -1820,7 +1831,7 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
   SDVTList VTs = getVTList(EltVT);
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, VTs, {});
-  ID.AddPointer(&V);
+  ID.AddPointer(Elt);
   void *IP = nullptr;
   SDNode *N = nullptr;
   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
@@ -1828,7 +1839,7 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
       return SDValue(N, 0);
 
   if (!N) {
-    N = newSDNode<ConstantFPSDNode>(isTarget, &V, VTs);
+    N = newSDNode<ConstantFPSDNode>(isTarget, Elt, VTs);
     CSEMap.InsertNode(N, IP);
     InsertNode(N);
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index d35a7d5439984b317c801a02e6620a4e6138c65b..625974e782e37c62f0a8591c35ff9f540a2afa58 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3184,15 +3184,10 @@ SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
   SDValue Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
                                   MVT::isVoid, {}, CallOptions, getCurSDLoc())
                       .second;
-  // On PS4/PS5, the "return address" must still be within the calling
-  // function, even if it's at the very end, so emit an explicit TRAP here.
-  // Passing 'true' for doesNotReturn above won't generate the trap for us.
-  if (TM.getTargetTriple().isPS())
-    Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
-  // WebAssembly needs an unreachable instruction after a non-returning call,
-  // because the function return type can be different from __stack_chk_fail's
-  // return type (void).
-  if (TM.getTargetTriple().isWasm())
+
+  // Emit a trap instruction if we are required to do so.
+  const TargetOptions &TargetOpts = DAG.getTarget().Options;
+  if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
 
   DAG.setRoot(Chain);
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index e6ff9e934041924da141865dee44560bdb0b7f59..a05202e9d3cc43d6bd7c0512ae38816998dd3fb1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8364,11 +8364,10 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
 }
 
 bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
-                                      SDValue &Chain,
-                                      SelectionDAG &DAG) const {
+                                      SDValue &Chain, SelectionDAG &DAG) const {
   // This transform is not correct for converting 0 when rounding mode is set
-  // to round toward negative infinity which will produce -0.0. So disable under
-  // strictfp.
+  // to round toward negative infinity which will produce -0.0. So disable
+  // under strictfp.
   if (Node->isStrictFPOpcode())
     return false;
 
@@ -8376,10 +8375,20 @@ bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
   EVT SrcVT = Src.getValueType();
   EVT DstVT = Node->getValueType(0);
 
+  // If the input is known to be non-negative and SINT_TO_FP is legal then use
+  // it.
+  if (Node->getFlags().hasNonNeg() &&
+      isOperationLegalOrCustom(ISD::SINT_TO_FP, SrcVT)) {
+    Result =
+        DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), DstVT, Node->getOperand(0));
+    return true;
+  }
+
   if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
     return false;
 
-  // Only expand vector types if we have the appropriate vector bit operations.
+  // Only expand vector types if we have the appropriate vector bit
+  // operations.
   if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
                            !isOperationLegalOrCustom(ISD::FADD, DstVT) ||
                            !isOperationLegalOrCustom(ISD::FSUB, DstVT) ||
@@ -8393,8 +8402,9 @@ bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
   // Implementation of unsigned i64 to f64 following the algorithm in
   // __floatundidf in compiler_rt.  This implementation performs rounding
   // correctly in all rounding modes with the exception of converting 0
-  // when rounding toward negative infinity. In that case the fsub will produce
-  // -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.
+  // when rounding toward negative infinity. In that case the fsub will
+  // produce -0.0. This will be added to +0.0 and produce -0.0 which is
+  // incorrect.
   SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
   SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
       llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), dl, DstVT);
@@ -8408,8 +8418,7 @@ bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
   SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
   SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
   SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
-  SDValue HiSub =
-      DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
+  SDValue HiSub = DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
   Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
   return true;
 }
diff --git a/llvm/lib/CodeGen/SjLjEHPrepare.cpp b/llvm/lib/CodeGen/SjLjEHPrepare.cpp
index 054f7d7215962e8bccf9ec4cf1cc254b0d1570a1..c4ad9f0b2172fc48767baaff3f969c634491b2c3 100644
--- a/llvm/lib/CodeGen/SjLjEHPrepare.cpp
+++ b/llvm/lib/CodeGen/SjLjEHPrepare.cpp
@@ -508,17 +508,19 @@ bool SjLjEHPrepareImpl::runOnFunction(Function &F) {
 
   PointerType *AllocaPtrTy = M.getDataLayout().getAllocaPtrType(M.getContext());
 
-  FrameAddrFn =
-      Intrinsic::getDeclaration(&M, Intrinsic::frameaddress, {AllocaPtrTy});
-  StackAddrFn =
-      Intrinsic::getDeclaration(&M, Intrinsic::stacksave, {AllocaPtrTy});
-  StackRestoreFn =
-      Intrinsic::getDeclaration(&M, Intrinsic::stackrestore, {AllocaPtrTy});
+  FrameAddrFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::frameaddress,
+                                                  {AllocaPtrTy});
+  StackAddrFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::stacksave,
+                                                  {AllocaPtrTy});
+  StackRestoreFn = Intrinsic::getOrInsertDeclaration(
+      &M, Intrinsic::stackrestore, {AllocaPtrTy});
   BuiltinSetupDispatchFn =
-    Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_setup_dispatch);
-  LSDAAddrFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_lsda);
-  CallSiteFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_callsite);
-  FuncCtxFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_functioncontext);
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_setup_dispatch);
+  LSDAAddrFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_lsda);
+  CallSiteFn =
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_callsite);
+  FuncCtxFn =
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::eh_sjlj_functioncontext);
 
   bool Res = setupEntryBlockAndCallSites(F);
   return Res;
diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp
index 341ec629bedd95f83df48990e1badf223f4a1d4c..0be31d5db11ae2b603affd69493e85556351e2f1 100644
--- a/llvm/lib/CodeGen/StackColoring.cpp
+++ b/llvm/lib/CodeGen/StackColoring.cpp
@@ -20,6 +20,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/StackColoring.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DepthFirstIterator.h"
@@ -376,7 +377,7 @@ namespace {
 
 /// StackColoring - A machine pass for merging disjoint stack allocations,
 /// marked by the LIFETIME_START and LIFETIME_END pseudo instructions.
-class StackColoring : public MachineFunctionPass {
+class StackColoring {
   MachineFrameInfo *MFI = nullptr;
   MachineFunction *MF = nullptr;
 
@@ -436,14 +437,8 @@ class StackColoring : public MachineFunctionPass {
   unsigned NumIterations;
 
 public:
-  static char ID;
-
-  StackColoring() : MachineFunctionPass(ID) {
-    initializeStackColoringPass(*PassRegistry::getPassRegistry());
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-  bool runOnMachineFunction(MachineFunction &Func) override;
+  StackColoring(SlotIndexes *Indexes) : Indexes(Indexes) {}
+  bool run(MachineFunction &Func);
 
 private:
   /// Used in collectMarkers
@@ -509,19 +504,29 @@ private:
   void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
 };
 
+class StackColoringLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  StackColoringLegacy() : MachineFunctionPass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnMachineFunction(MachineFunction &Func) override;
+};
+
 } // end anonymous namespace
 
-char StackColoring::ID = 0;
+char StackColoringLegacy::ID = 0;
 
-char &llvm::StackColoringID = StackColoring::ID;
+char &llvm::StackColoringLegacyID = StackColoringLegacy::ID;
 
-INITIALIZE_PASS_BEGIN(StackColoring, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(StackColoringLegacy, DEBUG_TYPE,
                       "Merge disjoint stack slots", false, false)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
-INITIALIZE_PASS_END(StackColoring, DEBUG_TYPE,
+INITIALIZE_PASS_END(StackColoringLegacy, DEBUG_TYPE,
                     "Merge disjoint stack slots", false, false)
 
-void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
+void StackColoringLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<SlotIndexesWrapperPass>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
@@ -1178,12 +1183,30 @@ void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
   }
 }
 
-bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
+bool StackColoringLegacy::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  StackColoring SC(&getAnalysis<SlotIndexesWrapperPass>().getSI());
+  return SC.run(MF);
+}
+
+PreservedAnalyses StackColoringPass::run(MachineFunction &MF,
+                                         MachineFunctionAnalysisManager &MFAM) {
+  if (MF.getFunction().hasOptNone())
+    return PreservedAnalyses::all();
+
+  StackColoring SC(&MFAM.getResult<SlotIndexesAnalysis>(MF));
+  if (SC.run(MF))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
+
+bool StackColoring::run(MachineFunction &Func) {
   LLVM_DEBUG(dbgs() << "********** Stack Coloring **********\n"
                     << "********** Function: " << Func.getName() << '\n');
   MF = &Func;
   MFI = &MF->getFrameInfo();
-  Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
   BlockLiveness.clear();
   BasicBlocks.clear();
   BasicBlockNumbering.clear();
@@ -1220,8 +1243,7 @@ bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
 
   // Don't continue because there are not enough lifetime markers, or the
   // stack is too small, or we are told not to optimize the slots.
-  if (NumMarkers < 2 || TotalSize < 16 || DisableColoring ||
-      skipFunction(Func.getFunction())) {
+  if (NumMarkers < 2 || TotalSize < 16 || DisableColoring) {
     LLVM_DEBUG(dbgs() << "Will not try to merge slots.\n");
     return removeAllMarkers();
   }
diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp
index 1f23838b2de0ca0375196174e85de3ae33dca38f..a192161bbd9481dc207e1ab9e548886e3958aac1 100644
--- a/llvm/lib/CodeGen/StackProtector.cpp
+++ b/llvm/lib/CodeGen/StackProtector.cpp
@@ -519,7 +519,8 @@ static Value *getStackGuard(const TargetLoweringBase *TLI, Module *M,
   if (SupportsSelectionDAGSP)
     *SupportsSelectionDAGSP = true;
   TLI->insertSSPDeclarations(*M);
-  return B.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::stackguard));
+  return B.CreateCall(
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::stackguard));
 }
 
 /// Insert code into the entry block that stores the stack guard
@@ -540,7 +541,7 @@ static bool CreatePrologue(Function *F, Module *M, Instruction *CheckLoc,
   AI = B.CreateAlloca(PtrTy, nullptr, "StackGuardSlot");
 
   Value *GuardSlot = getStackGuard(TLI, M, B, &SupportsSelectionDAGSP);
-  B.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::stackprotector),
+  B.CreateCall(Intrinsic::getOrInsertDeclaration(M, Intrinsic::stackprotector),
                {GuardSlot, AI});
   return SupportsSelectionDAGSP;
 }
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 0d3e4ba5662e015d01b043bbbba1fe8f859a20c9..ce50a3c19ffe0472f05fe80dabecb01bafc117ae 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -2171,7 +2171,11 @@ MCSection *TargetLoweringObjectFileWasm::getExplicitSectionGlobal(
   // This could be avoided if all data segements (the wasm sense) were
   // represented as their own sections (in the llvm sense).
   // TODO(sbc): https://github.com/WebAssembly/tool-conventions/issues/138
-  if (Name == ".llvmcmd" || Name == ".llvmbc")
+  if (Name == getInstrProfSectionName(IPSK_covmap, Triple::Wasm,
+                                      /*AddSegmentInfo=*/false) ||
+      Name == getInstrProfSectionName(IPSK_covfun, Triple::Wasm,
+                                      /*AddSegmentInfo=*/false) ||
+      Name == ".llvmbc" || Name == ".llvmcmd")
     Kind = SectionKind::getMetadata();
 
   StringRef Group = "";
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 11a7752ef7a3811af79834cf5f222c01b9031f29..cf9d63df2515ccc52e9a847889c833f69f4435b7 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -1287,7 +1287,7 @@ void TargetPassConfig::addMachineSSAOptimization() {
 
   // This pass merges large allocas. StackSlotColoring is a different pass
   // which merges spill slots.
-  addPass(&StackColoringID);
+  addPass(&StackColoringLegacyID);
 
   // If the target requests it, assign local variables to stack slots relative
   // to one another and simplify frame index references where possible.
diff --git a/llvm/lib/CodeGen/WasmEHPrepare.cpp b/llvm/lib/CodeGen/WasmEHPrepare.cpp
index 7514d49fb18a98a48fc49872d8a878328eeb8d91..1701b0d04425d2e6f0ff1e4f9bd79768bd6716b5 100644
--- a/llvm/lib/CodeGen/WasmEHPrepare.cpp
+++ b/llvm/lib/CodeGen/WasmEHPrepare.cpp
@@ -196,7 +196,7 @@ bool WasmEHPrepareImpl::prepareThrows(Function &F) {
   bool Changed = false;
 
   // wasm.throw() intinsic, which will be lowered to wasm 'throw' instruction.
-  ThrowF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_throw);
+  ThrowF = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_throw);
   // Insert an unreachable instruction after a call to @llvm.wasm.throw and
   // delete all following instructions within the BB, and delete all the dead
   // children of the BB as well.
@@ -260,18 +260,21 @@ bool WasmEHPrepareImpl::prepareEHPads(Function &F) {
                                                  0, 2, "selector_gep");
 
   // wasm.landingpad.index() intrinsic, which is to specify landingpad index
-  LPadIndexF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_landingpad_index);
+  LPadIndexF =
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_landingpad_index);
   // wasm.lsda() intrinsic. Returns the address of LSDA table for the current
   // function.
-  LSDAF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_lsda);
+  LSDAF = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_lsda);
   // wasm.get.exception() and wasm.get.ehselector() intrinsics. Calls to these
   // are generated in clang.
-  GetExnF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_exception);
-  GetSelectorF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_ehselector);
+  GetExnF =
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_get_exception);
+  GetSelectorF =
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_get_ehselector);
 
   // wasm.catch() will be lowered down to wasm 'catch' instruction in
   // instruction selection.
-  CatchF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_catch);
+  CatchF = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_catch);
 
   // _Unwind_CallPersonality() wrapper function, which calls the personality
   CallPersonalityF = M.getOrInsertFunction("_Unwind_CallPersonality",
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
index e89664d360a9a23c8634829f8203d4eb72bafb3e..a15f9cd85976793b0c45780e7ea08a7a8aba1593 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
@@ -173,10 +173,11 @@ class LVForwardReferences {
 
   // Update a previously recorded forward reference with its definition.
   void update(StringRef Name, TypeIndex TIReference) {
-    if (ForwardTypesNames.find(Name) != ForwardTypesNames.end()) {
+    auto It = ForwardTypesNames.find(Name);
+    if (It != ForwardTypesNames.end()) {
       // Update the recorded forward reference with its definition.
-      ForwardTypesNames[Name].second = TIReference;
-      add(ForwardTypesNames[Name].first, TIReference);
+      It->second.second = TIReference;
+      add(It->second.first, TIReference);
     } else {
       // We have not seen the forward reference. Insert the definition.
       ForwardTypesNames.emplace(
@@ -196,15 +197,14 @@ public:
   }
 
   TypeIndex find(TypeIndex TIForward) {
-    return (ForwardTypes.find(TIForward) != ForwardTypes.end())
-               ? ForwardTypes[TIForward]
-               : TypeIndex::None();
+    auto It = ForwardTypes.find(TIForward);
+    return It != ForwardTypes.end() ? It->second : TypeIndex::None();
   }
 
   TypeIndex find(StringRef Name) {
-    return (ForwardTypesNames.find(Name) != ForwardTypesNames.end())
-               ? ForwardTypesNames[Name].second
-               : TypeIndex::None();
+    auto It = ForwardTypesNames.find(Name);
+    return It != ForwardTypesNames.end() ? It->second.second
+                                         : TypeIndex::None();
   }
 
   // If the given TI corresponds to a reference, return the reference.
@@ -242,9 +242,8 @@ public:
 
   // Find the logical namespace for the 'Name' component.
   LVScope *find(StringRef Name) {
-    LVScope *Namespace = (NamespaceNames.find(Name) != NamespaceNames.end())
-                             ? NamespaceNames[Name]
-                             : nullptr;
+    auto It = NamespaceNames.find(Name);
+    LVScope *Namespace = It != NamespaceNames.end() ? It->second : nullptr;
     return Namespace;
   }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
index c11577b03fd7e9235a585ef343d6b4e1331e6db8..f0cf551ffae5ab769fda1e20fbd9a57552353df6 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
@@ -136,12 +136,10 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) {
       // Otherwise check if we previously had exactly one relocation at this
       // offset. If so, we now have a second one and move it from the TargetMap
       // into the Multiple set.
-      auto It = BlockEdges.TargetMap.find(E.getOffset());
-      if (It != BlockEdges.TargetMap.end()) {
+      auto [It, Inserted] = BlockEdges.TargetMap.try_emplace(E.getOffset(), E);
+      if (!Inserted) {
         BlockEdges.TargetMap.erase(It);
         BlockEdges.Multiple.insert(E.getOffset());
-      } else {
-        BlockEdges.TargetMap[E.getOffset()] = EdgeTarget(E);
       }
     }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index 5dfd621781e446334eb731824461a883733f04a9..008875118fdeff87a038d0008c8decfcab733a6b 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -26,6 +26,7 @@ add_llvm_component_library(LLVMOrcJIT
   IndirectionUtils.cpp
   IRCompileLayer.cpp
   IRTransformLayer.cpp
+  IRPartitionLayer.cpp
   JITTargetMachineBuilder.cpp
   LazyReexports.cpp
   Layer.cpp
@@ -50,6 +51,9 @@ add_llvm_component_library(LLVMOrcJIT
   ExecutorProcessControl.cpp
   TaskDispatch.cpp
   ThreadSafeModule.cpp
+  RedirectionManager.cpp
+  JITLinkRedirectableSymbolManager.cpp
+  ReOptimizeLayer.cpp
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc
 
diff --git a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
index 6448adaa0ceb36f18b7fa6b208599616635f6e19..9296bc2b389ab249078b378be0171dbdf007b72f 100644
--- a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
@@ -9,6 +9,7 @@
 #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/Layer.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -17,101 +18,6 @@
 using namespace llvm;
 using namespace llvm::orc;
 
-static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM,
-                                         StringRef Suffix,
-                                         GVPredicate ShouldExtract) {
-
-  auto DeleteExtractedDefs = [](GlobalValue &GV) {
-    // Bump the linkage: this global will be provided by the external module.
-    GV.setLinkage(GlobalValue::ExternalLinkage);
-
-    // Delete the definition in the source module.
-    if (isa<Function>(GV)) {
-      auto &F = cast<Function>(GV);
-      F.deleteBody();
-      F.setPersonalityFn(nullptr);
-    } else if (isa<GlobalVariable>(GV)) {
-      cast<GlobalVariable>(GV).setInitializer(nullptr);
-    } else if (isa<GlobalAlias>(GV)) {
-      // We need to turn deleted aliases into function or variable decls based
-      // on the type of their aliasee.
-      auto &A = cast<GlobalAlias>(GV);
-      Constant *Aliasee = A.getAliasee();
-      assert(A.hasName() && "Anonymous alias?");
-      assert(Aliasee->hasName() && "Anonymous aliasee");
-      std::string AliasName = std::string(A.getName());
-
-      if (isa<Function>(Aliasee)) {
-        auto *F = cloneFunctionDecl(*A.getParent(), *cast<Function>(Aliasee));
-        A.replaceAllUsesWith(F);
-        A.eraseFromParent();
-        F->setName(AliasName);
-      } else if (isa<GlobalVariable>(Aliasee)) {
-        auto *G = cloneGlobalVariableDecl(*A.getParent(),
-                                          *cast<GlobalVariable>(Aliasee));
-        A.replaceAllUsesWith(G);
-        A.eraseFromParent();
-        G->setName(AliasName);
-      } else
-        llvm_unreachable("Alias to unsupported type");
-    } else
-      llvm_unreachable("Unsupported global type");
-  };
-
-  auto NewTSM = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs);
-  NewTSM.withModuleDo([&](Module &M) {
-    M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str());
-  });
-
-  return NewTSM;
-}
-
-namespace llvm {
-namespace orc {
-
-class PartitioningIRMaterializationUnit : public IRMaterializationUnit {
-public:
-  PartitioningIRMaterializationUnit(ExecutionSession &ES,
-                                    const IRSymbolMapper::ManglingOptions &MO,
-                                    ThreadSafeModule TSM,
-                                    CompileOnDemandLayer &Parent)
-      : IRMaterializationUnit(ES, MO, std::move(TSM)), Parent(Parent) {}
-
-  PartitioningIRMaterializationUnit(
-      ThreadSafeModule TSM, Interface I,
-      SymbolNameToDefinitionMap SymbolToDefinition,
-      CompileOnDemandLayer &Parent)
-      : IRMaterializationUnit(std::move(TSM), std::move(I),
-                              std::move(SymbolToDefinition)),
-        Parent(Parent) {}
-
-private:
-  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
-    Parent.emitPartition(std::move(R), std::move(TSM),
-                         std::move(SymbolToDefinition));
-  }
-
-  void discard(const JITDylib &V, const SymbolStringPtr &Name) override {
-    // All original symbols were materialized by the CODLayer and should be
-    // final. The function bodies provided by M should never be overridden.
-    llvm_unreachable("Discard should never be called on an "
-                     "ExtractingIRMaterializationUnit");
-  }
-
-  mutable std::mutex SourceModuleMutex;
-  CompileOnDemandLayer &Parent;
-};
-
-std::optional<CompileOnDemandLayer::GlobalValueSet>
-CompileOnDemandLayer::compileRequested(GlobalValueSet Requested) {
-  return std::move(Requested);
-}
-
-std::optional<CompileOnDemandLayer::GlobalValueSet>
-CompileOnDemandLayer::compileWholeModule(GlobalValueSet Requested) {
-  return std::nullopt;
-}
-
 CompileOnDemandLayer::CompileOnDemandLayer(
     ExecutionSession &ES, IRLayer &BaseLayer, LazyCallThroughManager &LCTMgr,
     IndirectStubsManagerBuilder BuildIndirectStubsManager)
@@ -119,13 +25,10 @@ CompileOnDemandLayer::CompileOnDemandLayer(
       LCTMgr(LCTMgr),
       BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)) {}
 
-void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) {
-  this->Partition = std::move(Partition);
-}
-
 void CompileOnDemandLayer::setImplMap(ImplSymbolMap *Imp) {
   this->AliaseeImpls = Imp;
 }
+
 void CompileOnDemandLayer::emit(
     std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM) {
   assert(TSM && "Null module");
@@ -138,10 +41,6 @@ void CompileOnDemandLayer::emit(
 
   SymbolAliasMap NonCallables;
   SymbolAliasMap Callables;
-  TSM.withModuleDo([&](Module &M) {
-    // First, do some cleanup on the module:
-    cleanUpModule(M);
-  });
 
   for (auto &KV : R->getSymbols()) {
     auto &Name = KV.first;
@@ -152,11 +51,10 @@ void CompileOnDemandLayer::emit(
       NonCallables[Name] = SymbolAliasMapEntry(Name, Flags);
   }
 
-  // Create a partitioning materialization unit and lodge it with the
-  // implementation dylib.
+  // Lodge symbols with the implementation dylib.
   if (auto Err = PDR.getImplDylib().define(
-          std::make_unique<PartitioningIRMaterializationUnit>(
-              ES, *getManglingOptions(), std::move(TSM), *this))) {
+          std::make_unique<BasicIRLayerMaterializationUnit>(
+              BaseLayer, *getManglingOptions(), std::move(TSM)))) {
     ES.reportError(std::move(Err));
     R->failMaterialization();
     return;
@@ -210,173 +108,3 @@ CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) {
 
   return I->second;
 }
-
-void CompileOnDemandLayer::cleanUpModule(Module &M) {
-  for (auto &F : M.functions()) {
-    if (F.isDeclaration())
-      continue;
-
-    if (F.hasAvailableExternallyLinkage()) {
-      F.deleteBody();
-      F.setPersonalityFn(nullptr);
-      continue;
-    }
-  }
-}
-
-void CompileOnDemandLayer::expandPartition(GlobalValueSet &Partition) {
-  // Expands the partition to ensure the following rules hold:
-  // (1) If any alias is in the partition, its aliasee is also in the partition.
-  // (2) If any aliasee is in the partition, its aliases are also in the
-  //     partiton.
-  // (3) If any global variable is in the partition then all global variables
-  //     are in the partition.
-  assert(!Partition.empty() && "Unexpected empty partition");
-
-  const Module &M = *(*Partition.begin())->getParent();
-  bool ContainsGlobalVariables = false;
-  std::vector<const GlobalValue *> GVsToAdd;
-
-  for (const auto *GV : Partition)
-    if (isa<GlobalAlias>(GV))
-      GVsToAdd.push_back(
-          cast<GlobalValue>(cast<GlobalAlias>(GV)->getAliasee()));
-    else if (isa<GlobalVariable>(GV))
-      ContainsGlobalVariables = true;
-
-  for (auto &A : M.aliases())
-    if (Partition.count(cast<GlobalValue>(A.getAliasee())))
-      GVsToAdd.push_back(&A);
-
-  if (ContainsGlobalVariables)
-    for (auto &G : M.globals())
-      GVsToAdd.push_back(&G);
-
-  for (const auto *GV : GVsToAdd)
-    Partition.insert(GV);
-}
-
-void CompileOnDemandLayer::emitPartition(
-    std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM,
-    IRMaterializationUnit::SymbolNameToDefinitionMap Defs) {
-
-  // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the
-  //        extracted module key, extracted module, and source module key
-  //        together. This could be used, for example, to provide a specific
-  //        memory manager instance to the linking layer.
-
-  auto &ES = getExecutionSession();
-  GlobalValueSet RequestedGVs;
-  for (auto &Name : R->getRequestedSymbols()) {
-    if (Name == R->getInitializerSymbol())
-      TSM.withModuleDo([&](Module &M) {
-        for (auto &GV : getStaticInitGVs(M))
-          RequestedGVs.insert(&GV);
-      });
-    else {
-      assert(Defs.count(Name) && "No definition for symbol");
-      RequestedGVs.insert(Defs[Name]);
-    }
-  }
-
-  /// Perform partitioning with the context lock held, since the partition
-  /// function is allowed to access the globals to compute the partition.
-  auto GVsToExtract =
-      TSM.withModuleDo([&](Module &M) { return Partition(RequestedGVs); });
-
-  // Take a 'None' partition to mean the whole module (as opposed to an empty
-  // partition, which means "materialize nothing"). Emit the whole module
-  // unmodified to the base layer.
-  if (GVsToExtract == std::nullopt) {
-    Defs.clear();
-    BaseLayer.emit(std::move(R), std::move(TSM));
-    return;
-  }
-
-  // If the partition is empty, return the whole module to the symbol table.
-  if (GVsToExtract->empty()) {
-    if (auto Err =
-            R->replace(std::make_unique<PartitioningIRMaterializationUnit>(
-                std::move(TSM),
-                MaterializationUnit::Interface(R->getSymbols(),
-                                               R->getInitializerSymbol()),
-                std::move(Defs), *this))) {
-      getExecutionSession().reportError(std::move(Err));
-      R->failMaterialization();
-      return;
-    }
-    return;
-  }
-
-  // Ok -- we actually need to partition the symbols. Promote the symbol
-  // linkages/names, expand the partition to include any required symbols
-  // (i.e. symbols that can't be separated from our partition), and
-  // then extract the partition.
-  //
-  // FIXME: We apply this promotion once per partitioning. It's safe, but
-  // overkill.
-  auto ExtractedTSM =
-      TSM.withModuleDo([&](Module &M) -> Expected<ThreadSafeModule> {
-        auto PromotedGlobals = PromoteSymbols(M);
-        if (!PromotedGlobals.empty()) {
-
-          MangleAndInterner Mangle(ES, M.getDataLayout());
-          SymbolFlagsMap SymbolFlags;
-          IRSymbolMapper::add(ES, *getManglingOptions(),
-                              PromotedGlobals, SymbolFlags);
-
-          if (auto Err = R->defineMaterializing(SymbolFlags))
-            return std::move(Err);
-        }
-
-        expandPartition(*GVsToExtract);
-
-        // Submodule name is given by hashing the names of the globals.
-        std::string SubModuleName;
-        {
-          std::vector<const GlobalValue*> HashGVs;
-          HashGVs.reserve(GVsToExtract->size());
-          for (const auto *GV : *GVsToExtract)
-            HashGVs.push_back(GV);
-          llvm::sort(HashGVs, [](const GlobalValue *LHS, const GlobalValue *RHS) {
-              return LHS->getName() < RHS->getName();
-            });
-          hash_code HC(0);
-          for (const auto *GV : HashGVs) {
-            assert(GV->hasName() && "All GVs to extract should be named by now");
-            auto GVName = GV->getName();
-            HC = hash_combine(HC, hash_combine_range(GVName.begin(), GVName.end()));
-          }
-          raw_string_ostream(SubModuleName)
-            << ".submodule."
-            << formatv(sizeof(size_t) == 8 ? "{0:x16}" : "{0:x8}",
-                       static_cast<size_t>(HC))
-            << ".ll";
-        }
-
-        // Extract the requested partiton (plus any necessary aliases) and
-        // put the rest back into the impl dylib.
-        auto ShouldExtract = [&](const GlobalValue &GV) -> bool {
-          return GVsToExtract->count(&GV);
-        };
-
-        return extractSubModule(TSM, SubModuleName , ShouldExtract);
-      });
-
-  if (!ExtractedTSM) {
-    ES.reportError(ExtractedTSM.takeError());
-    R->failMaterialization();
-    return;
-  }
-
-  if (auto Err = R->replace(std::make_unique<PartitioningIRMaterializationUnit>(
-          ES, *getManglingOptions(), std::move(TSM), *this))) {
-    ES.reportError(std::move(Err));
-    R->failMaterialization();
-    return;
-  }
-  BaseLayer.emit(std::move(R), std::move(*ExtractedTSM));
-}
-
-} // end namespace orc
-} // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/Orc/IRPartitionLayer.cpp b/llvm/lib/ExecutionEngine/Orc/IRPartitionLayer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9ad171beac7fe2e123327a29b48a443c4fb3b15a
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/IRPartitionLayer.cpp
@@ -0,0 +1,303 @@
+//===----- IRPartitionLayer.cpp - Partition IR module into submodules -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/IRPartitionLayer.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+
+static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM,
+                                         StringRef Suffix,
+                                         GVPredicate ShouldExtract) {
+
+  auto DeleteExtractedDefs = [](GlobalValue &GV) {
+    // Bump the linkage: this global will be provided by the external module.
+    GV.setLinkage(GlobalValue::ExternalLinkage);
+
+    // Delete the definition in the source module.
+    if (isa<Function>(GV)) {
+      auto &F = cast<Function>(GV);
+      F.deleteBody();
+      F.setPersonalityFn(nullptr);
+    } else if (isa<GlobalVariable>(GV)) {
+      cast<GlobalVariable>(GV).setInitializer(nullptr);
+    } else if (isa<GlobalAlias>(GV)) {
+      // We need to turn deleted aliases into function or variable decls based
+      // on the type of their aliasee.
+      auto &A = cast<GlobalAlias>(GV);
+      Constant *Aliasee = A.getAliasee();
+      assert(A.hasName() && "Anonymous alias?");
+      assert(Aliasee->hasName() && "Anonymous aliasee");
+      std::string AliasName = std::string(A.getName());
+
+      if (isa<Function>(Aliasee)) {
+        auto *F = cloneFunctionDecl(*A.getParent(), *cast<Function>(Aliasee));
+        A.replaceAllUsesWith(F);
+        A.eraseFromParent();
+        F->setName(AliasName);
+      } else if (isa<GlobalVariable>(Aliasee)) {
+        auto *G = cloneGlobalVariableDecl(*A.getParent(),
+                                          *cast<GlobalVariable>(Aliasee));
+        A.replaceAllUsesWith(G);
+        A.eraseFromParent();
+        G->setName(AliasName);
+      } else
+        llvm_unreachable("Alias to unsupported type");
+    } else
+      llvm_unreachable("Unsupported global type");
+  };
+
+  auto NewTSM = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs);
+  NewTSM.withModuleDo([&](Module &M) {
+    M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str());
+  });
+
+  return NewTSM;
+}
+
+namespace llvm {
+namespace orc {
+
+class PartitioningIRMaterializationUnit : public IRMaterializationUnit {
+public:
+  PartitioningIRMaterializationUnit(ExecutionSession &ES,
+                                    const IRSymbolMapper::ManglingOptions &MO,
+                                    ThreadSafeModule TSM,
+                                    IRPartitionLayer &Parent)
+      : IRMaterializationUnit(ES, MO, std::move(TSM)), Parent(Parent) {}
+
+  PartitioningIRMaterializationUnit(
+      ThreadSafeModule TSM, Interface I,
+      SymbolNameToDefinitionMap SymbolToDefinition, IRPartitionLayer &Parent)
+      : IRMaterializationUnit(std::move(TSM), std::move(I),
+                              std::move(SymbolToDefinition)),
+        Parent(Parent) {}
+
+private:
+  void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
+    Parent.emitPartition(std::move(R), std::move(TSM),
+                         std::move(SymbolToDefinition));
+  }
+
+  void discard(const JITDylib &V, const SymbolStringPtr &Name) override {
+    // All original symbols were materialized by the CODLayer and should be
+    // final. The function bodies provided by M should never be overridden.
+    llvm_unreachable("Discard should never be called on an "
+                     "ExtractingIRMaterializationUnit");
+  }
+
+  IRPartitionLayer &Parent;
+};
+
+} // namespace orc
+} // namespace llvm
+
+IRPartitionLayer::IRPartitionLayer(ExecutionSession &ES, IRLayer &BaseLayer)
+    : IRLayer(ES, BaseLayer.getManglingOptions()), BaseLayer(BaseLayer) {}
+
+void IRPartitionLayer::setPartitionFunction(PartitionFunction Partition) {
+  this->Partition = Partition;
+}
+
+std::optional<IRPartitionLayer::GlobalValueSet>
+IRPartitionLayer::compileRequested(GlobalValueSet Requested) {
+  return std::move(Requested);
+}
+
+std::optional<IRPartitionLayer::GlobalValueSet>
+IRPartitionLayer::compileWholeModule(GlobalValueSet Requested) {
+  return std::nullopt;
+}
+
+void IRPartitionLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
+                            ThreadSafeModule TSM) {
+  assert(TSM && "Null module");
+
+  auto &ES = getExecutionSession();
+  TSM.withModuleDo([&](Module &M) {
+    // First, do some cleanup on the module:
+    cleanUpModule(M);
+  });
+
+  // Create a partitioning materialization unit and pass the responsibility.
+  if (auto Err = R->replace(std::make_unique<PartitioningIRMaterializationUnit>(
+          ES, *getManglingOptions(), std::move(TSM), *this))) {
+    ES.reportError(std::move(Err));
+    R->failMaterialization();
+    return;
+  }
+}
+
+void IRPartitionLayer::cleanUpModule(Module &M) {
+  for (auto &F : M.functions()) {
+    if (F.isDeclaration())
+      continue;
+
+    if (F.hasAvailableExternallyLinkage()) {
+      F.deleteBody();
+      F.setPersonalityFn(nullptr);
+      continue;
+    }
+  }
+}
+
+void IRPartitionLayer::expandPartition(GlobalValueSet &Partition) {
+  // Expands the partition to ensure the following rules hold:
+  // (1) If any alias is in the partition, its aliasee is also in the partition.
+  // (2) If any aliasee is in the partition, its aliases are also in the
+  //     partiton.
+  // (3) If any global variable is in the partition then all global variables
+  //     are in the partition.
+  assert(!Partition.empty() && "Unexpected empty partition");
+
+  const Module &M = *(*Partition.begin())->getParent();
+  bool ContainsGlobalVariables = false;
+  std::vector<const GlobalValue *> GVsToAdd;
+
+  for (const auto *GV : Partition)
+    if (isa<GlobalAlias>(GV))
+      GVsToAdd.push_back(
+          cast<GlobalValue>(cast<GlobalAlias>(GV)->getAliasee()));
+    else if (isa<GlobalVariable>(GV))
+      ContainsGlobalVariables = true;
+
+  for (auto &A : M.aliases())
+    if (Partition.count(cast<GlobalValue>(A.getAliasee())))
+      GVsToAdd.push_back(&A);
+
+  if (ContainsGlobalVariables)
+    for (auto &G : M.globals())
+      GVsToAdd.push_back(&G);
+
+  for (const auto *GV : GVsToAdd)
+    Partition.insert(GV);
+}
+
+void IRPartitionLayer::emitPartition(
+    std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM,
+    IRMaterializationUnit::SymbolNameToDefinitionMap Defs) {
+
+  // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the
+  //        extracted module key, extracted module, and source module key
+  //        together. This could be used, for example, to provide a specific
+  //        memory manager instance to the linking layer.
+
+  auto &ES = getExecutionSession();
+  GlobalValueSet RequestedGVs;
+  for (auto &Name : R->getRequestedSymbols()) {
+    if (Name == R->getInitializerSymbol())
+      TSM.withModuleDo([&](Module &M) {
+        for (auto &GV : getStaticInitGVs(M))
+          RequestedGVs.insert(&GV);
+      });
+    else {
+      assert(Defs.count(Name) && "No definition for symbol");
+      RequestedGVs.insert(Defs[Name]);
+    }
+  }
+
+  /// Perform partitioning with the context lock held, since the partition
+  /// function is allowed to access the globals to compute the partition.
+  auto GVsToExtract =
+      TSM.withModuleDo([&](Module &M) { return Partition(RequestedGVs); });
+
+  // Take a 'None' partition to mean the whole module (as opposed to an empty
+  // partition, which means "materialize nothing"). Emit the whole module
+  // unmodified to the base layer.
+  if (GVsToExtract == std::nullopt) {
+    Defs.clear();
+    BaseLayer.emit(std::move(R), std::move(TSM));
+    return;
+  }
+
+  // If the partition is empty, return the whole module to the symbol table.
+  if (GVsToExtract->empty()) {
+    if (auto Err =
+            R->replace(std::make_unique<PartitioningIRMaterializationUnit>(
+                std::move(TSM),
+                MaterializationUnit::Interface(R->getSymbols(),
+                                               R->getInitializerSymbol()),
+                std::move(Defs), *this))) {
+      getExecutionSession().reportError(std::move(Err));
+      R->failMaterialization();
+      return;
+    }
+    return;
+  }
+
+  // Ok -- we actually need to partition the symbols. Promote the symbol
+  // linkages/names, expand the partition to include any required symbols
+  // (i.e. symbols that can't be separated from our partition), and
+  // then extract the partition.
+  //
+  // FIXME: We apply this promotion once per partitioning. It's safe, but
+  // overkill.
+  auto ExtractedTSM = TSM.withModuleDo([&](Module &M)
+                                           -> Expected<ThreadSafeModule> {
+    auto PromotedGlobals = PromoteSymbols(M);
+    if (!PromotedGlobals.empty()) {
+
+      MangleAndInterner Mangle(ES, M.getDataLayout());
+      SymbolFlagsMap SymbolFlags;
+      IRSymbolMapper::add(ES, *getManglingOptions(), PromotedGlobals,
+                          SymbolFlags);
+
+      if (auto Err = R->defineMaterializing(SymbolFlags))
+        return std::move(Err);
+    }
+
+    expandPartition(*GVsToExtract);
+
+    // Submodule name is given by hashing the names of the globals.
+    std::string SubModuleName;
+    {
+      std::vector<const GlobalValue *> HashGVs;
+      HashGVs.reserve(GVsToExtract->size());
+      for (const auto *GV : *GVsToExtract)
+        HashGVs.push_back(GV);
+      llvm::sort(HashGVs, [](const GlobalValue *LHS, const GlobalValue *RHS) {
+        return LHS->getName() < RHS->getName();
+      });
+      hash_code HC(0);
+      for (const auto *GV : HashGVs) {
+        assert(GV->hasName() && "All GVs to extract should be named by now");
+        auto GVName = GV->getName();
+        HC = hash_combine(HC, hash_combine_range(GVName.begin(), GVName.end()));
+      }
+      raw_string_ostream(SubModuleName)
+          << ".submodule."
+          << formatv(sizeof(size_t) == 8 ? "{0:x16}" : "{0:x8}",
+                     static_cast<size_t>(HC))
+          << ".ll";
+    }
+
+    // Extract the requested partiton (plus any necessary aliases) and
+    // put the rest back into the impl dylib.
+    auto ShouldExtract = [&](const GlobalValue &GV) -> bool {
+      return GVsToExtract->count(&GV);
+    };
+
+    return extractSubModule(TSM, SubModuleName, ShouldExtract);
+  });
+
+  if (!ExtractedTSM) {
+    ES.reportError(ExtractedTSM.takeError());
+    R->failMaterialization();
+    return;
+  }
+
+  if (auto Err = R->replace(std::make_unique<PartitioningIRMaterializationUnit>(
+          ES, *getManglingOptions(), std::move(TSM), *this))) {
+    ES.reportError(std::move(Err));
+    R->failMaterialization();
+    return;
+  }
+  BaseLayer.emit(std::move(R), std::move(*ExtractedTSM));
+}
diff --git a/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4ef217e6c562db2330c0d9fa7328a372efd3c94e
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
@@ -0,0 +1,182 @@
+//===-- JITLinkRedirectableSymbolManager.cpp - JITLink redirection in Orc -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+
+#define DEBUG_TYPE "orc"
+
+using namespace llvm;
+using namespace llvm::orc;
+
+void JITLinkRedirectableSymbolManager::emitRedirectableSymbols(
+    std::unique_ptr<MaterializationResponsibility> R,
+    const SymbolAddrMap &InitialDests) {
+  auto &ES = ObjLinkingLayer.getExecutionSession();
+  std::unique_lock<std::mutex> Lock(Mutex);
+  if (GetNumAvailableStubs() < InitialDests.size())
+    if (auto Err = grow(InitialDests.size() - GetNumAvailableStubs())) {
+      ES.reportError(std::move(Err));
+      R->failMaterialization();
+      return;
+    }
+
+  JITDylib &TargetJD = R->getTargetJITDylib();
+  SymbolMap NewSymbolDefs;
+  std::vector<SymbolStringPtr> Symbols;
+  for (auto &[K, V] : InitialDests) {
+    StubHandle StubID = AvailableStubs.back();
+    if (SymbolToStubs[&TargetJD].count(K)) {
+      ES.reportError(make_error<StringError>(
+          "Tried to create duplicate redirectable symbols",
+          inconvertibleErrorCode()));
+      R->failMaterialization();
+      return;
+    }
+    SymbolToStubs[&TargetJD][K] = StubID;
+    NewSymbolDefs[K] = JumpStubs[StubID];
+    NewSymbolDefs[K].setFlags(V.getFlags());
+    Symbols.push_back(K);
+    AvailableStubs.pop_back();
+  }
+
+  // FIXME: when this fails we can return stubs to the pool
+  if (auto Err = redirectInner(TargetJD, InitialDests)) {
+    ES.reportError(std::move(Err));
+    R->failMaterialization();
+    return;
+  }
+
+  if (auto Err = R->replace(absoluteSymbols(NewSymbolDefs))) {
+    ES.reportError(std::move(Err));
+    R->failMaterialization();
+    return;
+  }
+
+  auto Err = R->withResourceKeyDo([&](ResourceKey Key) {
+    TrackedResources[Key].insert(TrackedResources[Key].end(), Symbols.begin(),
+                                 Symbols.end());
+  });
+  if (Err) {
+    ES.reportError(std::move(Err));
+    R->failMaterialization();
+    return;
+  }
+}
+
+Error JITLinkRedirectableSymbolManager::redirect(
+    JITDylib &TargetJD, const SymbolAddrMap &NewDests) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  return redirectInner(TargetJD, NewDests);
+}
+
+Error JITLinkRedirectableSymbolManager::redirectInner(
+    JITDylib &TargetJD, const SymbolAddrMap &NewDests) {
+  std::vector<tpctypes::PointerWrite> PtrWrites;
+  for (auto &[K, V] : NewDests) {
+    if (!SymbolToStubs[&TargetJD].count(K))
+      return make_error<StringError>(
+          "Tried to redirect non-existent redirectalbe symbol",
+          inconvertibleErrorCode());
+    StubHandle StubID = SymbolToStubs[&TargetJD].at(K);
+    PtrWrites.push_back({StubPointers[StubID].getAddress(), V.getAddress()});
+  }
+  return ObjLinkingLayer.getExecutionSession()
+      .getExecutorProcessControl()
+      .getMemoryAccess()
+      .writePointers(PtrWrites);
+}
+
+Error JITLinkRedirectableSymbolManager::grow(unsigned Need) {
+  unsigned OldSize = JumpStubs.size();
+  unsigned NumNewStubs = alignTo(Need, StubBlockSize);
+  unsigned NewSize = OldSize + NumNewStubs;
+
+  JumpStubs.resize(NewSize);
+  StubPointers.resize(NewSize);
+  AvailableStubs.reserve(NewSize);
+
+  SymbolLookupSet LookupSymbols;
+  DenseMap<SymbolStringPtr, ExecutorSymbolDef *> NewDefsMap;
+
+  auto &ES = ObjLinkingLayer.getExecutionSession();
+  Triple TT = ES.getTargetTriple();
+  auto G = std::make_unique<jitlink::LinkGraph>(
+      "<INDIRECT STUBS>", TT, TT.isArch64Bit() ? 8 : 4,
+      TT.isLittleEndian() ? endianness::little : endianness::big,
+      jitlink::getGenericEdgeKindName);
+  auto &PointerSection =
+      G->createSection(StubPtrTableName, MemProt::Write | MemProt::Read);
+  auto &StubsSection =
+      G->createSection(JumpStubTableName, MemProt::Exec | MemProt::Read);
+
+  // FIXME: We can batch the stubs into one block and use address to access them
+  for (size_t I = OldSize; I < NewSize; I++) {
+    auto Pointer = AnonymousPtrCreator(*G, PointerSection, nullptr, 0);
+    if (auto Err = Pointer.takeError())
+      return Err;
+
+    StringRef PtrSymName = StubPtrSymbolName(I);
+    Pointer->setName(PtrSymName);
+    Pointer->setScope(jitlink::Scope::Default);
+    LookupSymbols.add(ES.intern(PtrSymName));
+    NewDefsMap[ES.intern(PtrSymName)] = &StubPointers[I];
+
+    auto Stub = PtrJumpStubCreator(*G, StubsSection, *Pointer);
+    if (auto Err = Stub.takeError())
+      return Err;
+
+    StringRef JumpStubSymName = JumpStubSymbolName(I);
+    Stub->setName(JumpStubSymName);
+    Stub->setScope(jitlink::Scope::Default);
+    LookupSymbols.add(ES.intern(JumpStubSymName));
+    NewDefsMap[ES.intern(JumpStubSymName)] = &JumpStubs[I];
+  }
+
+  if (auto Err = ObjLinkingLayer.add(JD, std::move(G)))
+    return Err;
+
+  auto LookupResult = ES.lookup(makeJITDylibSearchOrder(&JD), LookupSymbols);
+  if (auto Err = LookupResult.takeError())
+    return Err;
+
+  for (auto &[K, V] : *LookupResult)
+    *NewDefsMap.at(K) = V;
+
+  for (size_t I = OldSize; I < NewSize; I++)
+    AvailableStubs.push_back(I);
+
+  return Error::success();
+}
+
+Error JITLinkRedirectableSymbolManager::handleRemoveResources(
+    JITDylib &TargetJD, ResourceKey K) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  for (auto &Symbol : TrackedResources[K]) {
+    if (!SymbolToStubs[&TargetJD].count(Symbol))
+      return make_error<StringError>(
+          "Tried to remove non-existent redirectable symbol",
+          inconvertibleErrorCode());
+    AvailableStubs.push_back(SymbolToStubs[&TargetJD].at(Symbol));
+    SymbolToStubs[&TargetJD].erase(Symbol);
+    if (SymbolToStubs[&TargetJD].empty())
+      SymbolToStubs.erase(&TargetJD);
+  }
+  TrackedResources.erase(K);
+
+  return Error::success();
+}
+
+void JITLinkRedirectableSymbolManager::handleTransferResources(
+    JITDylib &TargetJD, ResourceKey DstK, ResourceKey SrcK) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  TrackedResources[DstK].insert(TrackedResources[DstK].end(),
+                                TrackedResources[SrcK].begin(),
+                                TrackedResources[SrcK].end());
+  TrackedResources.erase(SrcK);
+}
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index d3dd3b6bedfb6569dab61ba6d0b26c75d3e441ee..d3db89a2c3e9f1062c100b9c35aa5e10b7d92224 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -1293,9 +1293,12 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
     return;
   }
 
+  // Create the IP Layer.
+  IPLayer = std::make_unique<IRPartitionLayer>(*ES, *InitHelperTransformLayer);
+
   // Create the COD layer.
-  CODLayer = std::make_unique<CompileOnDemandLayer>(
-      *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
+  CODLayer = std::make_unique<CompileOnDemandLayer>(*ES, *IPLayer, *LCTMgr,
+                                                    std::move(ISMBuilder));
 
   if (*S.SupportConcurrentCompilation)
     CODLayer->setCloneToNewContextOnEmit(true);
diff --git a/llvm/lib/ExecutionEngine/Orc/ReOptimizeLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ReOptimizeLayer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e2669fd1fc86b366926681cac2fea6f2c377bbfa
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/ReOptimizeLayer.cpp
@@ -0,0 +1,279 @@
+#include "llvm/ExecutionEngine/Orc/ReOptimizeLayer.h"
+#include "llvm/ExecutionEngine/Orc/Mangling.h"
+
+using namespace llvm;
+using namespace orc;
+
+bool ReOptimizeLayer::ReOptMaterializationUnitState::tryStartReoptimize() {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  if (Reoptimizing)
+    return false;
+
+  Reoptimizing = true;
+  return true;
+}
+
+void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeSucceeded() {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");
+  Reoptimizing = false;
+  CurVersion++;
+}
+
+void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeFailed() {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");
+  Reoptimizing = false;
+}
+
+Error ReOptimizeLayer::reigsterRuntimeFunctions(JITDylib &PlatformJD) {
+  ExecutionSession::JITDispatchHandlerAssociationMap WFs;
+  using ReoptimizeSPSSig = shared::SPSError(uint64_t, uint32_t);
+  WFs[Mangle("__orc_rt_reoptimize_tag")] =
+      ES.wrapAsyncWithSPS<ReoptimizeSPSSig>(this,
+                                            &ReOptimizeLayer::rt_reoptimize);
+  return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
+}
+
+void ReOptimizeLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
+                           ThreadSafeModule TSM) {
+  auto &JD = R->getTargetJITDylib();
+
+  bool HasNonCallable = false;
+  for (auto &KV : R->getSymbols()) {
+    auto &Flags = KV.second;
+    if (!Flags.isCallable())
+      HasNonCallable = true;
+  }
+
+  if (HasNonCallable) {
+    BaseLayer.emit(std::move(R), std::move(TSM));
+    return;
+  }
+
+  auto &MUState = createMaterializationUnitState(TSM);
+
+  if (auto Err = R->withResourceKeyDo([&](ResourceKey Key) {
+        registerMaterializationUnitResource(Key, MUState);
+      })) {
+    ES.reportError(std::move(Err));
+    R->failMaterialization();
+    return;
+  }
+
+  if (auto Err =
+          ProfilerFunc(*this, MUState.getID(), MUState.getCurVersion(), TSM)) {
+    ES.reportError(std::move(Err));
+    R->failMaterialization();
+    return;
+  }
+
+  auto InitialDests =
+      emitMUImplSymbols(MUState, MUState.getCurVersion(), JD, std::move(TSM));
+  if (!InitialDests) {
+    ES.reportError(InitialDests.takeError());
+    R->failMaterialization();
+    return;
+  }
+
+  RSManager.emitRedirectableSymbols(std::move(R), std::move(*InitialDests));
+}
+
+Error ReOptimizeLayer::reoptimizeIfCallFrequent(ReOptimizeLayer &Parent,
+                                                ReOptMaterializationUnitID MUID,
+                                                unsigned CurVersion,
+                                                ThreadSafeModule &TSM) {
+  return TSM.withModuleDo([&](Module &M) -> Error {
+    Type *I64Ty = Type::getInt64Ty(M.getContext());
+    GlobalVariable *Counter = new GlobalVariable(
+        M, I64Ty, false, GlobalValue::InternalLinkage,
+        Constant::getNullValue(I64Ty), "__orc_reopt_counter");
+    auto ArgBufferConst = createReoptimizeArgBuffer(M, MUID, CurVersion);
+    if (auto Err = ArgBufferConst.takeError())
+      return Err;
+    GlobalVariable *ArgBuffer =
+        new GlobalVariable(M, (*ArgBufferConst)->getType(), true,
+                           GlobalValue::InternalLinkage, (*ArgBufferConst));
+    for (auto &F : M) {
+      if (F.isDeclaration())
+        continue;
+      auto &BB = F.getEntryBlock();
+      auto *IP = &*BB.getFirstInsertionPt();
+      IRBuilder<> IRB(IP);
+      Value *Threshold = ConstantInt::get(I64Ty, CallCountThreshold, true);
+      Value *Cnt = IRB.CreateLoad(I64Ty, Counter);
+      // Use EQ to prevent further reoptimize calls.
+      Value *Cmp = IRB.CreateICmpEQ(Cnt, Threshold);
+      Value *Added = IRB.CreateAdd(Cnt, ConstantInt::get(I64Ty, 1));
+      (void)IRB.CreateStore(Added, Counter);
+      Instruction *SplitTerminator = SplitBlockAndInsertIfThen(Cmp, IP, false);
+      createReoptimizeCall(M, *SplitTerminator, ArgBuffer);
+    }
+    return Error::success();
+  });
+}
+
+Expected<SymbolMap>
+ReOptimizeLayer::emitMUImplSymbols(ReOptMaterializationUnitState &MUState,
+                                   uint32_t Version, JITDylib &JD,
+                                   ThreadSafeModule TSM) {
+  DenseMap<SymbolStringPtr, SymbolStringPtr> RenamedMap;
+  cantFail(TSM.withModuleDo([&](Module &M) -> Error {
+    MangleAndInterner Mangle(ES, M.getDataLayout());
+    for (auto &F : M)
+      if (!F.isDeclaration()) {
+        std::string NewName =
+            (F.getName() + ".__def__." + Twine(Version)).str();
+        RenamedMap[Mangle(F.getName())] = Mangle(NewName);
+        F.setName(NewName);
+      }
+    return Error::success();
+  }));
+
+  auto RT = JD.createResourceTracker();
+  if (auto Err =
+          JD.define(std::make_unique<BasicIRLayerMaterializationUnit>(
+                        BaseLayer, *getManglingOptions(), std::move(TSM)),
+                    RT))
+    return Err;
+  MUState.setResourceTracker(RT);
+
+  SymbolLookupSet LookupSymbols;
+  for (auto [K, V] : RenamedMap)
+    LookupSymbols.add(V);
+
+  auto ImplSymbols =
+      ES.lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}}, LookupSymbols,
+                LookupKind::Static, SymbolState::Resolved);
+  if (auto Err = ImplSymbols.takeError())
+    return Err;
+
+  SymbolMap Result;
+  for (auto [K, V] : RenamedMap)
+    Result[K] = (*ImplSymbols)[V];
+
+  return Result;
+}
+
+void ReOptimizeLayer::rt_reoptimize(SendErrorFn SendResult,
+                                    ReOptMaterializationUnitID MUID,
+                                    uint32_t CurVersion) {
+  auto &MUState = getMaterializationUnitState(MUID);
+  if (CurVersion < MUState.getCurVersion() || !MUState.tryStartReoptimize()) {
+    SendResult(Error::success());
+    return;
+  }
+
+  ThreadSafeModule TSM = cloneToNewContext(MUState.getThreadSafeModule());
+  auto OldRT = MUState.getResourceTracker();
+  auto &JD = OldRT->getJITDylib();
+
+  if (auto Err = ReOptFunc(*this, MUID, CurVersion + 1, OldRT, TSM)) {
+    ES.reportError(std::move(Err));
+    MUState.reoptimizeFailed();
+    SendResult(Error::success());
+    return;
+  }
+
+  auto SymbolDests =
+      emitMUImplSymbols(MUState, CurVersion + 1, JD, std::move(TSM));
+  if (!SymbolDests) {
+    ES.reportError(SymbolDests.takeError());
+    MUState.reoptimizeFailed();
+    SendResult(Error::success());
+    return;
+  }
+
+  if (auto Err = RSManager.redirect(JD, std::move(*SymbolDests))) {
+    ES.reportError(std::move(Err));
+    MUState.reoptimizeFailed();
+    SendResult(Error::success());
+    return;
+  }
+
+  MUState.reoptimizeSucceeded();
+  SendResult(Error::success());
+}
+
+Expected<Constant *> ReOptimizeLayer::createReoptimizeArgBuffer(
+    Module &M, ReOptMaterializationUnitID MUID, uint32_t CurVersion) {
+  size_t ArgBufferSize = SPSReoptimizeArgList::size(MUID, CurVersion);
+  std::vector<char> ArgBuffer(ArgBufferSize);
+  shared::SPSOutputBuffer OB(ArgBuffer.data(), ArgBuffer.size());
+  if (!SPSReoptimizeArgList::serialize(OB, MUID, CurVersion))
+    return make_error<StringError>("Could not serealize args list",
+                                   inconvertibleErrorCode());
+  return ConstantDataArray::get(M.getContext(), ArrayRef(ArgBuffer));
+}
+
+void ReOptimizeLayer::createReoptimizeCall(Module &M, Instruction &IP,
+                                           GlobalVariable *ArgBuffer) {
+  GlobalVariable *DispatchCtx =
+      M.getGlobalVariable("__orc_rt_jit_dispatch_ctx");
+  if (!DispatchCtx)
+    DispatchCtx = new GlobalVariable(M, PointerType::get(M.getContext(), 0),
+                                     false, GlobalValue::ExternalLinkage,
+                                     nullptr, "__orc_rt_jit_dispatch_ctx");
+  GlobalVariable *ReoptimizeTag =
+      M.getGlobalVariable("__orc_rt_reoptimize_tag");
+  if (!ReoptimizeTag)
+    ReoptimizeTag = new GlobalVariable(M, PointerType::get(M.getContext(), 0),
+                                       false, GlobalValue::ExternalLinkage,
+                                       nullptr, "__orc_rt_reoptimize_tag");
+  Function *DispatchFunc = M.getFunction("__orc_rt_jit_dispatch");
+  if (!DispatchFunc) {
+    std::vector<Type *> Args = {PointerType::get(M.getContext(), 0),
+                                PointerType::get(M.getContext(), 0),
+                                PointerType::get(M.getContext(), 0),
+                                IntegerType::get(M.getContext(), 64)};
+    FunctionType *FuncTy =
+        FunctionType::get(Type::getVoidTy(M.getContext()), Args, false);
+    DispatchFunc = Function::Create(FuncTy, GlobalValue::ExternalLinkage,
+                                    "__orc_rt_jit_dispatch", &M);
+  }
+  size_t ArgBufferSizeConst =
+      SPSReoptimizeArgList::size(ReOptMaterializationUnitID{}, uint32_t{});
+  Constant *ArgBufferSize = ConstantInt::get(
+      IntegerType::get(M.getContext(), 64), ArgBufferSizeConst, false);
+  IRBuilder<> IRB(&IP);
+  (void)IRB.CreateCall(DispatchFunc,
+                       {DispatchCtx, ReoptimizeTag, ArgBuffer, ArgBufferSize});
+}
+
+ReOptimizeLayer::ReOptMaterializationUnitState &
+ReOptimizeLayer::createMaterializationUnitState(const ThreadSafeModule &TSM) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  ReOptMaterializationUnitID MUID = NextID;
+  MUStates.emplace(MUID,
+                   ReOptMaterializationUnitState(MUID, cloneToNewContext(TSM)));
+  ++NextID;
+  return MUStates.at(MUID);
+}
+
+ReOptimizeLayer::ReOptMaterializationUnitState &
+ReOptimizeLayer::getMaterializationUnitState(ReOptMaterializationUnitID MUID) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  return MUStates.at(MUID);
+}
+
+void ReOptimizeLayer::registerMaterializationUnitResource(
+    ResourceKey Key, ReOptMaterializationUnitState &State) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  MUResources[Key].insert(State.getID());
+}
+
+Error ReOptimizeLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  for (auto MUID : MUResources[K])
+    MUStates.erase(MUID);
+
+  MUResources.erase(K);
+  return Error::success();
+}
+
+void ReOptimizeLayer::handleTransferResources(JITDylib &JD, ResourceKey DstK,
+                                              ResourceKey SrcK) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  MUResources[DstK].insert(MUResources[SrcK].begin(), MUResources[SrcK].end());
+  MUResources.erase(SrcK);
+}
diff --git a/llvm/lib/ExecutionEngine/Orc/RedirectionManager.cpp b/llvm/lib/ExecutionEngine/Orc/RedirectionManager.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..cbc77c5034303aec40483619739fd7855ed0ddd4
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/RedirectionManager.cpp
@@ -0,0 +1,24 @@
+//===---- RedirectionManager.cpp - Redirection manager interface in Orc ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
+
+#define DEBUG_TYPE "orc"
+
+using namespace llvm;
+using namespace llvm::orc;
+
+void RedirectionManager::anchor() {}
+
+Error RedirectableSymbolManager::createRedirectableSymbols(
+    ResourceTrackerSP RT, const SymbolMap &InitialDests) {
+  auto &JD = RT->getJITDylib();
+  return JD.define(
+      std::make_unique<RedirectableMaterializationUnit>(*this, InitialDests),
+      RT);
+}
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 47cc6ff7655caf29306b8cd7b89fe9f4d3fc2a3c..172812a3802d33cd42b3d1106ab480447d88ed95 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6968,8 +6968,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask(
   }
 
   OI.ExitBB = Builder.saveIP().getBlock();
-  OI.PostOutlineCB = [this, ToBeDeleted, Dependencies,
-                      HasNoWait](Function &OutlinedFn) mutable {
+  OI.PostOutlineCB = [this, ToBeDeleted, Dependencies, HasNoWait,
+                      DeviceID](Function &OutlinedFn) mutable {
     assert(OutlinedFn.getNumUses() == 1 &&
            "there must be a single user for the outlined function");
 
@@ -6989,9 +6989,15 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask(
         getOrCreateSrcLocStr(LocationDescription(Builder), SrcLocStrSize);
     Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
 
-    // @__kmpc_omp_task_alloc
+    // @__kmpc_omp_task_alloc or @__kmpc_omp_target_task_alloc
+    //
+    // If `HasNoWait == true`, we call  @__kmpc_omp_target_task_alloc to provide
+    // the DeviceID to the deferred task and also since
+    // @__kmpc_omp_target_task_alloc creates an untied/async task.
     Function *TaskAllocFn =
-        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
+        !HasNoWait ? getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
+                   : getOrCreateRuntimeFunctionPtr(
+                         OMPRTL___kmpc_omp_target_task_alloc);
 
     // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
     // call.
@@ -7032,10 +7038,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask(
     // Emit the @__kmpc_omp_task_alloc runtime call
     // The runtime call returns a pointer to an area where the task captured
     // variables must be copied before the task is run (TaskData)
-    CallInst *TaskData = Builder.CreateCall(
-        TaskAllocFn, {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
-                      /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
-                      /*task_func=*/ProxyFn});
+    CallInst *TaskData = nullptr;
+
+    SmallVector<llvm::Value *> TaskAllocArgs = {
+        /*loc_ref=*/Ident,        /*gtid=*/ThreadID,
+        /*flags=*/Flags,
+        /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
+        /*task_func=*/ProxyFn};
+
+    if (HasNoWait)
+      TaskAllocArgs.push_back(DeviceID);
+
+    TaskData = Builder.CreateCall(TaskAllocFn, TaskAllocArgs);
 
     if (HasShareds) {
       Value *Shareds = StaleCI->getArgOperand(1);
@@ -7118,13 +7132,14 @@ void OpenMPIRBuilder::emitOffloadingArraysAndArgs(
   emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);
 }
 
-static void emitTargetCall(
-    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
-    OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn,
-    Constant *OutlinedFnID, ArrayRef<int32_t> NumTeams,
-    ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Args,
-    OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
-    SmallVector<llvm::OpenMPIRBuilder::DependData> Dependencies = {}) {
+static void
+emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
+               OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn,
+               Constant *OutlinedFnID, ArrayRef<int32_t> NumTeams,
+               ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Args,
+               OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
+               SmallVector<llvm::OpenMPIRBuilder::DependData> Dependencies = {},
+               bool HasNoWait = false) {
   // Generate a function call to the host fallback implementation of the target
   // region. This is called by the host when no offload entry was generated for
   // the target region and when the offloading call fails at runtime.
@@ -7135,7 +7150,6 @@ static void emitTargetCall(
     return Builder.saveIP();
   };
 
-  bool HasNoWait = false;
   bool HasDependencies = Dependencies.size() > 0;
   bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
 
@@ -7211,7 +7225,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
     SmallVectorImpl<Value *> &Args, GenMapInfoCallbackTy GenMapInfoCB,
     OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
     OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
-    SmallVector<DependData> Dependencies) {
+    SmallVector<DependData> Dependencies, bool HasNowait) {
 
   if (!updateToLocation(Loc))
     return InsertPointTy();
@@ -7232,7 +7246,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
   // that represents the target region. Do that now.
   if (!Config.isTargetDevice())
     emitTargetCall(*this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams,
-                   NumThreads, Args, GenMapInfoCB, Dependencies);
+                   NumThreads, Args, GenMapInfoCB, Dependencies, HasNowait);
   return Builder.saveIP();
 }
 
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 280e347739cdb60d12c1134942194a002ac6e780..6adb8d6a742255ec5b01bc9dab1f07d4181a5ae1 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -1433,6 +1433,9 @@ static void WriteOptimizationInfo(raw_ostream &Out, const User *U) {
       Out << " nuw";
     if (TI->hasNoSignedWrap())
       Out << " nsw";
+  } else if (const auto *ICmp = dyn_cast<ICmpInst>(U)) {
+    if (ICmp->hasSameSign())
+      Out << " samesign";
   }
 }
 
@@ -3609,7 +3612,7 @@ void AssemblyWriter::printSummary(const GlobalValueSummary &Summary) {
 
 void AssemblyWriter::printSummaryInfo(unsigned Slot, const ValueInfo &VI) {
   Out << "^" << Slot << " = gv: (";
-  if (!VI.name().empty())
+  if (VI.hasName() && !VI.name().empty())
     Out << "name: \"" << VI.name() << "\"";
   else
     Out << "guid: " << VI.getGUID();
@@ -3623,7 +3626,7 @@ void AssemblyWriter::printSummaryInfo(unsigned Slot, const ValueInfo &VI) {
     Out << ")";
   }
   Out << ")";
-  if (!VI.name().empty())
+  if (VI.hasName() && !VI.name().empty())
     Out << " ; guid = " << VI.getGUID();
   Out << "\n";
 }
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 215bfc8c6cfe3e65b2279c24081645495eb40bb3..477b77a6dd5335a618275876e14f323f880e44f9 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -65,7 +65,7 @@ static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
 
   // Yes, it's old, replace it with new version.
   rename(F);
-  NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+  NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
   return true;
 }
 
@@ -81,7 +81,7 @@ static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
 
   // Move this function aside and map down.
   rename(F);
-  NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+  NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
   return true;
 }
 
@@ -94,7 +94,7 @@ static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
     return false;
 
   rename(F);
-  NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+  NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
   return true;
 }
 
@@ -104,7 +104,7 @@ static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
     return false;
 
   rename(F);
-  NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+  NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
   return true;
 }
 
@@ -114,7 +114,7 @@ static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
     return false;
 
   rename(F);
-  NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+  NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
   return true;
 }
 
@@ -502,8 +502,8 @@ static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
       return false;
 
     rename(F);
-    NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                      Intrinsic::x86_rdtscp);
+    NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                              Intrinsic::x86_rdtscp);
     return true;
   }
 
@@ -609,14 +609,15 @@ static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
 
     if (ID != Intrinsic::not_intrinsic) {
       rename(F);
-      NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+      NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
       return true;
     }
     return false; // No other 'x86.xop.*'
   }
 
   if (Name == "seh.recoverfp") {
-    NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
+    NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                              Intrinsic::eh_recoverfp);
     return true;
   }
 
@@ -630,15 +631,15 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
                                                  Function *&NewFn) {
   if (Name.starts_with("rbit")) {
     // '(arm|aarch64).rbit'.
-    NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
-                                      F->arg_begin()->getType());
+    NewFn = Intrinsic::getOrInsertDeclaration(
+        F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
     return true;
   }
 
   if (Name == "thread.pointer") {
     // '(arm|aarch64).thread.pointer'.
-    NewFn =
-        Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
+    NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                              Intrinsic::thread_pointer);
     return true;
   }
 
@@ -663,7 +664,7 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
         std::array<Type *, 2> Tys{
             {F->getReturnType(),
              FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
-        NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
         return true;
       }
       return false; // No other '(arm|aarch64).neon.bfdot.*'.
@@ -688,7 +689,7 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
                             : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
                 .Default(Intrinsic::not_intrinsic);
         if (ID != Intrinsic::not_intrinsic) {
-          NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
           return true;
         }
         return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
@@ -712,8 +713,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
                              .StartsWith("vqsubu.", Intrinsic::usub_sat)
                              .Default(Intrinsic::not_intrinsic);
       if (ID != Intrinsic::not_intrinsic) {
-        NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
-                                          F->arg_begin()->getType());
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
+                                                  F->arg_begin()->getType());
         return true;
       }
 
@@ -733,10 +734,10 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
           auto fArgs = F->getFunctionType()->params();
           Type *Tys[] = {fArgs[0], fArgs[1]};
           if (Groups[1].size() == 1)
-            NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                              StoreInts[fArgs.size() - 3], Tys);
+            NewFn = Intrinsic::getOrInsertDeclaration(
+                F->getParent(), StoreInts[fArgs.size() - 3], Tys);
           else
-            NewFn = Intrinsic::getDeclaration(
+            NewFn = Intrinsic::getOrInsertDeclaration(
                 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
           return true;
         }
@@ -810,8 +811,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
                              .StartsWith("rbit", Intrinsic::bitreverse)
                              .Default(Intrinsic::not_intrinsic);
       if (ID != Intrinsic::not_intrinsic) {
-        NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
-                                          F->arg_begin()->getType());
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
+                                                  F->arg_begin()->getType());
         return true;
       }
 
@@ -821,8 +822,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
           return false; // Invalid IR.
         VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
         if (Ty && Ty->getElementType()->isFloatingPointTy()) {
-          NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                            Intrinsic::aarch64_neon_faddp, Ty);
+          NewFn = Intrinsic::getOrInsertDeclaration(
+              F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
           return true;
         }
       }
@@ -840,7 +841,7 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
                   .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
                   .Default(Intrinsic::not_intrinsic);
           if (ID != Intrinsic::not_intrinsic) {
-            NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+            NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
             return true;
           }
           return false; // No other 'aarch64.sve.bf*.lane'.
@@ -861,8 +862,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
 
         auto Args = F->getFunctionType()->params();
         Type *Tys[] = {F->getReturnType(), Args[1]};
-        NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                          Intrinsic::aarch64_sve_faddqv, Tys);
+        NewFn = Intrinsic::getOrInsertDeclaration(
+            F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
         return true;
       }
 
@@ -880,8 +881,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
               Intrinsic::aarch64_sve_ld3_sret,
               Intrinsic::aarch64_sve_ld4_sret,
           };
-          NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                            LoadIDs[Name[0] - '2'], Ty);
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                    LoadIDs[Name[0] - '2'], Ty);
           return true;
         }
         return false; // No other 'aarch64.sve.ld*'.
@@ -892,8 +893,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
         if (Name.starts_with("get")) {
           // 'aarch64.sve.tuple.get*'.
           Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
-          NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                            Intrinsic::vector_extract, Tys);
+          NewFn = Intrinsic::getOrInsertDeclaration(
+              F->getParent(), Intrinsic::vector_extract, Tys);
           return true;
         }
 
@@ -901,8 +902,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
           // 'aarch64.sve.tuple.set*'.
           auto Args = F->getFunctionType()->params();
           Type *Tys[] = {Args[0], Args[2], Args[1]};
-          NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                            Intrinsic::vector_insert, Tys);
+          NewFn = Intrinsic::getOrInsertDeclaration(
+              F->getParent(), Intrinsic::vector_insert, Tys);
           return true;
         }
 
@@ -911,8 +912,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
           // 'aarch64.sve.tuple.create*'.
           auto Args = F->getFunctionType()->params();
           Type *Tys[] = {F->getReturnType(), Args[1]};
-          NewFn = Intrinsic::getDeclaration(F->getParent(),
-                                            Intrinsic::vector_insert, Tys);
+          NewFn = Intrinsic::getOrInsertDeclaration(
+              F->getParent(), Intrinsic::vector_insert, Tys);
           return true;
         }
         return false; // No other 'aarch64.sve.tuple.*'.
@@ -1026,8 +1027,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
     if (Name.consume_front("amdgcn.")) {
       if (Name == "alignbit") {
         // Target specific intrinsic became redundant
-        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr,
-                                          {F->getReturnType()});
+        NewFn = Intrinsic::getOrInsertDeclaration(
+            F->getParent(), Intrinsic::fshr, {F->getReturnType()});
         return true;
       }
 
@@ -1056,9 +1057,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
 
       if (Name.starts_with("ldexp.")) {
         // Target specific intrinsic became redundant
-        NewFn = Intrinsic::getDeclaration(
-          F->getParent(), Intrinsic::ldexp,
-          {F->getReturnType(), F->getArg(1)->getType()});
+        NewFn = Intrinsic::getOrInsertDeclaration(
+            F->getParent(), Intrinsic::ldexp,
+            {F->getReturnType(), F->getArg(1)->getType()});
         return true;
       }
       break; // No other 'amdgcn.*'
@@ -1074,15 +1075,16 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
                              .Default(Intrinsic::not_intrinsic);
       if (ID != Intrinsic::not_intrinsic) {
         rename(F);
-        NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
-                                          F->arg_begin()->getType());
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
+                                                  F->arg_begin()->getType());
         return true;
       }
     }
 
     if (F->arg_size() == 2 && Name == "coro.end") {
       rename(F);
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::coro_end);
+      NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                Intrinsic::coro_end);
       return true;
     }
 
@@ -1105,7 +1107,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
       // converted to DbgVariableRecords later.
       if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
         rename(F);
-        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                  Intrinsic::dbg_value);
         return true;
       }
       break; // No other 'dbg.*'.
@@ -1135,7 +1138,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
           // Inserting overloads the inserted type.
           Tys.push_back(FT->getParamType(1));
         rename(F);
-        NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
         return true;
       }
 
@@ -1171,8 +1174,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         if (ID != Intrinsic::not_intrinsic) {
           rename(F);
           auto Args = F->getFunctionType()->params();
-          NewFn =
-              Intrinsic::getDeclaration(F->getParent(), ID, {Args[V2 ? 1 : 0]});
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
+                                                    {Args[V2 ? 1 : 0]});
           return true;
         }
         break; // No other 'expermental.vector.reduce.*'.
@@ -1182,15 +1185,16 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
     if (Name.consume_front("experimental.stepvector.")) {
       Intrinsic::ID ID = Intrinsic::stepvector;
       rename(F);
-      NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
-                                        F->getFunctionType()->getReturnType());
+      NewFn = Intrinsic::getOrInsertDeclaration(
+          F->getParent(), ID, F->getFunctionType()->getReturnType());
       return true;
     }
     break; // No other 'e*'.
   case 'f':
     if (Name.starts_with("flt.rounds")) {
       rename(F);
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::get_rounding);
+      NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                Intrinsic::get_rounding);
       return true;
     }
     break;
@@ -1200,8 +1204,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
       auto Args = F->getFunctionType()->params();
       Type* ObjectPtr[1] = {Args[0]};
       rename(F);
-      NewFn = Intrinsic::getDeclaration(F->getParent(),
-          Intrinsic::launder_invariant_group, ObjectPtr);
+      NewFn = Intrinsic::getOrInsertDeclaration(
+          F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
       return true;
     }
     break;
@@ -1218,7 +1222,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         // Get the types of dest, src, and len
         ArrayRef<Type *> ParamTypes =
             F->getFunctionType()->params().slice(0, 3);
-        NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ParamTypes);
+        NewFn =
+            Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
         return true;
       }
     }
@@ -1230,8 +1235,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
           FT->getParamType(0), // Dest
           FT->getParamType(2)  // len
       };
-      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
-                                        ParamTypes);
+      NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                Intrinsic::memset, ParamTypes);
       return true;
     }
     break;
@@ -1247,8 +1252,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
                 .Case("popc.i", Intrinsic::ctpop)
                 .Default(Intrinsic::not_intrinsic);
         if (IID != Intrinsic::not_intrinsic) {
-          NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
-                                            {F->getReturnType()});
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
+                                                    {F->getReturnType()});
           return true;
         }
       }
@@ -1316,8 +1321,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
           F->getName() !=
               Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) {
         rename(F);
-        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
-                                          Tys);
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                  Intrinsic::objectsize, Tys);
         return true;
       }
     }
@@ -1326,7 +1331,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
   case 'p':
     if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
       rename(F);
-      NewFn = Intrinsic::getDeclaration(
+      NewFn = Intrinsic::getOrInsertDeclaration(
           F->getParent(), Intrinsic::ptr_annotation,
           {F->arg_begin()->getType(), F->getArg(1)->getType()});
       return true;
@@ -1345,7 +1350,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
       if (ID != Intrinsic::not_intrinsic) {
         if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
           rename(F);
-          NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
           return true;
         }
         break; // No other applicable upgrades.
@@ -1359,7 +1364,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
             F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
           rename(F);
-          NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
           return true;
         }
         break; // No other applicable upgrades.
@@ -1376,7 +1381,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
       if (ID != Intrinsic::not_intrinsic) {
         if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
           rename(F);
-          NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
           return true;
         }
         break; // No other applicable upgrades.
@@ -1395,7 +1400,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
   case 'v': {
     if (Name == "var.annotation" && F->arg_size() == 4) {
       rename(F);
-      NewFn = Intrinsic::getDeclaration(
+      NewFn = Intrinsic::getOrInsertDeclaration(
           F->getParent(), Intrinsic::var_annotation,
           {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
       return true;
@@ -1413,8 +1418,8 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
               .Default(Intrinsic::not_intrinsic);
       if (ID != Intrinsic::not_intrinsic) {
         rename(F);
-        NewFn =
-            Intrinsic::getDeclaration(F->getParent(), ID, F->getReturnType());
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
+                                                  F->getReturnType());
         return true;
       }
 
@@ -1426,7 +1431,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
                  .Default(Intrinsic::not_intrinsic);
         if (ID != Intrinsic::not_intrinsic) {
           rename(F);
-          NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+          NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
           return true;
         }
         break; // No other 'wasm.dot.i8x16.i7x16.*'.
@@ -1740,8 +1745,8 @@ static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
   if (!IndexForm)
     std::swap(Args[0], Args[1]);
 
-  Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
-                                Args);
+  Value *V = Builder.CreateCall(
+      Intrinsic::getOrInsertDeclaration(CI.getModule(), IID), Args);
   Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
                              : Builder.CreateBitCast(CI.getArgOperand(1),
                                                      Ty);
@@ -1753,7 +1758,7 @@ static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
   Type *Ty = CI.getType();
   Value *Op0 = CI.getOperand(0);
   Value *Op1 = CI.getOperand(1);
-  Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+  Function *Intrin = Intrinsic::getOrInsertDeclaration(CI.getModule(), IID, Ty);
   Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
 
   if (CI.arg_size() == 4) { // For masked intrinsics.
@@ -1780,7 +1785,7 @@ static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
   }
 
   Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
-  Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+  Function *Intrin = Intrinsic::getOrInsertDeclaration(CI.getModule(), IID, Ty);
   Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
 
   if (CI.arg_size() == 4) { // For masked intrinsics.
@@ -1850,7 +1855,7 @@ static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
   }
 
   Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
-  Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+  Function *Intrin = Intrinsic::getOrInsertDeclaration(CI.getModule(), IID, Ty);
   Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
 
   unsigned NumArgs = CI.arg_size();
@@ -1911,7 +1916,8 @@ static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
 static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
   Type *Ty = CI.getType();
   Value *Op0 = CI.getArgOperand(0);
-  Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
+  Function *F =
+      Intrinsic::getOrInsertDeclaration(CI.getModule(), Intrinsic::abs, Ty);
   Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
   if (CI.arg_size() == 3)
     Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
@@ -2004,7 +2010,7 @@ static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
 // Replace a masked intrinsic with an older unmasked intrinsic.
 static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
                                     Intrinsic::ID IID) {
-  Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
+  Function *Intrin = Intrinsic::getOrInsertDeclaration(CI.getModule(), IID);
   Value *Rep = Builder.CreateCall(Intrin,
                                  { CI.getArgOperand(0), CI.getArgOperand(1) });
   return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
@@ -2263,8 +2269,8 @@ static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
   SmallVector<Value *, 4> Args(CI.args());
   Args.pop_back();
   Args.pop_back();
-  Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
-                           Args);
+  Rep = Builder.CreateCall(
+      Intrinsic::getOrInsertDeclaration(CI.getModule(), IID), Args);
   unsigned NumArgs = CI.arg_size();
   Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
                       CI.getArgOperand(NumArgs - 2));
@@ -2320,8 +2326,8 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
     // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
     Value *Arg = CI->getArgOperand(0);
     Value *Ctlz = Builder.CreateCall(
-        Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
-                                  {Arg->getType()}),
+        Intrinsic::getOrInsertDeclaration(F->getParent(), Intrinsic::ctlz,
+                                          {Arg->getType()}),
         {Arg, Builder.getFalse()}, "ctlz");
     Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
   } else if (Name == "popc.ll") {
@@ -2329,15 +2335,15 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
     // i64.
     Value *Arg = CI->getArgOperand(0);
     Value *Popc = Builder.CreateCall(
-        Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
-                                  {Arg->getType()}),
+        Intrinsic::getOrInsertDeclaration(F->getParent(), Intrinsic::ctpop,
+                                          {Arg->getType()}),
         Arg, "ctpop");
     Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
   } else if (Name == "h2f") {
-    Rep = Builder.CreateCall(
-        Intrinsic::getDeclaration(F->getParent(), Intrinsic::convert_from_fp16,
-                                  {Builder.getFloatTy()}),
-        CI->getArgOperand(0), "h2f");
+    Rep = Builder.CreateCall(Intrinsic::getOrInsertDeclaration(
+                                 F->getParent(), Intrinsic::convert_from_fp16,
+                                 {Builder.getFloatTy()}),
+                             CI->getArgOperand(0), "h2f");
   } else if (Name.consume_front("bitcast.") &&
              (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
               Name == "d2ll")) {
@@ -2373,7 +2379,7 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
     if (IID != Intrinsic::not_intrinsic &&
         !F->getReturnType()->getScalarType()->isBFloatTy()) {
       rename(F);
-      Function *NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+      Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
       SmallVector<Value *, 2> Args;
       for (size_t I = 0; I < NewFn->arg_size(); ++I) {
         Value *Arg = CI->getArgOperand(I);
@@ -2480,15 +2486,15 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
   } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
     Value *Vec = CI->getArgOperand(0);
     Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
-    Function *Intr = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sqrt,
-                                               Elt0->getType());
+    Function *Intr = Intrinsic::getOrInsertDeclaration(
+        F->getParent(), Intrinsic::sqrt, Elt0->getType());
     Elt0 = Builder.CreateCall(Intr, Elt0);
     Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
   } else if (Name.starts_with("avx.sqrt.p") ||
              Name.starts_with("sse2.sqrt.p") ||
              Name.starts_with("sse.sqrt.p")) {
     Rep =
-        Builder.CreateCall(Intrinsic::getDeclaration(
+        Builder.CreateCall(Intrinsic::getOrInsertDeclaration(
                                F->getParent(), Intrinsic::sqrt, CI->getType()),
                            {CI->getArgOperand(0)});
   } else if (Name.starts_with("avx512.mask.sqrt.p")) {
@@ -2499,13 +2505,13 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
                                           : Intrinsic::x86_avx512_sqrt_pd_512;
 
       Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
-      Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
-                               Args);
+      Rep = Builder.CreateCall(
+          Intrinsic::getOrInsertDeclaration(CI->getModule(), IID), Args);
     } else {
-      Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
-                                                         Intrinsic::sqrt,
-                                                         CI->getType()),
-                               {CI->getArgOperand(0)});
+      Rep = Builder.CreateCall(
+          Intrinsic::getOrInsertDeclaration(F->getParent(), Intrinsic::sqrt,
+                                            CI->getType()),
+          {CI->getArgOperand(0)});
     }
     Rep =
         emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
@@ -2629,8 +2635,9 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
       break;
     }
 
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-                             {CI->getOperand(0), CI->getArgOperand(1)});
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
+        {CI->getOperand(0), CI->getArgOperand(1)});
     Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
   } else if (Name.starts_with("avx512.mask.fpclass.p")) {
     Type *OpTy = CI->getArgOperand(0)->getType();
@@ -2652,8 +2659,9 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     else
       llvm_unreachable("Unexpected intrinsic");
 
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-                             {CI->getOperand(0), CI->getArgOperand(1)});
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
+        {CI->getOperand(0), CI->getArgOperand(1)});
     Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
   } else if (Name.starts_with("avx512.cmp.p")) {
     SmallVector<Value *, 4> Args(CI->args());
@@ -2681,8 +2689,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
       std::swap(Mask, Args.back());
     Args.push_back(Mask);
 
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-                             Args);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(F->getParent(), IID), Args);
   } else if (Name.starts_with("avx512.mask.cmp.")) {
     // Integer compare intrinsics.
     unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
@@ -2776,8 +2784,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
               cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
       Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
                                      : Intrinsic::x86_avx512_sitofp_round;
-      Function *F =
-          Intrinsic::getDeclaration(CI->getModule(), IID, {DstTy, SrcTy});
+      Function *F = Intrinsic::getOrInsertDeclaration(CI->getModule(), IID,
+                                                      {DstTy, SrcTy});
       Rep = Builder.CreateCall(F, {Rep, CI->getArgOperand(3)});
     } else {
       Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
@@ -2819,7 +2827,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
                                    ResultTy->getNumElements());
 
-    Function *ELd = Intrinsic::getDeclaration(
+    Function *ELd = Intrinsic::getOrInsertDeclaration(
         F->getParent(), Intrinsic::masked_expandload, ResultTy);
     Rep = Builder.CreateCall(ELd, {Ptr, MaskVec, CI->getOperand(1)});
   } else if (Name.starts_with("avx512.mask.compress.store.")) {
@@ -2834,7 +2842,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
         getX86MaskVec(Builder, CI->getArgOperand(2),
                       cast<FixedVectorType>(ResultTy)->getNumElements());
 
-    Function *CSt = Intrinsic::getDeclaration(
+    Function *CSt = Intrinsic::getOrInsertDeclaration(
         F->getParent(), Intrinsic::masked_compressstore, ResultTy);
     Rep = Builder.CreateCall(CSt, {CI->getArgOperand(1), Ptr, MaskVec});
   } else if (Name.starts_with("avx512.mask.compress.") ||
@@ -2847,7 +2855,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     bool IsCompress = Name[12] == 'c';
     Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
                                    : Intrinsic::x86_avx512_mask_expand;
-    Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
+    Function *Intr =
+        Intrinsic::getOrInsertDeclaration(F->getParent(), IID, ResultTy);
     Rep = Builder.CreateCall(Intr,
                              {CI->getOperand(0), CI->getOperand(1), MaskVec});
   } else if (Name.starts_with("xop.vpcom")) {
@@ -2910,7 +2919,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     bool ZeroMask = Name[11] == 'z';
     Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
   } else if (Name == "sse42.crc32.64.8") {
-    Function *CRC32 = Intrinsic::getDeclaration(
+    Function *CRC32 = Intrinsic::getOrInsertDeclaration(
         F->getParent(), Intrinsic::x86_sse42_crc32_32_8);
     Value *Trunc0 =
         Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
@@ -3405,7 +3414,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
         IID = Intrinsic::x86_avx512_add_pd_512;
 
       Rep = Builder.CreateCall(
-          Intrinsic::getDeclaration(F->getParent(), IID),
+          Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
           {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
     } else {
       Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
@@ -3421,7 +3430,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
         IID = Intrinsic::x86_avx512_div_pd_512;
 
       Rep = Builder.CreateCall(
-          Intrinsic::getDeclaration(F->getParent(), IID),
+          Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
           {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
     } else {
       Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
@@ -3437,7 +3446,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
         IID = Intrinsic::x86_avx512_mul_pd_512;
 
       Rep = Builder.CreateCall(
-          Intrinsic::getDeclaration(F->getParent(), IID),
+          Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
           {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
     } else {
       Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
@@ -3453,7 +3462,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
         IID = Intrinsic::x86_avx512_sub_pd_512;
 
       Rep = Builder.CreateCall(
-          Intrinsic::getDeclaration(F->getParent(), IID),
+          Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
           {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
     } else {
       Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
@@ -3471,13 +3480,13 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
 
     Rep = Builder.CreateCall(
-        Intrinsic::getDeclaration(F->getParent(), IID),
+        Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
         {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
     Rep =
         emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
   } else if (Name.starts_with("avx512.mask.lzcnt.")) {
     Rep =
-        Builder.CreateCall(Intrinsic::getDeclaration(
+        Builder.CreateCall(Intrinsic::getOrInsertDeclaration(
                                F->getParent(), Intrinsic::ctlz, CI->getType()),
                            {CI->getArgOperand(0), Builder.getInt1(false)});
     Rep =
@@ -3723,10 +3732,10 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     if (NegAcc)
       Ops[2] = Builder.CreateFNeg(Ops[2]);
 
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
-                                                       Intrinsic::fma,
-                                                       Ops[0]->getType()),
-                             Ops);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(CI->getModule(), Intrinsic::fma,
+                                          Ops[0]->getType()),
+        Ops);
 
     if (IsScalar)
       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
@@ -3738,10 +3747,10 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
     Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
 
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
-                                                       Intrinsic::fma,
-                                                       Ops[0]->getType()),
-                             Ops);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(CI->getModule(), Intrinsic::fma,
+                                          Ops[0]->getType()),
+        Ops);
 
     Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
                                       Rep, (uint64_t)0);
@@ -3781,11 +3790,11 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
         IID = Intrinsic::x86_avx512_vfmadd_f64;
       else
         IID = Intrinsic::x86_avx512_vfmadd_f32;
-      Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
+      Function *FMA = Intrinsic::getOrInsertDeclaration(CI->getModule(), IID);
       Rep = Builder.CreateCall(FMA, Ops);
     } else {
-      Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
-                                                A->getType());
+      Function *FMA = Intrinsic::getOrInsertDeclaration(
+          CI->getModule(), Intrinsic::fma, A->getType());
       Rep = Builder.CreateCall(FMA, {A, B, C});
     }
 
@@ -3837,11 +3846,12 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
       else
         IID = Intrinsic::x86_avx512_vfmadd_pd_512;
 
-      Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-                               {A, B, C, CI->getArgOperand(4)});
+      Rep = Builder.CreateCall(
+          Intrinsic::getOrInsertDeclaration(F->getParent(), IID),
+          {A, B, C, CI->getArgOperand(4)});
     } else {
-      Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
-                                                A->getType());
+      Function *FMA = Intrinsic::getOrInsertDeclaration(
+          CI->getModule(), Intrinsic::fma, A->getType());
       Rep = Builder.CreateCall(FMA, {A, B, C});
     }
 
@@ -3868,8 +3878,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
                     CI->getArgOperand(2)};
     Ops[2] = Builder.CreateFNeg(Ops[2]);
-    Rep =
-        Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), Ops);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(F->getParent(), IID), Ops);
   } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
              Name.starts_with("avx512.mask3.vfmaddsub.p") ||
              Name.starts_with("avx512.maskz.vfmaddsub.p") ||
@@ -3892,16 +3902,16 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
       if (IsSubAdd)
         Ops[2] = Builder.CreateFNeg(Ops[2]);
 
-      Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-                               Ops);
+      Rep = Builder.CreateCall(
+          Intrinsic::getOrInsertDeclaration(F->getParent(), IID), Ops);
     } else {
       int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
 
       Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
                       CI->getArgOperand(2)};
 
-      Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
-                                                Ops[0]->getType());
+      Function *FMA = Intrinsic::getOrInsertDeclaration(
+          CI->getModule(), Intrinsic::fma, Ops[0]->getType());
       Value *Odd = Builder.CreateCall(FMA, Ops);
       Ops[2] = Builder.CreateFNeg(Ops[2]);
       Value *Even = Builder.CreateCall(FMA, Ops);
@@ -3944,8 +3954,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
 
     Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
                      CI->getArgOperand(2), CI->getArgOperand(3)};
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
-                             Args);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(CI->getModule(), IID), Args);
     Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                : CI->getArgOperand(0);
     Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
@@ -3972,8 +3982,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
 
     Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
                      CI->getArgOperand(2)};
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
-                             Args);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(CI->getModule(), IID), Args);
     Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                : CI->getArgOperand(0);
     Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
@@ -4008,8 +4018,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
 
     Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
                      CI->getArgOperand(2)};
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
-                             Args);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(CI->getModule(), IID), Args);
     Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                : CI->getArgOperand(0);
     Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
@@ -4038,8 +4048,8 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
 
     Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
                      CI->getArgOperand(2)};
-    Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
-                             Args);
+    Rep = Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(CI->getModule(), IID), Args);
     Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                : CI->getArgOperand(0);
     Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
@@ -4062,7 +4072,7 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
     Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
                      CI->getArgOperand(2)};
     Value *NewCall = Builder.CreateCall(
-        Intrinsic::getDeclaration(CI->getModule(), IID), Args);
+        Intrinsic::getOrInsertDeclaration(CI->getModule(), IID), Args);
 
     // Extract the second result and store it.
     Value *Data = Builder.CreateExtractValue(NewCall, 1);
@@ -4108,7 +4118,7 @@ static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
   Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
                                     GoodPredTy, Args[1]);
 
-  Function *NewF = Intrinsic::getDeclaration(CI->getModule(), NewID);
+  Function *NewF = Intrinsic::getOrInsertDeclaration(CI->getModule(), NewID);
   return Builder.CreateCall(NewF, Args, CI->getName());
 }
 
@@ -4117,16 +4127,17 @@ static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
   if (Name == "mve.vctp64.old") {
     // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
     // correct type.
-    Value *VCTP = Builder.CreateCall(
-        Intrinsic::getDeclaration(F->getParent(), Intrinsic::arm_mve_vctp64),
-        CI->getArgOperand(0), CI->getName());
+    Value *VCTP =
+        Builder.CreateCall(Intrinsic::getOrInsertDeclaration(
+                               F->getParent(), Intrinsic::arm_mve_vctp64),
+                           CI->getArgOperand(0), CI->getName());
     Value *C1 = Builder.CreateCall(
-        Intrinsic::getDeclaration(
+        Intrinsic::getOrInsertDeclaration(
             F->getParent(), Intrinsic::arm_mve_pred_v2i,
             {VectorType::get(Builder.getInt1Ty(), 2, false)}),
         VCTP);
     return Builder.CreateCall(
-        Intrinsic::getDeclaration(
+        Intrinsic::getOrInsertDeclaration(
             F->getParent(), Intrinsic::arm_mve_pred_i2v,
             {VectorType::get(Builder.getInt1Ty(), 4, false)}),
         C1);
@@ -4188,19 +4199,19 @@ static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
       Type *Ty = Op->getType();
       if (Ty->getScalarSizeInBits() == 1) {
         Value *C1 = Builder.CreateCall(
-            Intrinsic::getDeclaration(
+            Intrinsic::getOrInsertDeclaration(
                 F->getParent(), Intrinsic::arm_mve_pred_v2i,
                 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
             Op);
         Op = Builder.CreateCall(
-            Intrinsic::getDeclaration(F->getParent(),
-                                      Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
+            Intrinsic::getOrInsertDeclaration(
+                F->getParent(), Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
             C1);
       }
       Ops.push_back(Op);
     }
 
-    Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+    Function *Fn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
     return Builder.CreateCall(Fn, Ops, CI->getName());
   }
   llvm_unreachable("Unknown function for ARM CallBase upgrade.");
@@ -5088,7 +5099,8 @@ void llvm::UpgradeARCRuntime(Module &M) {
     if (!Fn)
       return;
 
-    Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
+    Function *NewFn =
+        llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
 
     for (User *U : make_early_inc_range(Fn->users())) {
       CallInst *CI = dyn_cast<CallInst>(U);
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index ee084e870263d03726aaa3acbb5f51edd3ddb8b3..1cf998c685006809a255236fd23d6db0a2f66121 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2468,7 +2468,7 @@ LLVMValueRef LLVMGetIntrinsicDeclaration(LLVMModuleRef Mod,
                                          size_t ParamCount) {
   ArrayRef<Type*> Tys(unwrap(ParamTypes), ParamCount);
   auto IID = llvm_map_to_intrinsic_id(ID);
-  return wrap(llvm::Intrinsic::getDeclaration(unwrap(Mod), IID, Tys));
+  return wrap(llvm::Intrinsic::getOrInsertDeclaration(unwrap(Mod), IID, Tys));
 }
 
 const char *LLVMIntrinsicGetName(unsigned ID, size_t *NameLength) {
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 0db82cdd6373c82c76aa64b12728528ecb26008c..447a9d65174636eefa689aa5c935e0847da67880 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -991,7 +991,7 @@ DbgInstPtr DIBuilder::insertDbgAssign(Instruction *LinkedInstr, Value *Val,
   LLVMContext &Ctx = LinkedInstr->getContext();
   Module *M = LinkedInstr->getModule();
   if (!AssignFn)
-    AssignFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_assign);
+    AssignFn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_assign);
 
   std::array<Value *, 6> Args = {
       MetadataAsValue::get(Ctx, ValueAsMetadata::get(Val)),
@@ -1060,7 +1060,7 @@ static Value *getDbgIntrinsicValueImpl(LLVMContext &VMContext, Value *V) {
 }
 
 static Function *getDeclareIntrin(Module &M) {
-  return Intrinsic::getDeclaration(&M, Intrinsic::dbg_declare);
+  return Intrinsic::getOrInsertDeclaration(&M, Intrinsic::dbg_declare);
 }
 
 DbgInstPtr DIBuilder::insertDbgValueIntrinsic(
@@ -1074,7 +1074,7 @@ DbgInstPtr DIBuilder::insertDbgValueIntrinsic(
   }
 
   if (!ValueFn)
-    ValueFn = Intrinsic::getDeclaration(&M, Intrinsic::dbg_value);
+    ValueFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::dbg_value);
   return insertDbgIntrinsic(ValueFn, Val, VarInfo, Expr, DL, InsertBB,
                             InsertBefore);
 }
@@ -1175,7 +1175,7 @@ DbgInstPtr DIBuilder::insertLabel(DILabel *LabelInfo, const DILocation *DL,
   }
 
   if (!LabelFn)
-    LabelFn = Intrinsic::getDeclaration(&M, Intrinsic::dbg_label);
+    LabelFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::dbg_label);
 
   Value *Args[] = {MetadataAsValue::get(VMContext, LabelInfo)};
 
diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp
index 0db908211b553c81387542f91a0290c702454918..b37dbd534092c3e9a9a4cabac91eabde70c78811 100644
--- a/llvm/lib/IR/DebugProgramInstruction.cpp
+++ b/llvm/lib/IR/DebugProgramInstruction.cpp
@@ -413,13 +413,13 @@ DbgVariableRecord::createDebugIntrinsic(Module *M,
   // Work out what sort of intrinsic we're going to produce.
   switch (getType()) {
   case DbgVariableRecord::LocationType::Declare:
-    IntrinsicFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_declare);
+    IntrinsicFn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_declare);
     break;
   case DbgVariableRecord::LocationType::Value:
-    IntrinsicFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_value);
+    IntrinsicFn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_value);
     break;
   case DbgVariableRecord::LocationType::Assign:
-    IntrinsicFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_assign);
+    IntrinsicFn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_assign);
     break;
   case DbgVariableRecord::LocationType::End:
   case DbgVariableRecord::LocationType::Any:
@@ -459,7 +459,7 @@ DbgVariableRecord::createDebugIntrinsic(Module *M,
 DbgLabelInst *
 DbgLabelRecord::createDebugIntrinsic(Module *M,
                                      Instruction *InsertBefore) const {
-  auto *LabelFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_label);
+  auto *LabelFn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_label);
   Value *Args[] = {
       MetadataAsValue::get(getDebugLoc()->getContext(), getLabel())};
   DbgLabelInst *DbgLabel = cast<DbgLabelInst>(
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 5367d262030292bb61f758c77c9de8412dba99e9..9e52689f6e0e2f7ca3a5ef135a92deb7d6d1473d 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -91,8 +91,8 @@ Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
   if (cast<ConstantInt>(Scaling)->isZero())
     return Scaling;
   Module *M = GetInsertBlock()->getParent()->getParent();
-  Function *TheFn =
-      Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::vscale,
+                                                      {Scaling->getType()});
   CallInst *CI = CreateCall(TheFn, {}, {}, Name);
   return cast<ConstantInt>(Scaling)->isOne() ? CI : CreateMul(CI, Scaling);
 }
@@ -142,7 +142,8 @@ CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
   Type *Tys[] = { Ptr->getType(), Size->getType() };
   Module *M = BB->getParent()->getParent();
-  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
+  Function *TheFn =
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::memset, Tys);
 
   CallInst *CI = CreateCall(TheFn, Ops);
 
@@ -170,7 +171,8 @@ CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
   Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
   Type *Tys[] = {Dst->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys);
+  Function *TheFn =
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::memset_inline, Tys);
 
   CallInst *CI = CreateCall(TheFn, Ops);
 
@@ -197,7 +199,7 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
   Type *Tys[] = {Ptr->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Function *TheFn = Intrinsic::getDeclaration(
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::memset_element_unordered_atomic, Tys);
 
   CallInst *CI = CreateCall(TheFn, Ops);
@@ -227,7 +229,7 @@ CallInst *IRBuilderBase::CreateMemTransferInst(
   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
   Module *M = BB->getParent()->getParent();
-  Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(M, IntrID, Tys);
 
   CallInst *CI = CreateCall(TheFn, Ops);
 
@@ -265,7 +267,7 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Function *TheFn = Intrinsic::getDeclaration(
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
 
   CallInst *CI = CreateCall(TheFn, Ops);
@@ -381,7 +383,7 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Function *TheFn = Intrinsic::getDeclaration(
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::memmove_element_unordered_atomic, Tys);
 
   CallInst *CI = CreateCall(TheFn, Ops);
@@ -411,23 +413,23 @@ CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
   Module *M = GetInsertBlock()->getParent()->getParent();
   Value *Ops[] = {Src};
   Type *Tys[] = { Src->getType() };
-  auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
+  auto Decl = Intrinsic::getOrInsertDeclaration(M, ID, Tys);
   return CreateCall(Decl, Ops);
 }
 
 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
   Module *M = GetInsertBlock()->getParent()->getParent();
   Value *Ops[] = {Acc, Src};
-  auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
-                                        {Src->getType()});
+  auto Decl = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::vector_reduce_fadd, {Src->getType()});
   return CreateCall(Decl, Ops);
 }
 
 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
   Module *M = GetInsertBlock()->getParent()->getParent();
   Value *Ops[] = {Acc, Src};
-  auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
-                                        {Src->getType()});
+  auto Decl = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::vector_reduce_fmul, {Src->getType()});
   return CreateCall(Decl, Ops);
 }
 
@@ -489,8 +491,8 @@ CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
            "lifetime.start requires the size to be an i64");
   Value *Ops[] = { Size, Ptr };
   Module *M = BB->getParent()->getParent();
-  Function *TheFn =
-      Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::lifetime_start, {Ptr->getType()});
   return CreateCall(TheFn, Ops);
 }
 
@@ -504,8 +506,8 @@ CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
            "lifetime.end requires the size to be an i64");
   Value *Ops[] = { Size, Ptr };
   Module *M = BB->getParent()->getParent();
-  Function *TheFn =
-      Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::lifetime_end, {Ptr->getType()});
   return CreateCall(TheFn, Ops);
 }
 
@@ -523,8 +525,8 @@ CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
   // Fill in the single overloaded type: memory object type.
   Type *ObjectPtr[1] = {Ptr->getType()};
   Module *M = BB->getParent()->getParent();
-  Function *TheFn =
-      Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::invariant_start, ObjectPtr);
   return CreateCall(TheFn, Ops);
 }
 
@@ -556,13 +558,13 @@ IRBuilderBase::CreateAssumption(Value *Cond,
 
   Value *Ops[] = { Cond };
   Module *M = BB->getParent()->getParent();
-  Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
+  Function *FnAssume = Intrinsic::getOrInsertDeclaration(M, Intrinsic::assume);
   return CreateCall(FnAssume, Ops, OpBundles);
 }
 
 Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
   Module *M = BB->getModule();
-  auto *FnIntrinsic = Intrinsic::getDeclaration(
+  auto *FnIntrinsic = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::experimental_noalias_scope_decl, {});
   return CreateCall(FnIntrinsic, {Scope});
 }
@@ -615,7 +617,7 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
                                                ArrayRef<Type *> OverloadedTypes,
                                                const Twine &Name) {
   Module *M = BB->getParent()->getParent();
-  Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(M, Id, OverloadedTypes);
   return CreateCall(TheFn, Ops, {}, Name);
 }
 
@@ -773,9 +775,9 @@ static CallInst *CreateGCStatepointCallCommon(
     const Twine &Name) {
   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
   // Fill in the one generic type'd argument (the function is also vararg)
-  Function *FnStatepoint =
-      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
-                                {ActualCallee.getCallee()->getType()});
+  Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::experimental_gc_statepoint,
+      {ActualCallee.getCallee()->getType()});
 
   std::vector<Value *> Args = getStatepointArgs(
       *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
@@ -828,9 +830,9 @@ static InvokeInst *CreateGCStatepointInvokeCommon(
     const Twine &Name) {
   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
   // Fill in the one generic type'd argument (the function is also vararg)
-  Function *FnStatepoint =
-      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
-                                {ActualInvokee.getCallee()->getType()});
+  Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::experimental_gc_statepoint,
+      {ActualInvokee.getCallee()->getType()});
 
   std::vector<Value *> Args =
       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
@@ -883,7 +885,7 @@ CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
   Intrinsic::ID ID = Intrinsic::experimental_gc_result;
   Module *M = BB->getParent()->getParent();
   Type *Types[] = {ResultType};
-  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
+  Function *FnGCResult = Intrinsic::getOrInsertDeclaration(M, ID, Types);
 
   Value *Args[] = {Statepoint};
   return CreateCall(FnGCResult, Args, {}, Name);
@@ -894,8 +896,8 @@ CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
                                           Type *ResultType, const Twine &Name) {
   Module *M = BB->getParent()->getParent();
   Type *Types[] = {ResultType};
-  Function *FnGCRelocate =
-      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
+  Function *FnGCRelocate = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::experimental_gc_relocate, Types);
 
   Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
   return CreateCall(FnGCRelocate, Args, {}, Name);
@@ -905,7 +907,7 @@ CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
                                                 const Twine &Name) {
   Module *M = BB->getParent()->getParent();
   Type *PtrTy = DerivedPtr->getType();
-  Function *FnGCFindBase = Intrinsic::getDeclaration(
+  Function *FnGCFindBase = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
   return CreateCall(FnGCFindBase, {DerivedPtr}, {}, Name);
 }
@@ -914,7 +916,7 @@ CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
                                                   const Twine &Name) {
   Module *M = BB->getParent()->getParent();
   Type *PtrTy = DerivedPtr->getType();
-  Function *FnGCGetOffset = Intrinsic::getDeclaration(
+  Function *FnGCGetOffset = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
   return CreateCall(FnGCGetOffset, {DerivedPtr}, {}, Name);
 }
@@ -923,7 +925,7 @@ CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
                                               Instruction *FMFSource,
                                               const Twine &Name) {
   Module *M = BB->getModule();
-  Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
+  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, {V->getType()});
   return createCallHelper(Fn, {V}, Name, FMFSource);
 }
 
@@ -931,7 +933,7 @@ Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
                                             Value *RHS, Instruction *FMFSource,
                                             const Twine &Name) {
   Module *M = BB->getModule();
-  Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
+  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, {LHS->getType()});
   if (Value *V = Folder.FoldBinaryIntrinsic(ID, LHS, RHS, Fn->getReturnType(),
                                             FMFSource))
     return V;
@@ -944,7 +946,7 @@ CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
                                          Instruction *FMFSource,
                                          const Twine &Name) {
   Module *M = BB->getModule();
-  Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
+  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, Types);
   return createCallHelper(Fn, Args, Name, FMFSource);
 }
 
@@ -971,7 +973,7 @@ CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
          "Wrong types for intrinsic!");
   // TODO: Handle varargs intrinsics.
 
-  Function *Fn = Intrinsic::getDeclaration(M, ID, OverloadTys);
+  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, OverloadTys);
   return createCallHelper(Fn, Args, Name, FMFSource);
 }
 
@@ -1128,7 +1130,7 @@ Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
          "launder.invariant.group only applies to pointers.");
   auto *PtrType = Ptr->getType();
   Module *M = BB->getParent()->getParent();
-  Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
+  Function *FnLaunderInvariantGroup = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::launder_invariant_group, {PtrType});
 
   assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
@@ -1145,7 +1147,7 @@ Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
 
   auto *PtrType = Ptr->getType();
   Module *M = BB->getParent()->getParent();
-  Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
+  Function *FnStripInvariantGroup = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::strip_invariant_group, {PtrType});
 
   assert(FnStripInvariantGroup->getReturnType() == PtrType &&
@@ -1160,7 +1162,8 @@ Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
   auto *Ty = cast<VectorType>(V->getType());
   if (isa<ScalableVectorType>(Ty)) {
     Module *M = BB->getParent()->getParent();
-    Function *F = Intrinsic::getDeclaration(M, Intrinsic::vector_reverse, Ty);
+    Function *F =
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::vector_reverse, Ty);
     return Insert(CallInst::Create(F, V), Name);
   }
   // Keep the original behaviour for fixed vector
@@ -1179,7 +1182,8 @@ Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
 
   if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
     Module *M = BB->getParent()->getParent();
-    Function *F = Intrinsic::getDeclaration(M, Intrinsic::vector_splice, VTy);
+    Function *F =
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::vector_splice, VTy);
 
     Value *Ops[] = {V1, V2, getInt32(Imm)};
     return Insert(CallInst::Create(F, Ops), Name);
@@ -1233,7 +1237,7 @@ Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
   Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList);
 
   Module *M = BB->getParent()->getParent();
-  Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
+  Function *FnPreserveArrayAccessIndex = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
 
   Value *DimV = getInt32(Dimension);
@@ -1254,7 +1258,7 @@ Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
   auto *BaseType = Base->getType();
 
   Module *M = BB->getParent()->getParent();
-  Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
+  Function *FnPreserveUnionAccessIndex = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
 
   Value *DIIndex = getInt32(FieldIndex);
@@ -1279,7 +1283,7 @@ Value *IRBuilderBase::CreatePreserveStructAccessIndex(
       GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex});
 
   Module *M = BB->getParent()->getParent();
-  Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
+  Function *FnPreserveStructAccessIndex = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
 
   Value *DIIndex = getInt32(FieldIndex);
@@ -1296,8 +1300,8 @@ Value *IRBuilderBase::CreatePreserveStructAccessIndex(
 Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
   ConstantInt *TestV = getInt32(Test);
   Module *M = BB->getParent()->getParent();
-  Function *FnIsFPClass =
-      Intrinsic::getDeclaration(M, Intrinsic::is_fpclass, {FPNum->getType()});
+  Function *FnIsFPClass = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::is_fpclass, {FPNum->getType()});
   return CreateCall(FnIsFPClass, {FPNum, TestV});
 }
 
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index e7a67e5529a0acd570e453269b0d61c98e994e02..968b492340f8a55bc825c7c5f5c3cd957f61c664 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -441,6 +441,10 @@ void Instruction::dropPoisonGeneratingFlags() {
     cast<TruncInst>(this)->setHasNoUnsignedWrap(false);
     cast<TruncInst>(this)->setHasNoSignedWrap(false);
     break;
+
+  case Instruction::ICmp:
+    cast<ICmpInst>(this)->setSameSign(false);
+    break;
   }
 
   if (isa<FPMathOperator>(this)) {
@@ -654,6 +658,10 @@ void Instruction::copyIRFlags(const Value *V, bool IncludeWrapFlags) {
   if (auto *NNI = dyn_cast<PossiblyNonNegInst>(V))
     if (isa<PossiblyNonNegInst>(this))
       setNonNeg(NNI->hasNonNeg());
+
+  if (auto *SrcICmp = dyn_cast<ICmpInst>(V))
+    if (auto *DestICmp = dyn_cast<ICmpInst>(this))
+      DestICmp->setSameSign(SrcICmp->hasSameSign());
 }
 
 void Instruction::andIRFlags(const Value *V) {
@@ -695,6 +703,10 @@ void Instruction::andIRFlags(const Value *V) {
   if (auto *NNI = dyn_cast<PossiblyNonNegInst>(V))
     if (isa<PossiblyNonNegInst>(this))
       setNonNeg(hasNonNeg() && NNI->hasNonNeg());
+
+  if (auto *SrcICmp = dyn_cast<ICmpInst>(V))
+    if (auto *DestICmp = dyn_cast<ICmpInst>(this))
+      DestICmp->setSameSign(DestICmp->hasSameSign() && SrcICmp->hasSameSign());
 }
 
 const char *Instruction::getOpcodeName(unsigned OpCode) {
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 0bac6bc0adead553e8671d32ebf05dbcb9cb82cd..912d4abb38082a046abe427c0af9dd1ecf884c47 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -640,9 +640,8 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const {
   return false;
 }
 
-Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
-                                               Type *ReturnType,
-                                               ArrayRef<Value *> Params) {
+Function *VPIntrinsic::getOrInsertDeclarationForParams(
+    Module *M, Intrinsic::ID VPID, Type *ReturnType, ArrayRef<Value *> Params) {
   assert(isVPIntrinsic(VPID) && "not a VP intrinsic");
   Function *VPFunc;
   switch (VPID) {
@@ -652,7 +651,7 @@ Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
       OverloadTy =
           Params[*VPReductionIntrinsic::getVectorParamPos(VPID)]->getType();
 
-    VPFunc = Intrinsic::getDeclaration(M, VPID, OverloadTy);
+    VPFunc = Intrinsic::getOrInsertDeclaration(M, VPID, OverloadTy);
     break;
   }
   case Intrinsic::vp_trunc:
@@ -669,46 +668,46 @@ Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
   case Intrinsic::vp_lrint:
   case Intrinsic::vp_llrint:
   case Intrinsic::vp_cttz_elts:
-    VPFunc =
-        Intrinsic::getDeclaration(M, VPID, {ReturnType, Params[0]->getType()});
+    VPFunc = Intrinsic::getOrInsertDeclaration(
+        M, VPID, {ReturnType, Params[0]->getType()});
     break;
   case Intrinsic::vp_is_fpclass:
-    VPFunc = Intrinsic::getDeclaration(M, VPID, {Params[0]->getType()});
+    VPFunc = Intrinsic::getOrInsertDeclaration(M, VPID, {Params[0]->getType()});
     break;
   case Intrinsic::vp_merge:
   case Intrinsic::vp_select:
-    VPFunc = Intrinsic::getDeclaration(M, VPID, {Params[1]->getType()});
+    VPFunc = Intrinsic::getOrInsertDeclaration(M, VPID, {Params[1]->getType()});
     break;
   case Intrinsic::vp_load:
-    VPFunc = Intrinsic::getDeclaration(
+    VPFunc = Intrinsic::getOrInsertDeclaration(
         M, VPID, {ReturnType, Params[0]->getType()});
     break;
   case Intrinsic::experimental_vp_strided_load:
-    VPFunc = Intrinsic::getDeclaration(
+    VPFunc = Intrinsic::getOrInsertDeclaration(
         M, VPID, {ReturnType, Params[0]->getType(), Params[1]->getType()});
     break;
   case Intrinsic::vp_gather:
-    VPFunc = Intrinsic::getDeclaration(
+    VPFunc = Intrinsic::getOrInsertDeclaration(
         M, VPID, {ReturnType, Params[0]->getType()});
     break;
   case Intrinsic::vp_store:
-    VPFunc = Intrinsic::getDeclaration(
+    VPFunc = Intrinsic::getOrInsertDeclaration(
         M, VPID, {Params[0]->getType(), Params[1]->getType()});
     break;
   case Intrinsic::experimental_vp_strided_store:
-    VPFunc = Intrinsic::getDeclaration(
+    VPFunc = Intrinsic::getOrInsertDeclaration(
         M, VPID,
         {Params[0]->getType(), Params[1]->getType(), Params[2]->getType()});
     break;
   case Intrinsic::vp_scatter:
-    VPFunc = Intrinsic::getDeclaration(
+    VPFunc = Intrinsic::getOrInsertDeclaration(
         M, VPID, {Params[0]->getType(), Params[1]->getType()});
     break;
   case Intrinsic::experimental_vp_stepvector:
     VPFunc = Intrinsic::getDeclaration(M, VPID, {ReturnType});
     break;
   case Intrinsic::experimental_vp_splat:
-    VPFunc = Intrinsic::getDeclaration(M, VPID, ReturnType);
+    VPFunc = Intrinsic::getOrInsertDeclaration(M, VPID, ReturnType);
     break;
   }
   assert(VPFunc && "Could not declare VP intrinsic");
diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp
index ef26b1926b97672e97c8576be34d1de65c6025ae..ff8b4b7a020c2ffd5d4798fe504095bf9cfcfa3d 100644
--- a/llvm/lib/IR/Intrinsics.cpp
+++ b/llvm/lib/IR/Intrinsics.cpp
@@ -713,7 +713,8 @@ Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
 #include "llvm/IR/IntrinsicImpl.inc"
 #undef GET_INTRINSIC_ATTRIBUTES
 
-Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef<Type *> Tys) {
+Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
+                                            ArrayRef<Type *> Tys) {
   // There can never be multiple globals with the same name of different types,
   // because intrinsics must be a specific type.
   auto *FT = getType(M->getContext(), id, Tys);
@@ -1078,7 +1079,7 @@ std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
       // invalid and we'll get an error.
       ExistingGV->setName(WantedName + ".renamed");
     }
-    return Intrinsic::getDeclaration(F->getParent(), ID, ArgTys);
+    return Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ArgTys);
   }();
 
   NewDecl->setCallingConv(F->getCallingConv());
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index 704bc8d339bc57fa379b8767e01e3f5f81263223..ab48d3e4101b72533df3c68db80be4caa3e6418b 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -89,21 +89,22 @@ Module::~Module() {
 
 void Module::removeDebugIntrinsicDeclarations() {
   auto *DeclareIntrinsicFn =
-      Intrinsic::getDeclaration(this, Intrinsic::dbg_declare);
+      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_declare);
   assert((!isMaterialized() || DeclareIntrinsicFn->hasZeroLiveUses()) &&
          "Debug declare intrinsic should have had uses removed.");
   DeclareIntrinsicFn->eraseFromParent();
   auto *ValueIntrinsicFn =
-      Intrinsic::getDeclaration(this, Intrinsic::dbg_value);
+      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_value);
   assert((!isMaterialized() || ValueIntrinsicFn->hasZeroLiveUses()) &&
          "Debug value intrinsic should have had uses removed.");
   ValueIntrinsicFn->eraseFromParent();
   auto *AssignIntrinsicFn =
-      Intrinsic::getDeclaration(this, Intrinsic::dbg_assign);
+      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_assign);
   assert((!isMaterialized() || AssignIntrinsicFn->hasZeroLiveUses()) &&
          "Debug assign intrinsic should have had uses removed.");
   AssignIntrinsicFn->eraseFromParent();
-  auto *LabelntrinsicFn = Intrinsic::getDeclaration(this, Intrinsic::dbg_label);
+  auto *LabelntrinsicFn =
+      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_label);
   assert((!isMaterialized() || LabelntrinsicFn->hasZeroLiveUses()) &&
          "Debug label intrinsic should have had uses removed.");
   LabelntrinsicFn->eraseFromParent();
diff --git a/llvm/lib/IR/VectorBuilder.cpp b/llvm/lib/IR/VectorBuilder.cpp
index f42948ba89042f289bfd19a4cb08e181ead10312..737f49b1334d76af0b44c0647f001e813c7d5d81 100644
--- a/llvm/lib/IR/VectorBuilder.cpp
+++ b/llvm/lib/IR/VectorBuilder.cpp
@@ -108,8 +108,8 @@ Value *VectorBuilder::createVectorInstructionImpl(Intrinsic::ID VPID,
   if (VLenPosOpt)
     IntrinParams[*VLenPosOpt] = &requestEVL();
 
-  auto *VPDecl = VPIntrinsic::getDeclarationForParams(&getModule(), VPID,
-                                                      ReturnTy, IntrinParams);
+  auto *VPDecl = VPIntrinsic::getOrInsertDeclarationForParams(
+      &getModule(), VPID, ReturnTy, IntrinParams);
   return Builder.CreateCall(VPDecl, IntrinParams, Name);
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 58203769b2fab50f3d369d1d33224c7262811941..2c7c3555716fddd03165b322b02e91ad5848b49d 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2401,6 +2401,19 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
     if (!Info)
       CheckFailed("invalid name for a VFABI variant: " + S, V);
   }
+
+  if (auto A = Attrs.getFnAttr("denormal-fp-math"); A.isValid()) {
+    StringRef S = A.getValueAsString();
+    if (!parseDenormalFPAttribute(S).isValid())
+      CheckFailed("invalid value for 'denormal-fp-math' attribute: " + S, V);
+  }
+
+  if (auto A = Attrs.getFnAttr("denormal-fp-math-f32"); A.isValid()) {
+    StringRef S = A.getValueAsString();
+    if (!parseDenormalFPAttribute(S).isValid())
+      CheckFailed("invalid value for 'denormal-fp-math-f32' attribute: " + S,
+                  V);
+  }
 }
 
 void Verifier::verifyFunctionMetadata(
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index ac3946b6ef46f32e23afb7212b63c299f42d3dff..b97f9d9f5fed0fd14734617b24a3b3d52b4f9b47 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -757,6 +757,11 @@ MCSectionWasm *MCContext::getWasmSection(const Twine &Section, SectionKind K,
   if (!Group.isTriviallyEmpty() && !Group.str().empty()) {
     GroupSym = cast<MCSymbolWasm>(getOrCreateSymbol(Group));
     GroupSym->setComdat(true);
+    if (K.isMetadata() && !GroupSym->getType().has_value()) {
+      // Comdat group symbol associated with a custom section is a section
+      // symbol (not a data symbol).
+      GroupSym->setType(wasm::WASM_SYMBOL_TYPE_SECTION);
+    }
   }
 
   return getWasmSection(Section, K, Flags, GroupSym, UniqueID);
diff --git a/llvm/lib/Object/COFFObjectFile.cpp b/llvm/lib/Object/COFFObjectFile.cpp
index 3ec7a449bae798d90b919626adf8a292255fbef7..7f21be25b68c840a22ccdd5043d8921da5fba0fe 100644
--- a/llvm/lib/Object/COFFObjectFile.cpp
+++ b/llvm/lib/Object/COFFObjectFile.cpp
@@ -1465,6 +1465,27 @@ StringRef COFFObjectFile::getRelocationTypeName(uint16_t Type) const {
       return "Unknown";
     }
     break;
+  case Triple::mipsel:
+    switch (Type) {
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_ABSOLUTE);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_REFHALF);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_REFWORD);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_JMPADDR);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_REFHI);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_REFLO);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_GPREL);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_LITERAL);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_SECTION);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_SECREL);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_SECRELLO);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_SECRELHI);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_JMPADDR16);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_REFWORDNB);
+      LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_MIPS_PAIR);
+    default:
+      return "Unknown";
+    }
+    break;
   default:
     return "Unknown";
   }
diff --git a/llvm/lib/ObjectYAML/COFFEmitter.cpp b/llvm/lib/ObjectYAML/COFFEmitter.cpp
index bb46de4c6f57f465bb34d67c18a6531dd1e7177e..3811376adebd26725d66534524ff1650922ac452 100644
--- a/llvm/lib/ObjectYAML/COFFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/COFFEmitter.cpp
@@ -125,15 +125,12 @@ struct COFFParser {
   }
 
   unsigned getStringIndex(StringRef Str) {
-    StringMap<unsigned>::iterator i = StringTableMap.find(Str);
-    if (i == StringTableMap.end()) {
-      unsigned Index = StringTable.size();
+    auto [It, Inserted] = StringTableMap.try_emplace(Str, StringTable.size());
+    if (Inserted) {
       StringTable.append(Str.begin(), Str.end());
       StringTable.push_back(0);
-      StringTableMap[Str] = Index;
-      return Index;
     }
-    return i->second;
+    return It->second;
   }
 
   COFFYAML::Object &Obj;
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 733c8f880f8166466e904a92427db282df474c7a..c9957e493264d6892d02ace173b0048b246c5735 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -118,6 +118,7 @@
 #include "llvm/CodeGen/ShadowStackGCLowering.h"
 #include "llvm/CodeGen/SjLjEHPrepare.h"
 #include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/CodeGen/StackColoring.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TwoAddressInstructionPass.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 908657ae568393e630b41227c521b60eadc4918c..41aa3eb46785216d87a01595e46cf47cd9f37a44 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1760,7 +1760,8 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
     // For ThinLTO we must apply the context disambiguation decisions early, to
     // ensure we can correctly match the callsites to summary data.
     if (EnableMemProfContextDisambiguation)
-      MPM.addPass(MemProfContextDisambiguation(ImportSummary));
+      MPM.addPass(MemProfContextDisambiguation(
+          ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
 
     // These passes import type identifier resolutions for whole-program
     // devirtualization and CFI. They must run early because other passes may
@@ -1973,7 +1974,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
   // amount of additional cloning required to distinguish the allocation
   // contexts.
   if (EnableMemProfContextDisambiguation)
-    MPM.addPass(MemProfContextDisambiguation());
+    MPM.addPass(MemProfContextDisambiguation(
+        /*Summary=*/nullptr,
+        PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
 
   // Optimize globals again after we ran the inliner.
   MPM.addPass(GlobalOptPass());
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index bc4e780fb67a603e53a2799d5276fb9aa7f05dc1..461fc43d32f8df223ad1ca4165754636ff3de945 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -18,12 +18,14 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/Wasm.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compression.h"
@@ -894,13 +896,15 @@ static Error readCoverageMappingData(
 Expected<std::unique_ptr<BinaryCoverageReader>>
 BinaryCoverageReader::createCoverageReaderFromBuffer(
     StringRef Coverage, FuncRecordsStorage &&FuncRecords,
+    CoverageMapCopyStorage &&CoverageMap,
     std::unique_ptr<InstrProfSymtab> ProfileNamesPtr, uint8_t BytesInAddress,
     llvm::endianness Endian, StringRef CompilationDir) {
   if (ProfileNamesPtr == nullptr)
     return make_error<CoverageMapError>(coveragemap_error::malformed,
                                         "Caller must provide ProfileNames");
-  std::unique_ptr<BinaryCoverageReader> Reader(new BinaryCoverageReader(
-      std::move(ProfileNamesPtr), std::move(FuncRecords)));
+  std::unique_ptr<BinaryCoverageReader> Reader(
+      new BinaryCoverageReader(std::move(ProfileNamesPtr),
+                               std::move(FuncRecords), std::move(CoverageMap)));
   InstrProfSymtab &ProfileNames = *Reader->ProfileNames;
   StringRef FuncRecordsRef = Reader->FuncRecords->getBuffer();
   if (BytesInAddress == 4 && Endian == llvm::endianness::little) {
@@ -1035,8 +1039,8 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) {
       MemoryBuffer::getMemBuffer(Data);
 
   return BinaryCoverageReader::createCoverageReaderFromBuffer(
-      CoverageMapping, std::move(CoverageRecords), std::move(ProfileNames),
-      BytesInAddress, Endian, CompilationDir);
+      CoverageMapping, std::move(CoverageRecords), nullptr,
+      std::move(ProfileNames), BytesInAddress, Endian, CompilationDir);
 }
 
 /// Find all sections that match \p IPSK name. There may be more than one if
@@ -1075,6 +1079,53 @@ lookupSections(ObjectFile &OF, InstrProfSectKind IPSK) {
   return Sections;
 }
 
+/// Find a section that matches \p Name and is allocatable at runtime.
+///
+/// Returns the contents of the section and its start offset in the object file.
+static Expected<std::pair<StringRef, uint64_t>>
+lookupAllocatableSection(ObjectFile &OF, InstrProfSectKind IPSK) {
+  // On Wasm, allocatable sections can live only in data segments.
+  if (auto *WOF = dyn_cast<WasmObjectFile>(&OF)) {
+    std::vector<const WasmSegment *> Segments;
+    auto ObjFormat = OF.getTripleObjectFormat();
+    auto Name =
+        getInstrProfSectionName(IPSK, ObjFormat, /*AddSegmentInfo=*/false);
+    for (const auto &DebugName : WOF->debugNames()) {
+      if (DebugName.Type != wasm::NameType::DATA_SEGMENT ||
+          DebugName.Name != Name)
+        continue;
+      if (DebugName.Index >= WOF->dataSegments().size())
+        return make_error<CoverageMapError>(coveragemap_error::malformed);
+      auto &Segment = WOF->dataSegments()[DebugName.Index];
+      Segments.push_back(&Segment);
+    }
+    if (Segments.empty())
+      return make_error<CoverageMapError>(coveragemap_error::no_data_found);
+    if (Segments.size() != 1)
+      return make_error<CoverageMapError>(coveragemap_error::malformed);
+
+    const auto &Segment = *Segments.front();
+    auto &Data = Segment.Data;
+    StringRef Content(reinterpret_cast<const char *>(Data.Content.data()),
+                      Data.Content.size());
+    return std::make_pair(Content, Segment.SectionOffset);
+  }
+
+  // On other object file types, delegate to lookupSections to find the section.
+  auto Sections = lookupSections(OF, IPSK);
+  if (!Sections)
+    return Sections.takeError();
+  if (Sections->size() != 1)
+    return make_error<CoverageMapError>(
+        coveragemap_error::malformed,
+        "the size of coverage mapping section is not one");
+  auto &Section = Sections->front();
+  auto ContentsOrErr = Section.getContents();
+  if (!ContentsOrErr)
+    return ContentsOrErr.takeError();
+  return std::make_pair(*ContentsOrErr, Section.getAddress());
+}
+
 static Expected<std::unique_ptr<BinaryCoverageReader>>
 loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
                  StringRef CompilationDir = "",
@@ -1105,23 +1156,20 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
 
   // Look for the sections that we are interested in.
   auto ProfileNames = std::make_unique<InstrProfSymtab>();
-  std::vector<SectionRef> NamesSectionRefs;
   // If IPSK_name is not found, fallback to search for IPK_covname, which is
   // used when binary correlation is enabled.
-  auto NamesSection = lookupSections(*OF, IPSK_name);
+  auto NamesSection = lookupAllocatableSection(*OF, IPSK_name);
   if (auto E = NamesSection.takeError()) {
     consumeError(std::move(E));
-    NamesSection = lookupSections(*OF, IPSK_covname);
+    NamesSection = lookupAllocatableSection(*OF, IPSK_covname);
     if (auto E = NamesSection.takeError())
       return std::move(E);
   }
-  NamesSectionRefs = *NamesSection;
 
-  if (NamesSectionRefs.size() != 1)
-    return make_error<CoverageMapError>(
-        coveragemap_error::malformed,
-        "the size of coverage mapping section is not one");
-  if (Error E = ProfileNames->create(NamesSectionRefs.back()))
+  uint64_t NamesAddress;
+  StringRef NamesContent;
+  std::tie(NamesContent, NamesAddress) = *NamesSection;
+  if (Error E = ProfileNames->create(NamesContent, NamesAddress))
     return std::move(E);
 
   auto CoverageSection = lookupSections(*OF, IPSK_covmap);
@@ -1136,6 +1184,15 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
     return CoverageMappingOrErr.takeError();
   StringRef CoverageMapping = CoverageMappingOrErr.get();
 
+  // If the coverage mapping section is not aligned to 8 bytes, copy it to a
+  // new buffer that is. Wasm format typically has unaligned section contents
+  // because it doesn't have a good way to insert padding bytes.
+  std::unique_ptr<MemoryBuffer> CoverageMapCopy;
+  if (!isAddrAligned(Align(8), CoverageMapping.data())) {
+    CoverageMapCopy = MemoryBuffer::getMemBufferCopy(CoverageMapping);
+    CoverageMapping = CoverageMapCopy->getBuffer();
+  }
+
   // Look for the coverage records section (Version4 only).
   auto CoverageRecordsSections = lookupSections(*OF, IPSK_covfun);
 
@@ -1184,8 +1241,8 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
     *BinaryID = getBuildID(OF.get());
 
   return BinaryCoverageReader::createCoverageReaderFromBuffer(
-      CoverageMapping, std::move(FuncRecords), std::move(ProfileNames),
-      BytesInAddress, Endian, CompilationDir);
+      CoverageMapping, std::move(FuncRecords), std::move(CoverageMapCopy),
+      std::move(ProfileNames), BytesInAddress, Endian, CompilationDir);
 }
 
 /// Determine whether \p Arch is invalid or empty, given \p Bin.
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index be33331be4e8ff7435c599f6a52f60bd81f27825..1b8eac7fac21f761ca55f1021919e4d5643e5e20 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1953,12 +1953,9 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   // pointer bump above.
   while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
          !IsSVECalleeSave(MBBI)) {
-    // Move past instructions generated to calculate VG
-    if (requiresSaveVG(MF))
-      while (isVGInstruction(MBBI))
-        ++MBBI;
-
-    if (CombineSPBump)
+    if (CombineSPBump &&
+        // Only fix-up frame-setup load/store instructions.
+        (!requiresSaveVG(MF) || !isVGInstruction(MBBI)))
       fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
                                         NeedsWinCFI, &HasWinCFI);
     ++MBBI;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8a217cd1ec5cf9883402f96bba5757f77ef571db..ed06d8a5d630136bb73402fb1bb99e2e68357813 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1146,7 +1146,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       Subtarget->requiresStrictAlign() ? MaxStoresPerMemcpyOptSize : 16;
 
   MaxStoresPerMemmoveOptSize = 4;
-  MaxStoresPerMemmove = 4;
+  MaxStoresPerMemmove =
+      Subtarget->requiresStrictAlign() ? MaxStoresPerMemmoveOptSize : 16;
 
   MaxLoadsPerMemcmpOptSize = 4;
   MaxLoadsPerMemcmp =
@@ -16454,8 +16455,8 @@ static void createTblForTrunc(TruncInst *TI, bool IsLittleEndian) {
         Builder.CreateShuffleVector(TI->getOperand(0), ShuffleLanes), VecTy));
 
     if (Parts.size() == 4) {
-      auto *F = Intrinsic::getDeclaration(TI->getModule(),
-                                          Intrinsic::aarch64_neon_tbl4, VecTy);
+      auto *F = Intrinsic::getOrInsertDeclaration(
+          TI->getModule(), Intrinsic::aarch64_neon_tbl4, VecTy);
       Parts.push_back(ConstantVector::get(MaskConst));
       Results.push_back(Builder.CreateCall(F, Parts));
       Parts.clear();
@@ -16484,7 +16485,7 @@ static void createTblForTrunc(TruncInst *TI, bool IsLittleEndian) {
       break;
     }
 
-    auto *F = Intrinsic::getDeclaration(TI->getModule(), TblID, VecTy);
+    auto *F = Intrinsic::getOrInsertDeclaration(TI->getModule(), TblID, VecTy);
     Parts.push_back(ConstantVector::get(MaskConst));
     Results.push_back(Builder.CreateCall(F, Parts));
   }
@@ -16765,9 +16766,10 @@ static Function *getStructuredLoadFunction(Module *M, unsigned Factor,
                                              Intrinsic::aarch64_neon_ld3,
                                              Intrinsic::aarch64_neon_ld4};
   if (Scalable)
-    return Intrinsic::getDeclaration(M, SVELoads[Factor - 2], {LDVTy});
+    return Intrinsic::getOrInsertDeclaration(M, SVELoads[Factor - 2], {LDVTy});
 
-  return Intrinsic::getDeclaration(M, NEONLoads[Factor - 2], {LDVTy, PtrTy});
+  return Intrinsic::getOrInsertDeclaration(M, NEONLoads[Factor - 2],
+                                           {LDVTy, PtrTy});
 }
 
 static Function *getStructuredStoreFunction(Module *M, unsigned Factor,
@@ -16781,9 +16783,10 @@ static Function *getStructuredStoreFunction(Module *M, unsigned Factor,
                                               Intrinsic::aarch64_neon_st3,
                                               Intrinsic::aarch64_neon_st4};
   if (Scalable)
-    return Intrinsic::getDeclaration(M, SVEStores[Factor - 2], {STVTy});
+    return Intrinsic::getOrInsertDeclaration(M, SVEStores[Factor - 2], {STVTy});
 
-  return Intrinsic::getDeclaration(M, NEONStores[Factor - 2], {STVTy, PtrTy});
+  return Intrinsic::getOrInsertDeclaration(M, NEONStores[Factor - 2],
+                                           {STVTy, PtrTy});
 }
 
 /// Lower an interleaved load into a ldN intrinsic.
@@ -27247,7 +27250,7 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
   if (ValueTy->getPrimitiveSizeInBits() == 128) {
     Intrinsic::ID Int =
         IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp;
-    Function *Ldxr = Intrinsic::getDeclaration(M, Int);
+    Function *Ldxr = Intrinsic::getOrInsertDeclaration(M, Int);
 
     Value *LoHi = Builder.CreateCall(Ldxr, Addr, "lohi");
 
@@ -27266,7 +27269,7 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
   Type *Tys[] = { Addr->getType() };
   Intrinsic::ID Int =
       IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
-  Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
+  Function *Ldxr = Intrinsic::getOrInsertDeclaration(M, Int, Tys);
 
   const DataLayout &DL = M->getDataLayout();
   IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy));
@@ -27281,7 +27284,8 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
 void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
     IRBuilderBase &Builder) const {
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
-  Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::aarch64_clrex));
+  Builder.CreateCall(
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::aarch64_clrex));
 }
 
 Value *AArch64TargetLowering::emitStoreConditional(IRBuilderBase &Builder,
@@ -27296,7 +27300,7 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilderBase &Builder,
   if (Val->getType()->getPrimitiveSizeInBits() == 128) {
     Intrinsic::ID Int =
         IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
-    Function *Stxr = Intrinsic::getDeclaration(M, Int);
+    Function *Stxr = Intrinsic::getOrInsertDeclaration(M, Int);
     Type *Int64Ty = Type::getInt64Ty(M->getContext());
     Type *Int128Ty = Type::getInt128Ty(M->getContext());
 
@@ -27311,7 +27315,7 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilderBase &Builder,
   Intrinsic::ID Int =
       IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
   Type *Tys[] = { Addr->getType() };
-  Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
+  Function *Stxr = Intrinsic::getOrInsertDeclaration(M, Int, Tys);
 
   const DataLayout &DL = M->getDataLayout();
   IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
@@ -27348,7 +27352,7 @@ bool AArch64TargetLowering::shouldNormalizeToSelectSequence(LLVMContext &,
 static Value *UseTlsOffset(IRBuilderBase &IRB, unsigned Offset) {
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   Function *ThreadPointerFunc =
-      Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::thread_pointer);
   return IRB.CreatePointerCast(
       IRB.CreateConstGEP1_32(IRB.getInt8Ty(), IRB.CreateCall(ThreadPointerFunc),
                              Offset),
@@ -27516,6 +27520,21 @@ bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
   return OptSize && !VT.isVector();
 }
 
+bool AArch64TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
+                                             const MachineFunction &MF) const {
+  // Avoid merging stores into fixed-length vectors when Neon is unavailable.
+  // In future, we could allow this when SVE is available, but currently,
+  // the SVE lowerings for BUILD_VECTOR are limited to a few specific cases (and
+  // the general lowering may introduce stack spills/reloads).
+  if (MemVT.isFixedLengthVector() && !Subtarget->isNeonAvailable())
+    return false;
+
+  // Do not merge to float value size (128 bytes) if no implicit float attribute
+  // is set.
+  bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
+  return !NoFloat || MemVT.getSizeInBits() <= 64;
+}
+
 bool AArch64TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
   // We want inc-of-add for scalars and sub-of-not for vectors.
   return VT.isScalarInteger();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 035a802cd49b3c110ca451495272967b6e95e44c..cf2ae5fd027c7a3cc49358fd500cc5cf6ea666f9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -846,16 +846,7 @@ public:
   bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
 
   bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
-                        const MachineFunction &MF) const override {
-    // Do not merge to float value size (128 bytes) if no implicit
-    // float attribute is set.
-
-    bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
-
-    if (NoFloat)
-      return (MemVT.getSizeInBits() <= 64);
-    return true;
-  }
+                        const MachineFunction &MF) const override;
 
   bool isCheapToSpeculateCttz(Type *) const override {
     return true;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
index f7e6545f0dd386870707617fbb779ae30968e1a9..fb4d2f3d7bcd3ade1222e225d66ff6eebd494e9e 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
@@ -469,6 +469,89 @@ def V1Write_11c_9L01_9S_9V : SchedWriteRes<[V1UnitL01, V1UnitL01, V1UnitL01,
                                             V1UnitV, V1UnitV, V1UnitV,
                                             V1UnitV, V1UnitV, V1UnitV]>;
 
+//===----------------------------------------------------------------------===//
+// Define forwarded types
+
+// NOTE: SOG, p. 20, n. 2: Accumulator forwarding is not supported for
+// consumers of 64 bit multiply high operations?
+def V1Wr_IM   : SchedWriteRes<[V1UnitM]>  { let Latency = 2; }
+def V1Wr_IMA  : SchedWriteRes<[V1UnitM0]> { let Latency = 2; }
+def V1WriteIM : SchedWriteVariant<
+                  [SchedVar<NeoverseMULIdiomPred, [V1Wr_IM]>,
+                   SchedVar<NoSchedPred,          [V1Wr_IMA]>]>;
+def V1Rd_IMA : SchedReadAdvance<1, [V1Wr_IMA]>;
+
+def V1Wr_FMA : SchedWriteRes<[V1UnitV]> { let Latency = 4; }
+def V1Rd_FMA : SchedReadAdvance<2, [WriteFMul, V1Wr_FMA]>;
+
+def V1Wr_ADA : SchedWriteRes<[V1UnitV13]> { let Latency = 4; }
+def V1Rd_ADA : SchedReadAdvance<3, [V1Wr_ADA]>;
+
+def V1Wr_VDOT : SchedWriteRes<[V1UnitV]> { let Latency = 3; }
+def V1Rd_VDOT : SchedReadAdvance<2, [V1Wr_VDOT]>;
+
+def V1Wr_VMMA : SchedWriteRes<[V1UnitV]> { let Latency = 3; }
+def V1Rd_VMMA : SchedReadAdvance<2, [V1Wr_VMMA]>;
+
+def V1Wr_VMA : SchedWriteRes<[V1UnitV02]> { let Latency = 4; }
+def V1Rd_VMA : SchedReadAdvance<3, [V1Wr_VMA]>;
+
+def V1Wr_VMAL : SchedWriteRes<[V1UnitV02]> { let Latency = 4; }
+def V1Rd_VMAL : SchedReadAdvance<3, [V1Wr_VMAL]>;
+
+def V1Wr_VSA : SchedWriteRes<[V1UnitV13]> { let Latency = 4; }
+def V1Rd_VSA : SchedReadAdvance<3, [V1Wr_VSA]>;
+
+def V1Wr_FCMA : SchedWriteRes<[V1UnitV]> { let Latency = 4; }
+def V1Rd_FCMA : SchedReadAdvance<2, [V1Wr_FCMA]>;
+
+def V1Wr_FPM : SchedWriteRes<[V1UnitV]> { let Latency = 3; }
+def V1Wr_FPMA : SchedWriteRes<[V1UnitV]> { let Latency = 4; }
+def V1Rd_FPMA : SchedReadAdvance<2, [V1Wr_FPM, V1Wr_FPMA]>;
+
+def V1Wr_FPMAL : SchedWriteRes<[V1UnitV]> { let Latency = 5; }
+def V1Rd_FPMAL : SchedReadAdvance<3, [V1Wr_FPMAL]>;
+
+def V1Wr_BFD : SchedWriteRes<[V1UnitV]> { let Latency = 4; }
+def V1Rd_BFD : SchedReadAdvance<2, [V1Wr_BFD]>;
+
+def V1Wr_BFMMA : SchedWriteRes<[V1UnitV]> { let Latency = 5; }
+def V1Rd_BFMMA : SchedReadAdvance<2, [V1Wr_BFMMA]>;
+
+def V1Wr_BFMLA : SchedWriteRes<[V1UnitV]> { let Latency = 4; }
+def V1Rd_BFMLA : SchedReadAdvance<2, [V1Wr_BFMLA]>;
+
+def V1Wr_CRC : SchedWriteRes<[V1UnitM0]> { let Latency = 2; }
+def V1Rd_CRC : SchedReadAdvance<1, [V1Wr_CRC]>;
+
+def V1Wr_ZDOTB : SchedWriteRes<[V1UnitV01]> { let Latency = 3; }
+def V1Rd_ZDOTB : SchedReadAdvance<2, [V1Wr_ZDOTB]>;
+
+def V1Wr_ZUDOTB : SchedWriteRes<[V1UnitV]> { let Latency = 3; }
+def V1Rd_ZUDOTB : SchedReadAdvance<2, [V1Wr_ZUDOTB]>;
+
+def V1Wr_ZDOTH : SchedWriteRes<[V1UnitV0]> { let Latency = 4; }
+def V1Rd_ZDOTH : SchedReadAdvance<3, [V1Wr_ZDOTH]>;
+
+def V1Wr_ZMMA : SchedWriteRes<[V1UnitV01]> { let Latency = 3; }
+def V1Rd_ZMMA : SchedReadAdvance<2, [V1Wr_ZMMA]>;
+
+let Latency = 5, NumMicroOps = 2 in
+def V1Wr_ZMAD : SchedWriteRes<[V1UnitV0, V1UnitV0]>;
+def V1Rd_ZMAD : SchedReadAdvance<3, [V1Wr_ZMAD]>;
+
+def V1Wr_ZFCMA : SchedWriteRes<[V1UnitV01]> { let Latency = 5; }
+def V1Rd_ZFCMA : SchedReadAdvance<3, [V1Wr_ZFCMA]>;
+
+def V1Wr_ZFMA : SchedWriteRes<[V1UnitV01]> { let Latency = 4; }
+def V1Rd_ZFMA : SchedReadAdvance<2, [V1Wr_ZFMA]>;
+
+def V1Wr_ZBFDOT : SchedWriteRes<[V1UnitV01]> { let Latency = 4; }
+def V1Rd_ZBFDOT : SchedReadAdvance<2, [V1Wr_ZBFDOT]>;
+def V1Wr_ZBFMMA : SchedWriteRes<[V1UnitV01]> { let Latency = 5; }
+def V1Rd_ZBFMMA : SchedReadAdvance<2, [V1Wr_ZBFMMA]>;
+def V1Wr_ZBFMAL : SchedWriteRes<[V1UnitV01]> { let Latency = 5; }
+def V1Rd_ZBFMAL : SchedReadAdvance<3, [V1Wr_ZBFMAL]>;
 
 // Miscellaneous Instructions
 // -----------------------------------------------------------------------------
@@ -553,16 +636,19 @@ def : InstRW<[V1Write_1c_1J], (instrs SETF8, SETF16, RMIF, CFINV)>;
 def : SchedAlias<WriteID32, V1Write_12c5_1M0>;
 def : SchedAlias<WriteID64, V1Write_20c5_1M0>;
 
+def : SchedAlias<WriteIM32, V1Write_2c_1M>;
+def : SchedAlias<WriteIM64, V1Write_2c_1M>;
+
 // Multiply
-// Multiply accumulate
-// Multiply accumulate, long
-// Multiply long
-def V1WriteIM : SchedWriteVariant<
-                  [SchedVar<NeoverseMULIdiomPred, [V1Write_2c_1M]>,
-                   SchedVar<NoSchedPred,          [V1Write_2c_1M0]>]>;
-def           : SchedAlias<WriteIM32, V1WriteIM>;
-def           : SchedAlias<WriteIM64, V1WriteIM>;
+// Multiply accumulate, W-form
+// Multiply accumulate, X-form
+def : InstRW<[V1WriteIM, ReadIM, ReadIM, V1Rd_IMA],
+             (instregex "^M(ADD|SUB)[WX]rrr$")>;
 
+// Multiply accumulate long
+// Multiply long
+def : InstRW<[V1WriteIM, ReadIM, ReadIM, V1Rd_IMA],
+             (instregex "^(S|U)M(ADD|SUB)Lrrr$")>;
 // Multiply high
 def : InstRW<[V1Write_3c_1M, ReadIM, ReadIM], (instrs SMULHrr, UMULHrr)>;
 
@@ -680,10 +766,11 @@ def : InstRW<[V1Write_15c7_1V02], (instrs FDIVDrr)>;
 def : InstRW<[V1Write_16c7_1V02], (instrs FSQRTDr)>;
 
 // FP multiply
-def : SchedAlias<WriteFMul, V1Write_3c_1V>;
+def : WriteRes<WriteFMul, [V1UnitV]> { let Latency = 3; }
 
 // FP multiply accumulate
-def : InstRW<[V1Write_4c_1V], (instregex "^FN?M(ADD|SUB)[HSD]rrr$")>;
+def : InstRW<[V1Wr_FMA, ReadDefault, ReadDefault, V1Rd_FMA],
+             (instregex "^FN?M(ADD|SUB)[HSD]rrr$")>;
 
 // FP round to integral
 def : InstRW<[V1Write_3c_1V02], (instregex "^FRINT[AIMNPXZ][HSD]r$",
@@ -824,7 +911,7 @@ def : SchedAlias<WriteVq, V1Write_2c_1V>;
 // ASIMD absolute diff accum
 // ASIMD absolute diff accum long
 // ASIMD pairwise add and accumulate long
-def : InstRW<[V1Write_4c_1V13], (instregex "^[SU]ABAL?v", "^[SU]ADALPv")>;
+def : InstRW<[V1Wr_ADA, V1Rd_ADA], (instregex "^[SU]ABAL?v", "^[SU]ADALPv")>;
 
 // ASIMD arith, reduce, 4H/4S
 // ASIMD max/min, reduce, 4H/4S
@@ -843,23 +930,26 @@ def : InstRW<[V1Write_4c_2V13], (instregex "^(ADD|[SU]ADDL)Vv16i8v$",
 
 // ASIMD dot product
 // ASIMD dot product using signed and unsigned integers
-def : InstRW<[V1Write_2c_1V], (instregex "^([SU]|SU|US)DOT(lane)?v(8|16)i8$")>;
+def : InstRW<[V1Wr_VDOT, V1Rd_VDOT],
+             (instregex "^([SU]|SU|US)DOT(lane)?v(8|16)i8$")>;
 
-// ASIMD matrix multiply- accumulate
-def : InstRW<[V1Write_3c_1V], (instrs SMMLA, UMMLA, USMMLA)>;
+// ASIMD matrix multiply-accumulate
+def : InstRW<[V1Wr_VMMA, V1Rd_VMMA], (instrs SMMLA, UMMLA, USMMLA)>;
 
 // ASIMD multiply
+def : InstRW<[V1Write_4c_1V02], (instregex "^MULv", "^SQ(R)?DMULHv")>;
+
 // ASIMD multiply accumulate
+def : InstRW<[V1Wr_VMA, V1Rd_VMA], (instregex "^MLAv", "^MLSv")>;
+
 // ASIMD multiply accumulate long
+def : InstRW<[V1Wr_VMAL, V1Rd_VMAL], (instregex "^[SU]MLALv", "^[SU]MLSLv")>;
+
 // ASIMD multiply accumulate high
+def : InstRW<[V1Write_4c_1V02], (instregex "^SQRDMLAHv", "^SQRDMLSHv")>;
+
 // ASIMD multiply accumulate saturating long
-def : InstRW<[V1Write_4c_1V02], 
-             (instregex "^MUL(v[148]i16|v[124]i32)$",
-                        "^SQR?DMULH(v[48]i16|v[24]i32)$",
-                        "^ML[AS](v[148]i16|v[124]i32)$",
-                        "^[SU]ML[AS]Lv",
-                        "^SQRDML[AS]H(v[148]i16|v[124]i32)$",
-                        "^SQDML[AS]Lv")>;
+def : InstRW<[V1Write_4c_1V02], (instregex "^SQDML[AS]L[iv]")>;
 
 // ASIMD multiply/multiply long (8x8) polynomial
 def : InstRW<[V1Write_3c_1V01], (instregex "^PMULL?v(8|16)i8$")>;
@@ -868,11 +958,12 @@ def : InstRW<[V1Write_3c_1V01], (instregex "^PMULL?v(8|16)i8$")>;
 def : InstRW<[V1Write_3c_1V02], (instregex "^([SU]|SQD)MULLv")>;
 
 // ASIMD shift accumulate
+def : InstRW<[V1Wr_VSA, V1Rd_VSA], (instregex "^[SU]SRAv", "^[SU]RSRAv")>;
+
 // ASIMD shift by immed, complex
 // ASIMD shift by register, complex
 def : InstRW<[V1Write_4c_1V13],
-             (instregex "^[SU]R?SRAv",
-                        "^RSHRNv", "^SQRSHRU?Nv", "^(SQSHLU?|UQSHL)[bhsd]$",
+             (instregex "^RSHRNv", "^SQRSHRU?Nv", "^(SQSHLU?|UQSHL)[bhsd]$",
                         "^(SQSHLU?|UQSHL)(v8i8|v16i8|v4i16|v8i16|v2i32|v4i32|v2i64)_shift$",
                         "^SQSHU?RNv", "^[SU]RSHRv", "^UQR?SHRNv", 
                         "^[SU]Q?RSHLv", "^[SU]QSHLv")>;
@@ -890,16 +981,25 @@ def : InstRW<[V1Write_2c_1V13], (instregex "^SHLL?v", "^SHRNv", "^[SU]SHLLv",
 // ASIMD FP absolute value/difference
 // ASIMD FP arith, normal
 // ASIMD FP compare
-// ASIMD FP complex add
 // ASIMD FP max/min, normal
 // ASIMD FP max/min, pairwise
 // ASIMD FP negate
 // Covered by "SchedAlias (WriteV[dq]...)" above
 
+// ASIMD FP complex add
+def : InstRW<[V1Write_4c_1V], (instregex "^FCADD(v[48]f16|v[24]f32|v2f64)$")>;
+
 // ASIMD FP complex multiply add
+def : InstRW<[V1Wr_FCMA, V1Rd_FCMA], (instregex "^FCMLAv")>;
+
+// ASIMD FP multiply
+def : InstRW<[V1Wr_FPM], (instregex "^FMULX?v")>;
+
 // ASIMD FP multiply accumulate
-def : InstRW<[V1Write_4c_1V], (instregex "^FCADD(v[48]f16|v[24]f32|v2f64)$",
-                                         "^FML[AS]v")>;
+def : InstRW<[V1Wr_FPMA, V1Rd_FPMA], (instregex "^FML[AS]v")>;
+
+// ASIMD FP multiply accumulate long
+def : InstRW<[V1Wr_FPMAL, V1Rd_FPMAL], (instregex "^FML[AS]L2?v")>;
 
 // ASIMD FP convert, long (F16 to F32)
 def : InstRW<[V1Write_4c_2V02], (instregex "^FCVTLv[48]i16$")>;
@@ -953,12 +1053,6 @@ def : InstRW<[V1Write_4c_2V], (instregex "^F(MAX|MIN)(NM)?Vv4(i16|i32)v$")>;
 // ASIMD FP max/min, reduce, Q-form F16
 def : InstRW<[V1Write_6c_3V], (instregex "^F(MAX|MIN)(NM)?Vv8i16v$")>;
 
-// ASIMD FP multiply
-def : InstRW<[V1Write_3c_1V], (instregex "^FMULX?v")>;
-
-// ASIMD FP multiply accumulate long
-def : InstRW<[V1Write_5c_1V], (instregex "^FML[AS]L2?v")>;
-
 // ASIMD FP round, D-form F32 and Q-form F64
 def : InstRW<[V1Write_3c_1V02], (instregex "^FRINT[AIMNPXZ]v2f(32|64)$")>;
 
@@ -976,13 +1070,13 @@ def : InstRW<[V1Write_6c_4V02], (instregex "^FRINT[AIMNPXZ]v8f16$")>;
 def : InstRW<[V1Write_4c_1V02], (instrs BFCVTN, BFCVTN2)>;
 
 // ASIMD dot product
-def : InstRW<[V1Write_4c_1V], (instregex "^BF(DOT|16DOTlane)v[48]bf16$")>;
+def : InstRW<[V1Wr_BFD, V1Rd_BFD], (instregex "^BF(DOT|16DOTlane)v[48]bf16$")>;
 
 // ASIMD matrix multiply accumulate
-def : InstRW<[V1Write_5c_1V], (instrs BFMMLA)>;
+def : InstRW<[V1Wr_BFMMA, V1Rd_BFMMA], (instrs BFMMLA)>;
 
 // ASIMD multiply accumulate long
-def : InstRW<[V1Write_4c_1V], (instregex "^BFMLAL[BT](Idx)?$")>;
+def : InstRW<[V1Wr_BFMLA, V1Rd_BFMLA], (instregex "^BFMLAL[BT](Idx)?$")>;
 
 // Scalar convert, F32 to BF16
 def : InstRW<[V1Write_3c_1V02], (instrs BFCVT)>;
@@ -1300,7 +1394,7 @@ def : InstRW<[V1Write_2c_1V0], (instrs BCAX, EOR3, RAX1, XAR)>;
 // -----------------------------------------------------------------------------
 
 // CRC checksum ops
-def : InstRW<[V1Write_2c_1M0], (instregex "^CRC32C?[BHWX]rr$")>;
+def : InstRW<[V1Wr_CRC, V1Rd_CRC], (instregex "^CRC32C?[BHWX]rr$")>;
 
 
 // SVE Predicate instructions
@@ -1440,13 +1534,14 @@ def : InstRW<[V1Write_20c7_1V0], (instregex "^[SU]DIVR?_ZPmZ_D",
                                              "^[SU]DIV_ZPZZ_D")>;
 
 // Dot product, 8 bit
-def : InstRW<[V1Write_3c_1V01], (instregex "^[SU]DOT_ZZZI?_S$")>;
+def : InstRW<[V1Wr_ZDOTB, V1Rd_ZDOTB], (instregex "^[SU]DOT_ZZZI?_S$")>;
 
 // Dot product, 8 bit, using signed and unsigned integers
-def : InstRW<[V1Write_3c_1V], (instrs SUDOT_ZZZI, USDOT_ZZZ, USDOT_ZZZI)>;
+def : InstRW<[V1Wr_ZUDOTB, V1Rd_ZUDOTB],
+             (instrs SUDOT_ZZZI, USDOT_ZZZ, USDOT_ZZZI)>;
 
 // Dot product, 16 bit
-def : InstRW<[V1Write_4c_1V0], (instregex "^[SU]DOT_ZZZI?_D$")>;
+def : InstRW<[V1Wr_ZDOTH, V1Rd_ZDOTH], (instregex "^[SU]DOT_ZZZI?_D$")>;
 
 // Duplicate, immediate and indexed form
 def : InstRW<[V1Write_2c_1V01], (instregex "^DUP_ZI_[BHSD]$",
@@ -1488,7 +1583,7 @@ def : InstRW<[V1Write_2c_1V01], (instregex "^MOVPRFX_ZP[mz]Z_[BHSD]$",
                                            "^MOVPRFX_ZZ$")>;
 
 // Matrix multiply-accumulate
-def : InstRW<[V1Write_3c_1V01], (instrs SMMLA_ZZZ, UMMLA_ZZZ, USMMLA_ZZZ)>;
+def : InstRW<[V1Wr_ZMMA, V1Rd_ZMMA], (instrs SMMLA_ZZZ, UMMLA_ZZZ, USMMLA_ZZZ)>;
 
 // Multiply, B, H, S element size
 def : InstRW<[V1Write_4c_1V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]",
@@ -1497,12 +1592,16 @@ def : InstRW<[V1Write_4c_1V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]",
                                           "^[SU]MULH_ZPZZ_[BHS]")>;
 
 // Multiply, D element size
-// Multiply accumulate, D element size
 def : InstRW<[V1Write_5c_2V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_D",
                                           "^MUL_ZPZZ_D",
                                           "^[SU]MULH_(ZPmZ|ZZZ)_D",
-                                          "^[SU]MULH_ZPZZ_D",
-                                          "^(MLA|MLS|MAD|MSB)_(ZPmZZ|ZPZZZ)_D")>;
+                                          "^[SU]MULH_ZPZZ_D")>;
+
+// Multiply accumulate, D element size
+def : InstRW<[V1Wr_ZMAD, V1Rd_ZMAD],
+             (instregex "^ML[AS]_ZPZZZ_D")>;
+def : InstRW<[V1Wr_ZMAD, ReadDefault, V1Rd_ZMAD],
+             (instregex "^(ML[AS]|MAD|MSB)_ZPmZZ_D")>;
 
 // Multiply accumulate, B, H, S element size
 // NOTE: This is not specified in the SOG.
@@ -1583,8 +1682,8 @@ def : InstRW<[V1Write_2c_1V0], (instregex "^FAC(GE|GT)_PPzZZ_[HSD]$",
 def : InstRW<[V1Write_3c_1V01], (instregex "^FCADD_ZPmZ_[HSD]$")>;
 
 // Floating point complex multiply add
-def : InstRW<[V1Write_5c_1V01], (instregex "^FCMLA_ZPmZZ_[HSD]$",
-                                           "^FCMLA_ZZZI_[HS]$")>;
+def : InstRW<[V1Wr_ZFCMA, ReadDefault, V1Rd_ZFCMA], (instregex "^FCMLA_ZPmZZ_[HSD]")>;
+def : InstRW<[V1Wr_ZFCMA, V1Rd_ZFCMA],              (instregex "^FCMLA_ZZZI_[HS]")>;
 
 // Floating point convert, long or narrow (F16 to F32 or F32 to F16)
 // Floating point convert to integer, F32
@@ -1623,11 +1722,15 @@ def : InstRW<[V1Write_3c_1V01], (instregex "^(FSCALE|FMULX)_ZPmZ_[HSD]",
                                            "^FMUL_ZPZ[IZ]_[HSD]")>;
 
 // Floating point multiply accumulate
+def : InstRW<[V1Wr_ZFMA, ReadDefault, V1Rd_ZFMA],
+             (instregex "^FN?ML[AS]_ZPmZZ_[HSD]",
+                        "^FN?(MAD|MSB)_ZPmZZ_[HSD]")>;
+def : InstRW<[V1Wr_ZFMA, V1Rd_ZFMA],
+             (instregex "^FML[AS]_ZZZI_[HSD]",
+                        "^FN?ML[AS]_ZPZZZ_[HSD]")>;
+
 // Floating point reciprocal step
-def : InstRW<[V1Write_4c_1V01], (instregex "^F(N?M(AD|SB)|N?ML[AS])_ZPmZZ_[HSD]$",
-                                           "^FN?ML[AS]_ZPZZZ_[HSD]",
-                                           "^FML[AS]_ZZZI_[HSD]$",
-                                           "^F(RECPS|RSQRTS)_ZZZ_[HSD]$")>;
+def : InstRW<[V1Write_4c_1V01], (instregex "^F(RECPS|RSQRTS)_ZZZ_[HSD]")>;
 
 // Floating point reciprocal estimate, F16
 def : InstRW<[V1Write_6c_4V0], (instrs FRECPE_ZZ_H, FRSQRTE_ZZ_H)>;
@@ -1681,13 +1784,13 @@ def : InstRW<[V1Write_3c_1V01], (instregex "^FEXPA_ZZ_[HSD]$",
 def : InstRW<[V1Write_4c_1V0], (instrs BFCVT_ZPmZ, BFCVTNT_ZPmZ)>;
 
 // Dot product
-def : InstRW<[V1Write_4c_1V01], (instrs BFDOT_ZZI, BFDOT_ZZZ)>;
+def : InstRW<[V1Wr_ZBFDOT, V1Rd_ZBFDOT], (instrs BFDOT_ZZI, BFDOT_ZZZ)>;
 
 // Matrix multiply accumulate
-def : InstRW<[V1Write_5c_1V01], (instrs BFMMLA_ZZZ)>;
+def : InstRW<[V1Wr_ZBFMMA, V1Rd_ZBFMMA], (instrs BFMMLA_ZZZ)>;
 
 // Multiply accumulate long
-def : InstRW<[V1Write_5c_1V01], (instregex "^BFMLAL[BT]_ZZZ(I)?$")>;
+def : InstRW<[V1Wr_ZBFMAL, V1Rd_ZBFMAL], (instregex "^BFMLAL[BT]_ZZZ(I)?$")>;
 
 
 // SVE Load instructions
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index e62437c28b863fd60dbf33e97a9f8537cca6e7c5..fe96fedcfb82dcda86d2c6e3a8310bb1f4b005f2 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -436,10 +436,10 @@ Instruction *AArch64StackTagging::collectInitializers(Instruction *StartInst,
 
 void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore,
                                     Value *Ptr, uint64_t Size) {
-  auto SetTagZeroFunc =
-      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag_zero);
-  auto StgpFunc =
-      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_stgp);
+  auto SetTagZeroFunc = Intrinsic::getOrInsertDeclaration(
+      F->getParent(), Intrinsic::aarch64_settag_zero);
+  auto StgpFunc = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                    Intrinsic::aarch64_stgp);
 
   InitializerBuilder IB(Size, DL, Ptr, SetTagFunc, SetTagZeroFunc, StgpFunc);
   bool LittleEndian =
@@ -481,8 +481,8 @@ Instruction *AArch64StackTagging::insertBaseTaggedPointer(
   assert(PrologueBB);
 
   IRBuilder<> IRB(&PrologueBB->front());
-  Function *IRG_SP =
-      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_irg_sp);
+  Function *IRG_SP = Intrinsic::getOrInsertDeclaration(
+      F->getParent(), Intrinsic::aarch64_irg_sp);
   Instruction *Base =
       IRB.CreateCall(IRG_SP, {Constant::getNullValue(IRB.getInt64Ty())});
   Base->setName("basetag");
@@ -563,8 +563,8 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
     LI = DeleteLI.get();
   }
 
-  SetTagFunc =
-      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag);
+  SetTagFunc = Intrinsic::getOrInsertDeclaration(F->getParent(),
+                                                 Intrinsic::aarch64_settag);
 
   Instruction *Base =
       insertBaseTaggedPointer(*Fn.getParent(), SInfo.AllocasToInstrument, DT);
@@ -580,7 +580,7 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
     NextTag = (NextTag + 1) % 16;
     // Replace alloca with tagp(alloca).
     IRBuilder<> IRB(Info.AI->getNextNode());
-    Function *TagP = Intrinsic::getDeclaration(
+    Function *TagP = Intrinsic::getOrInsertDeclaration(
         F->getParent(), Intrinsic::aarch64_tagp, {Info.AI->getType()});
     Instruction *TagPCall =
         IRB.CreateCall(TagP, {Constant::getNullValue(Info.AI->getType()), Base,
diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
index dd0ce1cf47a792657b88332a2e324b5079614bf5..474a543a857494307cfe9f06438bb3596fd20ebc 100644
--- a/llvm/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -1970,3 +1970,14 @@ def : RWSysReg<"HACDBSCONS_EL2",    0b11, 0b100, 0b0010, 0b0011, 0b101>;
 // v9.5a Fine Grained Write Trap EL3 (FEAT_FGWTE3)
 //                                  Op0   Op1    CRn     CRm     Op2
 def : RWSysReg<"FGWTE3_EL3",        0b11, 0b110, 0b0001, 0b0001, 0b101>;
+
+// v9.6a Memory partitioning and monitoring (FEAT_MPAM) registers
+//                                        Op0   Op1    CRn     CRm     Op2
+def : ROSysReg<"MPAMBWIDR_EL1",           0b11, 0b000, 0b1010, 0b0100, 0b101>;
+def : RWSysReg<"MPAMBW3_EL3",             0b11, 0b110, 0b1010, 0b0101, 0b100>;
+def : RWSysReg<"MPAMBW2_EL2",             0b11, 0b100, 0b1010, 0b0101, 0b100>;
+def : RWSysReg<"MPAMBW1_EL1",             0b11, 0b000, 0b1010, 0b0101, 0b100>;
+def : RWSysReg<"MPAMBW1_EL12",            0b11, 0b101, 0b1010, 0b0101, 0b100>;
+def : RWSysReg<"MPAMBW0_EL1",             0b11, 0b000, 0b1010, 0b0101, 0b101>;
+def : RWSysReg<"MPAMBWCAP_EL2",           0b11, 0b100, 0b1010, 0b0101, 0b110>;
+def : RWSysReg<"MPAMBWSM_EL1",            0b11, 0b000, 0b1010, 0b0101, 0b111>;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 7b74bb2a03a6424c92306bb98c1de532d322fbe8..91ab3fcfc4c70eae4378bbda4dda75a268a600d6 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1637,7 +1637,7 @@ static std::optional<Instruction *> instCombineSVEAllActive(IntrinsicInst &II,
     return std::nullopt;
 
   auto *Mod = II.getModule();
-  auto *NewDecl = Intrinsic::getDeclaration(Mod, IID, {II.getType()});
+  auto *NewDecl = Intrinsic::getOrInsertDeclaration(Mod, IID, {II.getType()});
   II.setCalledFunction(NewDecl);
 
   return &II;
diff --git a/llvm/lib/Target/AArch64/SMEABIPass.cpp b/llvm/lib/Target/AArch64/SMEABIPass.cpp
index 174d95333d918df9ce94c4bd56771e2c2636e28d..2ee16a873e33b8c8d10ced10162151d62a6156f2 100644
--- a/llvm/lib/Target/AArch64/SMEABIPass.cpp
+++ b/llvm/lib/Target/AArch64/SMEABIPass.cpp
@@ -71,7 +71,7 @@ void emitTPIDR2Save(Module *M, IRBuilder<> &Builder) {
 
   // A save to TPIDR2 should be followed by clearing TPIDR2_EL0.
   Function *WriteIntr =
-      Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_set_tpidr2);
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::aarch64_sme_set_tpidr2);
   Builder.CreateCall(WriteIntr->getFunctionType(), WriteIntr,
                      Builder.getInt64(0));
 }
@@ -114,7 +114,7 @@ bool SMEABI::updateNewStateFunctions(Module *M, Function *F,
     // Read TPIDR2_EL0 in PreludeBB & branch to SaveBB if not 0.
     Builder.SetInsertPoint(PreludeBB);
     Function *TPIDR2Intr =
-        Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_get_tpidr2);
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::aarch64_sme_get_tpidr2);
     auto *TPIDR2 = Builder.CreateCall(TPIDR2Intr->getFunctionType(), TPIDR2Intr,
                                       {}, "tpidr2");
     auto *Cmp = Builder.CreateCmp(ICmpInst::ICMP_NE, TPIDR2,
@@ -128,20 +128,20 @@ bool SMEABI::updateNewStateFunctions(Module *M, Function *F,
     // Enable pstate.za at the start of the function.
     Builder.SetInsertPoint(&OrigBB->front());
     Function *EnableZAIntr =
-        Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_za_enable);
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::aarch64_sme_za_enable);
     Builder.CreateCall(EnableZAIntr->getFunctionType(), EnableZAIntr);
   }
 
   if (FnAttrs.isNewZA()) {
     Function *ZeroIntr =
-        Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_zero);
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::aarch64_sme_zero);
     Builder.CreateCall(ZeroIntr->getFunctionType(), ZeroIntr,
                        Builder.getInt32(0xff));
   }
 
   if (FnAttrs.isNewZT0()) {
     Function *ClearZT0Intr =
-        Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_zero_zt);
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::aarch64_sme_zero_zt);
     Builder.CreateCall(ClearZT0Intr->getFunctionType(), ClearZT0Intr,
                        {Builder.getInt32(0)});
   }
@@ -153,8 +153,8 @@ bool SMEABI::updateNewStateFunctions(Module *M, Function *F,
       if (!T || !isa<ReturnInst>(T))
         continue;
       Builder.SetInsertPoint(T);
-      Function *DisableZAIntr =
-          Intrinsic::getDeclaration(M, Intrinsic::aarch64_sme_za_disable);
+      Function *DisableZAIntr = Intrinsic::getOrInsertDeclaration(
+          M, Intrinsic::aarch64_sme_za_disable);
       Builder.CreateCall(DisableZAIntr->getFunctionType(), DisableZAIntr);
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 25117544d6a849deffc4d48948d22ae59a86d70c..62fac085897ab6876ea1f317a5d7eafc8bf5e2fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1178,9 +1178,9 @@ def FeatureGFX9 : GCNSubtargetFeatureGeneration<"GFX9",
    FeatureAddNoCarryInsts, FeatureGFX8Insts, FeatureGFX7GFX8GFX9Insts,
    FeatureScalarFlatScratchInsts, FeatureScalarAtomics, FeatureR128A16,
    FeatureA16, FeatureSMemTimeInst, FeatureFastDenormalF32, FeatureSupportsXNACK,
-   FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess,
-   FeatureNegativeScratchOffsetBug, FeatureGWS, FeatureDefaultComponentZero,
-   FeatureVmemWriteVgprInOrder
+   FeatureUnalignedBufferAccess, FeatureUnalignedScratchAccess,
+   FeatureUnalignedDSAccess, FeatureNegativeScratchOffsetBug, FeatureGWS,
+   FeatureDefaultComponentZero,FeatureVmemWriteVgprInOrder
   ]
 >;
 
@@ -1199,9 +1199,9 @@ def FeatureGFX10 : GCNSubtargetFeatureGeneration<"GFX10",
    FeatureVOP3Literal, FeatureDPP8, FeatureExtendedImageInsts,
    FeatureNoDataDepHazard, FeaturePkFmacF16Inst,
    FeatureA16, FeatureSMemTimeInst, FeatureFastDenormalF32, FeatureG16,
-   FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess, FeatureImageInsts,
-   FeatureGDS, FeatureGWS, FeatureDefaultComponentZero,
-   FeatureMaxHardClauseLength63,
+   FeatureUnalignedBufferAccess, FeatureUnalignedScratchAccess,
+   FeatureUnalignedDSAccess, FeatureImageInsts, FeatureGDS, FeatureGWS,
+   FeatureDefaultComponentZero, FeatureMaxHardClauseLength63,
    FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF64GlobalInsts,
    FeatureAtomicFMinFMaxF32FlatInsts, FeatureAtomicFMinFMaxF64FlatInsts,
    FeatureVmemWriteVgprInOrder
@@ -1223,9 +1223,9 @@ def FeatureGFX11 : GCNSubtargetFeatureGeneration<"GFX11",
    FeatureVOP3Literal, FeatureDPP8, FeatureExtendedImageInsts,
    FeatureNoDataDepHazard, FeaturePkFmacF16Inst,
    FeatureA16, FeatureFastDenormalF32, FeatureG16,
-   FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess, FeatureGDS,
-   FeatureGWS, FeatureDefaultComponentZero,
-   FeatureMaxHardClauseLength32,
+   FeatureUnalignedBufferAccess, FeatureUnalignedScratchAccess,
+   FeatureUnalignedDSAccess, FeatureGDS, FeatureGWS,
+   FeatureDefaultComponentZero, FeatureMaxHardClauseLength32,
    FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF32FlatInsts,
    FeatureVmemWriteVgprInOrder
   ]
@@ -1246,9 +1246,9 @@ def FeatureGFX12 : GCNSubtargetFeatureGeneration<"GFX12",
    FeatureVOP3Literal, FeatureDPP8,
    FeatureNoDataDepHazard, FeaturePkFmacF16Inst,
    FeatureA16, FeatureFastDenormalF32, FeatureG16,
-   FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess,
-   FeatureTrue16BitInsts, FeatureDefaultComponentBroadcast,
-   FeatureMaxHardClauseLength32,
+   FeatureUnalignedBufferAccess, FeatureUnalignedScratchAccess,
+   FeatureUnalignedDSAccess, FeatureTrue16BitInsts,
+   FeatureDefaultComponentBroadcast, FeatureMaxHardClauseLength32,
    FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF32FlatInsts,
    FeatureAgentScopeFineGrainedRemoteMemoryAtomics
   ]
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index f408a013d7a37920f083b518d47b5c77d76c3280..ea88ed424dc597e55f67e70efebeea3088a46829 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -407,8 +407,8 @@ Value *AMDGPUAtomicOptimizerImpl::buildReduction(IRBuilder<> &B,
                                                  Value *const Identity) const {
   Type *AtomicTy = V->getType();
   Module *M = B.GetInsertBlock()->getModule();
-  Function *UpdateDPP =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, AtomicTy);
+  Function *UpdateDPP = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::amdgcn_update_dpp, AtomicTy);
 
   // Reduce within each row of 16 lanes.
   for (unsigned Idx = 0; Idx < 4; Idx++) {
@@ -439,8 +439,8 @@ Value *AMDGPUAtomicOptimizerImpl::buildReduction(IRBuilder<> &B,
 
   // Pick an arbitrary lane from 0..31 and an arbitrary lane from 32..63 and
   // combine them with a scalar operation.
-  Function *ReadLane =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, AtomicTy);
+  Function *ReadLane = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::amdgcn_readlane, AtomicTy);
   Value *Lane0 = B.CreateCall(ReadLane, {V, B.getInt32(0)});
   Value *Lane32 = B.CreateCall(ReadLane, {V, B.getInt32(32)});
   return buildNonAtomicBinOp(B, Op, Lane0, Lane32);
@@ -453,8 +453,8 @@ Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
                                             Value *Identity) const {
   Type *AtomicTy = V->getType();
   Module *M = B.GetInsertBlock()->getModule();
-  Function *UpdateDPP =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, AtomicTy);
+  Function *UpdateDPP = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::amdgcn_update_dpp, AtomicTy);
 
   for (unsigned Idx = 0; Idx < 4; Idx++) {
     V = buildNonAtomicBinOp(
@@ -513,18 +513,18 @@ Value *AMDGPUAtomicOptimizerImpl::buildShiftRight(IRBuilder<> &B, Value *V,
                                                   Value *Identity) const {
   Type *AtomicTy = V->getType();
   Module *M = B.GetInsertBlock()->getModule();
-  Function *UpdateDPP =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, AtomicTy);
+  Function *UpdateDPP = Intrinsic::getOrInsertDeclaration(
+      M, Intrinsic::amdgcn_update_dpp, AtomicTy);
   if (ST->hasDPPWavefrontShifts()) {
     // GFX9 has DPP wavefront shift operations.
     V = B.CreateCall(UpdateDPP,
                      {Identity, V, B.getInt32(DPP::WAVE_SHR1), B.getInt32(0xf),
                       B.getInt32(0xf), B.getFalse()});
   } else {
-    Function *ReadLane =
-        Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, AtomicTy);
-    Function *WriteLane =
-        Intrinsic::getDeclaration(M, Intrinsic::amdgcn_writelane, AtomicTy);
+    Function *ReadLane = Intrinsic::getOrInsertDeclaration(
+        M, Intrinsic::amdgcn_readlane, AtomicTy);
+    Function *WriteLane = Intrinsic::getOrInsertDeclaration(
+        M, Intrinsic::amdgcn_writelane, AtomicTy);
 
     // On GFX10 all DPP operations are confined to a single row. To get cross-
     // row operations we have to use permlane or readlane.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 052e1140533f3f637b6f0ff002db8274276aa675..7d3164c79089e0e7fca04d935dd87696436592a1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -119,8 +119,8 @@ public:
       return SqrtF32;
 
     LLVMContext &Ctx = Mod->getContext();
-    SqrtF32 = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_sqrt,
-                                        {Type::getFloatTy(Ctx)});
+    SqrtF32 = Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::amdgcn_sqrt,
+                                                {Type::getFloatTy(Ctx)});
     return SqrtF32;
   }
 
@@ -129,7 +129,7 @@ public:
       return LdexpF32;
 
     LLVMContext &Ctx = Mod->getContext();
-    LdexpF32 = Intrinsic::getDeclaration(
+    LdexpF32 = Intrinsic::getOrInsertDeclaration(
         Mod, Intrinsic::ldexp, {Type::getFloatTy(Ctx), Type::getInt32Ty(Ctx)});
     return LdexpF32;
   }
@@ -577,7 +577,7 @@ bool AMDGPUCodeGenPrepareImpl::promoteUniformBitreverseToI32(
 
   Type *I32Ty = getI32Ty(Builder, I.getType());
   Function *I32 =
-      Intrinsic::getDeclaration(Mod, Intrinsic::bitreverse, { I32Ty });
+      Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::bitreverse, {I32Ty});
   Value *ExtOp = Builder.CreateZExt(I.getOperand(0), I32Ty);
   Value *ExtRes = Builder.CreateCall(I32, { ExtOp });
   Value *LShrOp =
@@ -1260,8 +1260,8 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem24Impl(
   Value *FB = IsSigned ? Builder.CreateSIToFP(IB,F32Ty)
                        : Builder.CreateUIToFP(IB,F32Ty);
 
-  Function *RcpDecl = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_rcp,
-                                                Builder.getFloatTy());
+  Function *RcpDecl = Intrinsic::getOrInsertDeclaration(
+      Mod, Intrinsic::amdgcn_rcp, Builder.getFloatTy());
   Value *RCP = Builder.CreateCall(RcpDecl, { FB });
   Value *FQM = Builder.CreateFMul(FA, RCP);
 
@@ -1455,7 +1455,8 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder,
 
   // Initial estimate of inv(y).
   Value *FloatY = Builder.CreateUIToFP(Y, F32Ty);
-  Function *Rcp = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_rcp, F32Ty);
+  Function *Rcp =
+      Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::amdgcn_rcp, F32Ty);
   Value *RcpY = Builder.CreateCall(Rcp, {FloatY});
   Constant *Scale = ConstantFP::get(F32Ty, llvm::bit_cast<float>(0x4F7FFFFE));
   Value *ScaledY = Builder.CreateFMul(RcpY, Scale);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
index 45207c06a788a2fe8344c3ae0491b91364424fbc..e48fed025857fa0250a9c28b79f0f18b3534d83b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
@@ -237,7 +237,7 @@ bool optimizeSection(ArrayRef<SmallVector<IntrinsicInst *, 4>> MergeableInsts) {
       else
         NewIntrinID = Intrinsic::amdgcn_image_msaa_load_2darraymsaa;
 
-      Function *NewIntrin = Intrinsic::getDeclaration(
+      Function *NewIntrin = Intrinsic::getOrInsertDeclaration(
           IIList.front()->getModule(), NewIntrinID, OverloadTys);
       Args[ImageDimIntr->DMaskIndex] =
           ConstantInt::get(DMask->getType(), NewMaskVal);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index ecb4d4fa5d5c3964060b7822e8d7449923f51308..6a5a48778197e47f0529f6a114d1275fa9984956 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -130,7 +130,8 @@ static std::optional<Instruction *> modifyIntrinsicCall(
   // Modify arguments and types
   Func(Args, ArgTys);
 
-  Function *I = Intrinsic::getDeclaration(OldIntr.getModule(), NewIntr, ArgTys);
+  Function *I =
+      Intrinsic::getOrInsertDeclaration(OldIntr.getModule(), NewIntr, ArgTys);
 
   CallInst *NewCall = IC.Builder.CreateCall(I, Args);
   NewCall->takeName(&OldIntr);
@@ -502,7 +503,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
       if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
         break;
 
-      Function *NewDecl = Intrinsic::getDeclaration(
+      Function *NewDecl = Intrinsic::getOrInsertDeclaration(
           SrcCI->getModule(), Intrinsic::amdgcn_rsq, {SrcCI->getType()});
 
       InnerFMF |= FMF;
@@ -527,7 +528,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 
     // f16 amdgcn.sqrt is identical to regular sqrt.
     if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
-      Function *NewDecl = Intrinsic::getDeclaration(
+      Function *NewDecl = Intrinsic::getOrInsertDeclaration(
           II.getModule(), Intrinsic::sqrt, {II.getType()});
       II.setCalledFunction(NewDecl);
       return &II;
@@ -614,7 +615,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     Value *Src1 = II.getArgOperand(1);
     const ConstantInt *CMask = dyn_cast<ConstantInt>(Src1);
     if (CMask) {
-      II.setCalledOperand(Intrinsic::getDeclaration(
+      II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
           II.getModule(), Intrinsic::is_fpclass, Src0->getType()));
 
       // Clamp any excess bits, as they're illegal for the generic intrinsic.
@@ -890,7 +891,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
         // register (which contains the bitmask of live threads). So a
         // comparison that always returns true is the same as a read of the
         // EXEC register.
-        Function *NewF = Intrinsic::getDeclaration(
+        Function *NewF = Intrinsic::getOrInsertDeclaration(
             II.getModule(), Intrinsic::read_register, II.getType());
         Metadata *MDArgs[] = {MDString::get(II.getContext(), "exec")};
         MDNode *MD = MDNode::get(II.getContext(), MDArgs);
@@ -989,7 +990,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
       } else if (!Ty->isFloatTy() && !Ty->isDoubleTy() && !Ty->isHalfTy())
         break;
 
-      Function *NewF = Intrinsic::getDeclaration(
+      Function *NewF = Intrinsic::getOrInsertDeclaration(
           II.getModule(), NewIID, {II.getType(), SrcLHS->getType()});
       Value *Args[] = {SrcLHS, SrcRHS,
                        ConstantInt::get(CC->getType(), SrcPred)};
@@ -1205,7 +1206,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     // If we can prove we don't have one of the special cases then we can use a
     // normal fma instead.
     if (canSimplifyLegacyMulToMul(II, Op0, Op1, IC)) {
-      II.setCalledOperand(Intrinsic::getDeclaration(
+      II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
           II.getModule(), Intrinsic::fma, II.getType()));
       return &II;
     }
@@ -1401,7 +1402,7 @@ static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
       Args[0] = IC.Builder.CreateShuffleVector(II.getOperand(0), EltMask);
   }
 
-  Function *NewIntrin = Intrinsic::getDeclaration(
+  Function *NewIntrin = Intrinsic::getOrInsertDeclaration(
       II.getModule(), II.getIntrinsicID(), OverloadTys);
   CallInst *NewCall = IC.Builder.CreateCall(NewIntrin, Args);
   NewCall->takeName(&II);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 5be0a049cc5827f36d28cc754a3471e7d10a51ff..800bdbe04cf70d3df682ca0b9a5f0b9b247a4e23 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1413,50 +1413,101 @@ bool AMDGPUInstructionSelector::selectIntrinsicCmp(MachineInstr &I) const {
   return true;
 }
 
+// Ballot has to zero bits in input lane-mask that are zero in current exec,
+// Done as AND with exec. For inputs that are results of instruction that
+// implicitly use same exec, for example compares in same basic block or SCC to
+// VCC copy, use copy.
+static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
+                                    MachineBasicBlock *MBB) {
+  MachineInstr *MI = MRI.getVRegDef(Reg);
+  if (MI->getParent() != MBB)
+    return false;
+
+  // Lane mask generated by SCC to VCC copy.
+  if (MI->getOpcode() == AMDGPU::COPY) {
+    auto DstRB = MRI.getRegBankOrNull(MI->getOperand(0).getReg());
+    auto SrcRB = MRI.getRegBankOrNull(MI->getOperand(1).getReg());
+    if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
+        SrcRB->getID() == AMDGPU::SGPRRegBankID)
+      return true;
+  }
+
+  // Lane mask generated using compare with same exec.
+  if (isa<GAnyCmp>(MI))
+    return true;
+
+  Register LHS, RHS;
+  // Look through AND.
+  if (mi_match(Reg, MRI, m_GAnd(m_Reg(LHS), m_Reg(RHS))))
+    return isLaneMaskFromSameBlock(LHS, MRI, MBB) ||
+           isLaneMaskFromSameBlock(RHS, MRI, MBB);
+
+  return false;
+}
+
 bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
   const DebugLoc &DL = I.getDebugLoc();
   Register DstReg = I.getOperand(0).getReg();
-  const unsigned Size = MRI->getType(DstReg).getSizeInBits();
-  const bool Is64 = Size == 64;
-  const bool IsWave32 = (STI.getWavefrontSize() == 32);
+  Register SrcReg = I.getOperand(2).getReg();
+  const unsigned BallotSize = MRI->getType(DstReg).getSizeInBits();
+  const unsigned WaveSize = STI.getWavefrontSize();
 
   // In the common case, the return type matches the wave size.
   // However we also support emitting i64 ballots in wave32 mode.
-  if (Size != STI.getWavefrontSize() && (!Is64 || !IsWave32))
+  if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
     return false;
 
   std::optional<ValueAndVReg> Arg =
-      getIConstantVRegValWithLookThrough(I.getOperand(2).getReg(), *MRI);
+      getIConstantVRegValWithLookThrough(SrcReg, *MRI);
+
+  Register Dst = DstReg;
+  // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
+  if (BallotSize != WaveSize) {
+    Dst = MRI->createVirtualRegister(TRI.getBoolRC());
+  }
 
-  const auto BuildCopy = [&](Register SrcReg) {
-    if (Size == STI.getWavefrontSize()) {
-      BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg)
-          .addReg(SrcReg);
-      return;
+  if (Arg) {
+    const int64_t Value = Arg->Value.getZExtValue();
+    if (Value == 0) {
+      // Dst = S_MOV 0
+      unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
+      BuildMI(*BB, &I, DL, TII.get(Opcode), Dst).addImm(0);
+    } else {
+      // Dst = COPY EXEC
+      assert(Value == 1);
+      BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst).addReg(TRI.getExec());
     }
+    if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI))
+      return false;
+  } else {
+    if (isLaneMaskFromSameBlock(SrcReg, *MRI, BB)) {
+      // Dst = COPY SrcReg
+      BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst).addReg(SrcReg);
+      if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI))
+        return false;
+    } else {
+      // Dst = S_AND SrcReg, EXEC
+      unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
+      auto And = BuildMI(*BB, &I, DL, TII.get(AndOpc), Dst)
+                     .addReg(SrcReg)
+                     .addReg(TRI.getExec())
+                     .setOperandDead(3); // Dead scc
+      if (!constrainSelectedInstRegOperands(*And, TII, TRI, RBI))
+        return false;
+    }
+  }
 
-    // If emitting a i64 ballot in wave32, fill the upper bits with zeroes.
+  // i64 ballot on Wave32: zero-extend i32 ballot to i64.
+  if (BallotSize != WaveSize) {
     Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
     BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg).addImm(0);
     BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
-        .addReg(SrcReg)
+        .addReg(Dst)
         .addImm(AMDGPU::sub0)
         .addReg(HiReg)
         .addImm(AMDGPU::sub1);
-  };
-
-  if (Arg) {
-    const int64_t Value = Arg->Value.getSExtValue();
-    if (Value == 0) {
-      unsigned Opcode = Is64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
-      BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg).addImm(0);
-    } else if (Value == -1) // all ones
-      BuildCopy(IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC);
-    else
-      return false;
-  } else
-    BuildCopy(I.getOperand(2).getReg());
+  }
 
   I.eraseFromParent();
   return true;
@@ -1504,8 +1555,8 @@ bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
     MIB.addImm(MFI->getLDSSize());
   } else {
     Module *M = MF->getFunction().getParent();
-    const GlobalValue *GV
-      = Intrinsic::getDeclaration(M, Intrinsic::amdgcn_groupstaticsize);
+    const GlobalValue *GV =
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_groupstaticsize);
     MIB.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_LO);
   }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index e01c9dc66a3f1fb6485820fce8feba7349d44e73..eb553ae4eb80ff975870456b4326d79f98a44792 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -753,7 +753,7 @@ bool AMDGPULibCalls::fold(CallInst *CI) {
         CI->setArgOperand(1, SplatArg1);
       }
 
-      CI->setCalledFunction(Intrinsic::getDeclaration(
+      CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
           CI->getModule(), Intrinsic::ldexp,
           {CI->getType(), CI->getArgOperand(1)->getType()}));
       return true;
@@ -1034,7 +1034,8 @@ bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
   // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
   FunctionCallee ExpExpr;
   if (ShouldUseIntrinsic)
-    ExpExpr = Intrinsic::getDeclaration(M, Intrinsic::exp2, {FPOp->getType()});
+    ExpExpr = Intrinsic::getOrInsertDeclaration(M, Intrinsic::exp2,
+                                                {FPOp->getType()});
   else {
     ExpExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
     if (!ExpExpr)
@@ -1108,8 +1109,8 @@ bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
   if (needlog) {
     FunctionCallee LogExpr;
     if (ShouldUseIntrinsic) {
-      LogExpr =
-          Intrinsic::getDeclaration(M, Intrinsic::log2, {FPOp->getType()});
+      LogExpr = Intrinsic::getOrInsertDeclaration(M, Intrinsic::log2,
+                                                  {FPOp->getType()});
     } else {
       LogExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
       if (!LogExpr)
@@ -1298,8 +1299,8 @@ void AMDGPULibCalls::replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B,
     }
   }
 
-  CI->setCalledFunction(
-      Intrinsic::getDeclaration(CI->getModule(), IntrID, {CI->getType()}));
+  CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(
+      CI->getModule(), IntrID, {CI->getType()}));
 }
 
 bool AMDGPULibCalls::tryReplaceLibcallWithSimpleIntrinsic(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 51a5b7702c00938672677a11a9ce59922298c3ab..ff5eb81490106f8dfdf18addccf14a1167f93db4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -285,8 +285,8 @@ class AMDGPULowerModuleLDS {
     BasicBlock *Entry = &Func->getEntryBlock();
     IRBuilder<> Builder(Entry, Entry->getFirstNonPHIIt());
 
-    Function *Decl =
-        Intrinsic::getDeclaration(Func->getParent(), Intrinsic::donothing, {});
+    Function *Decl = Intrinsic::getOrInsertDeclaration(
+        Func->getParent(), Intrinsic::donothing, {});
 
     Value *UseInstance[1] = {
         Builder.CreateConstInBoundsGEP1_32(SGV->getValueType(), SGV, 0)};
@@ -529,8 +529,8 @@ public:
     // block to spare deduplicating it later.
     auto [It, Inserted] = tableKernelIndexCache.try_emplace(F);
     if (Inserted) {
-      Function *Decl =
-          Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_lds_kernel_id, {});
+      Function *Decl = Intrinsic::getOrInsertDeclaration(
+          &M, Intrinsic::amdgcn_lds_kernel_id, {});
 
       auto InsertAt = F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca();
       IRBuilder<> Builder(&*InsertAt);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 24bfbff41ec5c050ca1b63f99466a10fc659eb11..63da3443479be3bb5aa0e9722f2cd7ed5aae44dc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -973,10 +973,10 @@ AMDGPUPromoteAllocaImpl::getLocalSizeYZ(IRBuilder<> &Builder) {
   const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
 
   if (!IsAMDHSA) {
-    Function *LocalSizeYFn =
-        Intrinsic::getDeclaration(Mod, Intrinsic::r600_read_local_size_y);
-    Function *LocalSizeZFn =
-        Intrinsic::getDeclaration(Mod, Intrinsic::r600_read_local_size_z);
+    Function *LocalSizeYFn = Intrinsic::getOrInsertDeclaration(
+        Mod, Intrinsic::r600_read_local_size_y);
+    Function *LocalSizeZFn = Intrinsic::getOrInsertDeclaration(
+        Mod, Intrinsic::r600_read_local_size_z);
 
     CallInst *LocalSizeY = Builder.CreateCall(LocalSizeYFn, {});
     CallInst *LocalSizeZ = Builder.CreateCall(LocalSizeZFn, {});
@@ -1022,7 +1022,7 @@ AMDGPUPromoteAllocaImpl::getLocalSizeYZ(IRBuilder<> &Builder) {
   //   } hsa_kernel_dispatch_packet_t
   //
   Function *DispatchPtrFn =
-      Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_dispatch_ptr);
+      Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::amdgcn_dispatch_ptr);
 
   CallInst *DispatchPtr = Builder.CreateCall(DispatchPtrFn, {});
   DispatchPtr->addRetAttr(Attribute::NoAlias);
@@ -1082,7 +1082,7 @@ Value *AMDGPUPromoteAllocaImpl::getWorkitemID(IRBuilder<> &Builder,
     llvm_unreachable("invalid dimension");
   }
 
-  Function *WorkitemIdFn = Intrinsic::getDeclaration(Mod, IntrID);
+  Function *WorkitemIdFn = Intrinsic::getOrInsertDeclaration(Mod, IntrID);
   CallInst *CI = Builder.CreateCall(WorkitemIdFn);
   ST.makeLIDRangeMetadata(CI);
   F->removeFnAttr(AttrName);
@@ -1564,7 +1564,7 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToLDS(AllocaInst &I,
       continue;
     case Intrinsic::objectsize: {
       Value *Src = Intr->getOperand(0);
-      Function *ObjectSize = Intrinsic::getDeclaration(
+      Function *ObjectSize = Intrinsic::getOrInsertDeclaration(
           Mod, Intrinsic::objectsize,
           {Intr->getType(),
            PointerType::get(Context, AMDGPUAS::LOCAL_ADDRESS)});
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
index a62c72d124825eb1c274e0dd537631ed976ada7f..7d86e0c72bd0759523ec0b51e8d767abf341baf5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
@@ -482,6 +482,29 @@ void SplitGraph::Node::visitAllDependencies(
   }
 }
 
+/// Checks if \p I has MD_callees and if it does, parse it and put the function
+/// in \p Callees.
+///
+/// \returns true if there was metadata and it was parsed correctly. false if
+/// there was no MD or if it contained unknown entries and parsing failed.
+/// If this returns false, \p Callees will contain incomplete information
+/// and must not be used.
+static bool handleCalleesMD(const Instruction &I,
+                            SetVector<Function *> &Callees) {
+  auto *MD = I.getMetadata(LLVMContext::MD_callees);
+  if (!MD)
+    return false;
+
+  for (const auto &Op : MD->operands()) {
+    Function *Callee = mdconst::extract_or_null<Function>(Op);
+    if (!Callee)
+      return false;
+    Callees.insert(Callee);
+  }
+
+  return true;
+}
+
 void SplitGraph::buildGraph(CallGraph &CG) {
   SplitModuleTimer SMT("buildGraph", "graph construction");
   LLVM_DEBUG(
@@ -519,28 +542,38 @@ void SplitGraph::buildGraph(CallGraph &CG) {
                  Fn.printAsOperand(dbgs());
                  dbgs() << " - analyzing function\n");
 
-      bool HasIndirectCall = false;
+      SetVector<Function *> KnownCallees;
+      bool HasUnknownIndirectCall = false;
       for (const auto &Inst : instructions(Fn)) {
         // look at all calls without a direct callee.
-        if (const auto *CB = dyn_cast<CallBase>(&Inst);
-            CB && !CB->getCalledFunction()) {
-          // inline assembly can be ignored, unless InlineAsmIsIndirectCall is
-          // true.
-          if (CB->isInlineAsm()) {
-            LLVM_DEBUG(dbgs() << "    found inline assembly\n");
-            continue;
-          }
-
-          // everything else is handled conservatively.
-          HasIndirectCall = true;
-          break;
+        const auto *CB = dyn_cast<CallBase>(&Inst);
+        if (!CB || CB->getCalledFunction())
+          continue;
+
+        // inline assembly can be ignored, unless InlineAsmIsIndirectCall is
+        // true.
+        if (CB->isInlineAsm()) {
+          LLVM_DEBUG(dbgs() << "    found inline assembly\n");
+          continue;
         }
+
+        if (handleCalleesMD(Inst, KnownCallees))
+          continue;
+        // If we failed to parse any !callees MD, or some was missing,
+        // the entire KnownCallees list is now unreliable.
+        KnownCallees.clear();
+
+        // Everything else is handled conservatively. If we fall into the
+        // conservative case don't bother analyzing further.
+        HasUnknownIndirectCall = true;
+        break;
       }
 
-      if (HasIndirectCall) {
+      if (HasUnknownIndirectCall) {
         LLVM_DEBUG(dbgs() << "    indirect call found\n");
         FnsWithIndirectCalls.push_back(&Fn);
-      }
+      } else if (!KnownCallees.empty())
+        DirectCallees.insert(KnownCallees.begin(), KnownCallees.end());
     }
 
     Node &N = getNode(Cache, Fn);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
index 4669bb45473cb057376326698443a6159c948b69..cfce56f0bfe968571e31908099c570437117aac8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
@@ -336,8 +336,8 @@ static void markUsedByKernel(Function *Func, GlobalVariable *SGV) {
   BasicBlock *Entry = &Func->getEntryBlock();
   IRBuilder<> Builder(Entry, Entry->getFirstNonPHIIt());
 
-  Function *Decl =
-      Intrinsic::getDeclaration(Func->getParent(), Intrinsic::donothing, {});
+  Function *Decl = Intrinsic::getOrInsertDeclaration(Func->getParent(),
+                                                     Intrinsic::donothing, {});
 
   Value *UseInstance[1] = {
       Builder.CreateConstInBoundsGEP1_32(SGV->getValueType(), SGV, 0)};
@@ -922,7 +922,8 @@ void AMDGPUSwLowerLDS::lowerKernelLDSAccesses(Function *Func,
       StringRef("__asan_free_impl"),
       FunctionType::get(IRB.getVoidTy(), {Int64Ty, Int64Ty}, false));
   Value *ReturnAddr = IRB.CreateCall(
-      Intrinsic::getDeclaration(&M, Intrinsic::returnaddress), IRB.getInt32(0));
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::returnaddress),
+      IRB.getInt32(0));
   Value *RAPToInt = IRB.CreatePtrToInt(ReturnAddr, Int64Ty);
   Value *MallocPtrToInt = IRB.CreatePtrToInt(LoadMallocPtr, Int64Ty);
   IRB.CreateCall(AsanFreeFunc, {MallocPtrToInt, RAPToInt});
@@ -1055,8 +1056,8 @@ void AMDGPUSwLowerLDS::lowerNonKernelLDSAccesses(
   SetVector<Instruction *> LDSInstructions;
   getLDSMemoryInstructions(Func, LDSInstructions);
 
-  Function *Decl =
-      Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_lds_kernel_id, {});
+  Function *Decl = Intrinsic::getOrInsertDeclaration(
+      &M, Intrinsic::amdgcn_lds_kernel_id, {});
   auto *KernelId = IRB.CreateCall(Decl, {});
   GlobalVariable *LDSBaseTable = NKLDSParams.LDSBaseTable;
   GlobalVariable *LDSOffsetTable = NKLDSParams.LDSOffsetTable;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 23ee0c3e896eb3b5ba6b34a10ef26c39158f08a8..16e23879cd735c89ce62b224a28419d735e919d1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1718,6 +1718,17 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
     MFI->reserveWWMRegister(ParsedReg);
   }
 
+  for (const auto &[_, Info] : PFS.VRegInfosNamed) {
+    for (uint8_t Flag : Info->Flags) {
+      MFI->setFlag(Info->VReg, Flag);
+    }
+  }
+  for (const auto &[_, Info] : PFS.VRegInfos) {
+    for (uint8_t Flag : Info->Flags) {
+      MFI->setFlag(Info->VReg, Flag);
+    }
+  }
+
   auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,
                                    const TargetRegisterClass &RC,
                                    ArgDescriptor &Arg, unsigned UserSGPRs,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 3f4f42377d56ee05a480dcefa433756cb8d04aee..8f9495d83cde2dc6d9766555c22d1db12544b25d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -387,8 +387,8 @@ bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
   // them later if they may access private memory. We don't have enough context
   // here, and legalization can handle it.
   if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
-    return (Alignment >= 4 || ST->hasUnalignedScratchAccess()) &&
-      ChainSizeInBytes <= ST->getMaxPrivateElementSize();
+    return (Alignment >= 4 || ST->hasUnalignedScratchAccessEnabled()) &&
+           ChainSizeInBytes <= ST->getMaxPrivateElementSize();
   }
   return true;
 }
@@ -1112,8 +1112,8 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
     if (!AMDGPU::isExtendedGlobalAddrSpace(NewAS))
       return nullptr;
     Module *M = II->getModule();
-    Function *NewDecl = Intrinsic::getDeclaration(M, II->getIntrinsicID(),
-                                                  {DestTy, SrcTy, DestTy});
+    Function *NewDecl = Intrinsic::getOrInsertDeclaration(
+        M, II->getIntrinsicID(), {DestTy, SrcTy, DestTy});
     II->setArgOperand(0, NewV);
     II->setCalledFunction(NewDecl);
     return II;
@@ -1314,6 +1314,11 @@ static unsigned getCallArgsTotalAllocaSize(const CallBase *CB,
   return AllocaSize;
 }
 
+int GCNTTIImpl::getInliningLastCallToStaticBonus() const {
+  return BaseT::getInliningLastCallToStaticBonus() *
+         getInliningThresholdMultiplier();
+}
+
 unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
   unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, this);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 30da002376251c701d3f1834c77a4ab9fbe3272d..022af501289af83a8a6fd04b9d440436ac043ad6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -243,6 +243,7 @@ public:
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
 
+  int getInliningLastCallToStaticBonus() const;
   unsigned getInliningThresholdMultiplier() const { return 11; }
   unsigned adjustInliningThreshold(const CallBase *CB) const;
   unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
index 3758c768b8673f6148de9301ecb6956d5605becf..59cc61e347bc0a61746a474c85cd34e02f678a0f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
@@ -295,8 +295,8 @@ bool AMDGPUUnifyDivergentExitNodesImpl::run(Function &F, DominatorTree *DT,
       // Remove and delete the unreachable inst.
       UnreachableBlock->getTerminator()->eraseFromParent();
 
-      Function *UnreachableIntrin =
-        Intrinsic::getDeclaration(F.getParent(), Intrinsic::amdgcn_unreachable);
+      Function *UnreachableIntrin = Intrinsic::getOrInsertDeclaration(
+          F.getParent(), Intrinsic::amdgcn_unreachable);
 
       // Insert a call to an intrinsic tracking that this is an unreachable
       // point, in case we want to kill the active lanes or something later.
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 19458126093167ad527f543bdac1b01b8733a2bd..1ea3beb2855d6941406ea468034cd19952c6bd54 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -591,6 +591,10 @@ public:
     return UnalignedScratchAccess;
   }
 
+  bool hasUnalignedScratchAccessEnabled() const {
+    return UnalignedScratchAccess && UnalignedAccessMode;
+  }
+
   bool hasUnalignedAccessMode() const {
     return UnalignedAccessMode;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index edd881c84078c6a2d81521b84354732fbd88e5cf..a7f2b66e3cd1168fca53c1ac509de6289b27fc35 100644
--- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -117,13 +117,15 @@ void SIAnnotateControlFlow::initialize(Module &M, const GCNSubtarget &ST) {
   BoolUndef = PoisonValue::get(Boolean);
   IntMaskZero = ConstantInt::get(IntMask, 0);
 
-  If = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_if, { IntMask });
-  Else = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_else,
-                                   { IntMask, IntMask });
-  IfBreak = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_if_break,
-                                      { IntMask });
-  Loop = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_loop, { IntMask });
-  EndCf = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_end_cf, { IntMask });
+  If = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_if, {IntMask});
+  Else = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_else,
+                                           {IntMask, IntMask});
+  IfBreak = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_if_break,
+                                              {IntMask});
+  Loop =
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_loop, {IntMask});
+  EndCf = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::amdgcn_end_cf,
+                                            {IntMask});
 }
 
 /// Is the branch condition uniform or did the StructurizeCFG pass
diff --git a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
index 0d3a221970bf85e411408260c70e03a6c9abf29f..ddfa7790cabc6b3698af2a5ea2f3f625aa91b59e 100644
--- a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
@@ -227,11 +227,9 @@ void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
                            : LaneBitmask::getAll();
     RegUse &Map = MO.isDef() ? Defs : Uses;
 
-    auto Loc = Map.find(Reg);
     unsigned State = getMopState(MO);
-    if (Loc == Map.end()) {
-      Map[Reg] = std::pair(State, Mask);
-    } else {
+    auto [Loc, Inserted] = Map.try_emplace(Reg, State, Mask);
+    if (!Inserted) {
       Loc->second.first |= State;
       Loc->second.second |= Mask;
     }
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 3d8e03521e2b905e5a43cbd10268ec6be280c0ba..8c197f231496128da1fa065e010278d5c8872b84 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1824,26 +1824,16 @@ bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
            Subtarget->hasUnalignedDSAccessEnabled();
   }
 
-  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
-    bool AlignedBy4 = Alignment >= Align(4);
-    if (IsFast)
-      *IsFast = AlignedBy4;
-
-    return AlignedBy4 ||
-           Subtarget->enableFlatScratch() ||
-           Subtarget->hasUnalignedScratchAccess();
-  }
-
   // FIXME: We have to be conservative here and assume that flat operations
   // will access scratch.  If we had access to the IR function, then we
   // could determine if any private memory was used in the function.
-  if (AddrSpace == AMDGPUAS::FLAT_ADDRESS &&
-      !Subtarget->hasUnalignedScratchAccess()) {
+  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
+      AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
     bool AlignedBy4 = Alignment >= Align(4);
     if (IsFast)
       *IsFast = AlignedBy4;
 
-    return AlignedBy4;
+    return AlignedBy4 || Subtarget->hasUnalignedScratchAccessEnabled();
   }
 
   // So long as they are correct, wide global memory operations perform better
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 15f4114826e4019658209f8d61ae0fa1ff19cdc9..8f1757db8a85f5e89e5d317206e76a2c8856437d 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -41,17 +41,18 @@ using namespace llvm;
 
 #define DEBUG_TYPE "si-insert-waitcnts"
 
-DEBUG_COUNTER(ForceExpCounter, DEBUG_TYPE"-forceexp",
+DEBUG_COUNTER(ForceExpCounter, DEBUG_TYPE "-forceexp",
               "Force emit s_waitcnt expcnt(0) instrs");
-DEBUG_COUNTER(ForceLgkmCounter, DEBUG_TYPE"-forcelgkm",
+DEBUG_COUNTER(ForceLgkmCounter, DEBUG_TYPE "-forcelgkm",
               "Force emit s_waitcnt lgkmcnt(0) instrs");
-DEBUG_COUNTER(ForceVMCounter, DEBUG_TYPE"-forcevm",
+DEBUG_COUNTER(ForceVMCounter, DEBUG_TYPE "-forcevm",
               "Force emit s_waitcnt vmcnt(0) instrs");
 
-static cl::opt<bool> ForceEmitZeroFlag(
-  "amdgpu-waitcnt-forcezero",
-  cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
-  cl::init(false), cl::Hidden);
+static cl::opt<bool>
+    ForceEmitZeroFlag("amdgpu-waitcnt-forcezero",
+                      cl::desc("Force all waitcnt instrs to be emitted as "
+                               "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
+                      cl::init(false), cl::Hidden);
 
 namespace {
 // Class of object that encapsulates latest instruction counter score
@@ -611,9 +612,6 @@ private:
 
   MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
 
-  // ForceEmitZeroWaitcnts: force all waitcnts insts to be s_waitcnt 0
-  // because of amdgpu-waitcnt-forcezero flag
-  bool ForceEmitZeroWaitcnts;
   bool ForceEmitWaitcnt[NUM_INST_CNTS];
 
   // In any given run of this pass, WCG will point to one of these two
@@ -825,7 +823,7 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
   if (T == EXP_CNT) {
     // Put score on the source vgprs. If this is a store, just use those
     // specific register(s).
-    if (TII->isDS(Inst) && (Inst.mayStore() || Inst.mayLoad())) {
+    if (TII->isDS(Inst) && Inst.mayLoadOrStore()) {
       // All GDS operations must protect their address register (same as
       // export.)
       if (const auto *AddrOp = TII->getNamedOperand(Inst, AMDGPU::OpName::addr))
@@ -1599,9 +1597,7 @@ static bool callWaitsOnFunctionEntry(const MachineInstr &MI) {
 
 /// \returns true if the callee is expected to wait for any outstanding waits
 /// before returning.
-static bool callWaitsOnFunctionReturn(const MachineInstr &MI) {
-  return true;
-}
+static bool callWaitsOnFunctionReturn(const MachineInstr &MI) { return true; }
 
 ///  Generate s_waitcnt instruction to be placed before cur_Inst.
 ///  Instructions of a given type are returned in order,
@@ -1828,7 +1824,9 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
   // Verify that the wait is actually needed.
   ScoreBrackets.simplifyWaitcnt(Wait);
 
-  if (ForceEmitZeroWaitcnts)
+  // When forcing emit, we need to skip terminators because that would break the
+  // terminators of the MBB if we emit a waitcnt between terminators.
+  if (ForceEmitZeroFlag && !MI.isTerminator())
     Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
 
   if (ForceEmitWaitcnt[LOAD_CNT])
@@ -2424,7 +2422,6 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
     WCG = &WCGPreGFX12;
   }
 
-  ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
   for (auto T : inst_counter_types())
     ForceEmitWaitcnt[T] = false;
 
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
index 50f536c532afc7d410197f4c90004b48f27c92fb..162270524263cd0946cf3d772c82eeef11e4ee73 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
@@ -189,7 +189,7 @@ void SIOptimizeVGPRLiveRange::collectElseRegionBlocks(
   unsigned Cur = 0;
   while (MBB) {
     for (auto *Pred : MBB->predecessors()) {
-      if (Pred != Flow && !Blocks.contains(Pred))
+      if (Pred != Flow)
         Blocks.insert(Pred);
     }
 
@@ -407,10 +407,8 @@ void SIOptimizeVGPRLiveRange::updateLiveRangeInThenRegion(
   while (!WorkList.empty()) {
     auto *MBB = WorkList.pop_back_val();
     for (auto *Succ : MBB->successors()) {
-      if (Succ != Flow && !Blocks.contains(Succ)) {
+      if (Succ != Flow && Blocks.insert(Succ))
         WorkList.push_back(Succ);
-        Blocks.insert(Succ);
-      }
     }
   }
 
diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
index 3bf2ea0f9e53ef3cd422ef82d1ee80206784e399..b17fb7e70826b577edb0fda349ac10be053e0183 100644
--- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
@@ -215,13 +215,10 @@ bool SIPreAllocateWWMRegs::runOnMachineFunction(MachineFunction &MF) {
   for (MachineBasicBlock *MBB : RPOT) {
     bool InWWM = false;
     for (MachineInstr &MI : *MBB) {
-      if (MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B32)
-        RegsAssigned |= processDef(MI.getOperand(0));
-
       if (MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR) {
-        if (!PreallocateSGPRSpillVGPRs)
-          continue;
-        RegsAssigned |= processDef(MI.getOperand(0));
+        if (PreallocateSGPRSpillVGPRs)
+          RegsAssigned |= processDef(MI.getOperand(0));
+        continue;
       }
 
       if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 1334029544f999ca7a7329bfdf6660c9f3673bd7..701084844cd9b44ff1205b9dc2a9d0e24e33fcc0 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -15,19 +15,13 @@
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/TargetSchedule.h"
+#include "llvm/Support/BranchProbability.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "si-pre-emit-peephole"
 
-static unsigned SkipThreshold;
-
-static cl::opt<unsigned, true> SkipThresholdFlag(
-    "amdgpu-skip-threshold", cl::Hidden,
-    cl::desc(
-        "Number of instructions before jumping over divergent control flow"),
-    cl::location(SkipThreshold), cl::init(12));
-
 namespace {
 
 class SIPreEmitPeephole : public MachineFunctionPass {
@@ -41,7 +35,8 @@ private:
                             MachineBasicBlock *&TrueMBB,
                             MachineBasicBlock *&FalseMBB,
                             SmallVectorImpl<MachineOperand> &Cond);
-  bool mustRetainExeczBranch(const MachineBasicBlock &From,
+  bool mustRetainExeczBranch(const MachineInstr &Branch,
+                             const MachineBasicBlock &From,
                              const MachineBasicBlock &To) const;
   bool removeExeczBranch(MachineInstr &MI, MachineBasicBlock &SrcMBB);
 
@@ -304,11 +299,58 @@ bool SIPreEmitPeephole::getBlockDestinations(
   return true;
 }
 
+namespace {
+class BranchWeightCostModel {
+  const SIInstrInfo &TII;
+  const TargetSchedModel &SchedModel;
+  BranchProbability BranchProb;
+  static constexpr uint64_t BranchNotTakenCost = 1;
+  uint64_t BranchTakenCost;
+  uint64_t ThenCyclesCost = 0;
+
+public:
+  BranchWeightCostModel(const SIInstrInfo &TII, const MachineInstr &Branch,
+                        const MachineBasicBlock &Succ)
+      : TII(TII), SchedModel(TII.getSchedModel()) {
+    const MachineBasicBlock &Head = *Branch.getParent();
+    const auto *FromIt = find(Head.successors(), &Succ);
+    assert(FromIt != Head.succ_end());
+
+    BranchProb = Head.getSuccProbability(FromIt);
+    if (BranchProb.isUnknown())
+      BranchProb = BranchProbability::getZero();
+    BranchTakenCost = SchedModel.computeInstrLatency(&Branch);
+  }
+
+  bool isProfitable(const MachineInstr &MI) {
+    if (TII.isWaitcnt(MI.getOpcode()))
+      return false;
+
+    ThenCyclesCost += SchedModel.computeInstrLatency(&MI);
+
+    // Consider `P = N/D` to be the probability of execz being false (skipping
+    // the then-block) The transformation is profitable if always executing the
+    // 'then' block is cheaper than executing sometimes 'then' and always
+    // executing s_cbranch_execz:
+    // * ThenCost <= P*ThenCost + (1-P)*BranchTakenCost + P*BranchNotTakenCost
+    // * (1-P) * ThenCost <= (1-P)*BranchTakenCost + P*BranchNotTakenCost
+    // * (D-N)/D * ThenCost <= (D-N)/D * BranchTakenCost + N/D *
+    // BranchNotTakenCost
+    uint64_t Numerator = BranchProb.getNumerator();
+    uint64_t Denominator = BranchProb.getDenominator();
+    return (Denominator - Numerator) * ThenCyclesCost <=
+           ((Denominator - Numerator) * BranchTakenCost +
+            Numerator * BranchNotTakenCost);
+  }
+};
+
 bool SIPreEmitPeephole::mustRetainExeczBranch(
-    const MachineBasicBlock &From, const MachineBasicBlock &To) const {
-  unsigned NumInstr = 0;
-  const MachineFunction *MF = From.getParent();
+    const MachineInstr &Branch, const MachineBasicBlock &From,
+    const MachineBasicBlock &To) const {
+  assert(is_contained(Branch.getParent()->successors(), &From));
+  BranchWeightCostModel CostModel{*TII, Branch, From};
 
+  const MachineFunction *MF = From.getParent();
   for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
        MBBI != End && MBBI != ToI; ++MBBI) {
     const MachineBasicBlock &MBB = *MBBI;
@@ -326,23 +368,22 @@ bool SIPreEmitPeephole::mustRetainExeczBranch(
       if (TII->hasUnwantedEffectsWhenEXECEmpty(MI))
         return true;
 
-      // These instructions are potentially expensive even if EXEC = 0.
-      if (TII->isSMRD(MI) || TII->isVMEM(MI) || TII->isFLAT(MI) ||
-          TII->isDS(MI) || TII->isWaitcnt(MI.getOpcode()))
-        return true;
-
-      ++NumInstr;
-      if (NumInstr >= SkipThreshold)
+      if (!CostModel.isProfitable(MI))
         return true;
     }
   }
 
   return false;
 }
+} // namespace
 
 // Returns true if the skip branch instruction is removed.
 bool SIPreEmitPeephole::removeExeczBranch(MachineInstr &MI,
                                           MachineBasicBlock &SrcMBB) {
+
+  if (!TII->getSchedModel().hasInstrSchedModel())
+    return false;
+
   MachineBasicBlock *TrueMBB = nullptr;
   MachineBasicBlock *FalseMBB = nullptr;
   SmallVector<MachineOperand, 1> Cond;
@@ -351,8 +392,11 @@ bool SIPreEmitPeephole::removeExeczBranch(MachineInstr &MI,
     return false;
 
   // Consider only the forward branches.
-  if ((SrcMBB.getNumber() >= TrueMBB->getNumber()) ||
-      mustRetainExeczBranch(*FalseMBB, *TrueMBB))
+  if (SrcMBB.getNumber() >= TrueMBB->getNumber())
+    return false;
+
+  // Consider only when it is legal and profitable
+  if (mustRetainExeczBranch(MI, *FalseMBB, *TrueMBB))
     return false;
 
   LLVM_DEBUG(dbgs() << "Removing the execz branch: " << MI);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index de9cbe403ab6182430e76c1f32adbe22bf9a8bfc..20d48aa57adbdfa84b957d8b24e62804df61eba4 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3851,3 +3851,13 @@ SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
   }
   return 0;
 }
+
+SmallVector<StringLiteral>
+SIRegisterInfo::getVRegFlagsOfReg(Register Reg,
+                                  const MachineFunction &MF) const {
+  SmallVector<StringLiteral> RegFlags;
+  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+  if (FuncInfo->checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG))
+    RegFlags.push_back("WWM_REG");
+  return RegFlags;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 99fa632c0300be328f60c70ae4c2ce259d37aca2..fe0b66f75bbaa229b5a2e27eb5e846da65fa7566 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -457,6 +457,14 @@ public:
   // No check if the subreg is supported by the current RC is made.
   unsigned getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
                                      unsigned SubReg) const;
+
+  std::optional<uint8_t> getVRegFlagValue(StringRef Name) const override {
+    return Name == "WWM_REG" ? AMDGPU::VirtRegFlag::WWM_REG
+                             : std::optional<uint8_t>{};
+  }
+
+  SmallVector<StringLiteral>
+  getVRegFlagsOfReg(Register Reg, const MachineFunction &MF) const override;
 };
 
 namespace AMDGPU {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index ef9adde13348fe3c3e673d60877431f11f74ba78..3556f6a95b521e68c94ded69704472b9a551f891 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -88,15 +88,10 @@ class getSubRegs<int size> {
 
 // Generates list of sequential register tuple names.
 // E.g. RegSeq<3,2,2,"s">.ret -> [ "s[0:1]", "s[2:3]" ]
-class RegSeqNames<int last_reg, int stride, int size, string prefix,
-                  int start = 0> {
-  int next = !add(start, stride);
-  int end_reg = !add(start, size, -1);
-  list<string> ret =
-    !if(!le(end_reg, last_reg),
-        !listconcat([prefix # "[" # start # ":" # end_reg # "]"],
-                    RegSeqNames<last_reg, stride, size, prefix, next>.ret),
-                    []);
+class RegSeqNames<int last_reg, int stride, int size, string prefix> {
+  defvar numtuples = !div(!sub(!add(last_reg, stride, 1), size), stride);
+  defvar range = !range(0, !mul(numtuples, stride), stride);
+  list<string> ret = !foreach(n, range, prefix # "[" # n # ":" # !add(n, size, -1) # "]");
 }
 
 // Generates list of dags for register tuples.
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 0921e364498186c0f353c2100b1c95ca5795dd1a..51a5f895f341db30bb2154383f438f4ce24bf996 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -6868,15 +6868,13 @@ bool ARMPipelinerLoopInfo::tooMuchRegisterPressure(SwingSchedulerDAG &SSD,
       if (MI->isPHI() && S.getKind() == SDep::Anti) {
         Register Reg = S.getReg();
         if (Reg.isVirtual())
-          CrossIterationNeeds.insert(std::make_pair(Reg.id(), IterNeed()))
-              .first->second.set(0);
+          CrossIterationNeeds[Reg.id()].set(0);
       } else if (S.isAssignedRegDep()) {
         int OStg = SMS.stageScheduled(S.getSUnit());
         if (OStg >= 0 && OStg != Stg) {
           Register Reg = S.getReg();
           if (Reg.isVirtual())
-            CrossIterationNeeds.insert(std::make_pair(Reg.id(), IterNeed()))
-                .first->second |= ((1 << (OStg - Stg)) - 1);
+            CrossIterationNeeds[Reg.id()] |= ((1 << (OStg - Stg)) - 1);
         }
       }
   }
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index bf757edfa858902903af87e38637384d22dcdcc1..a35582bebb08a322bd33be75cbd00d5f3a805dde 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -21149,7 +21149,7 @@ Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
     // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
     // here.
     if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
-      Function *MCR = Intrinsic::getDeclaration(M, Intrinsic::arm_mcr);
+      Function *MCR = Intrinsic::getOrInsertDeclaration(M, Intrinsic::arm_mcr);
       Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
                         Builder.getInt32(0), Builder.getInt32(7),
                         Builder.getInt32(10), Builder.getInt32(5)};
@@ -21160,7 +21160,7 @@ Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
       llvm_unreachable("makeDMB on a target so old that it has no barriers");
     }
   } else {
-    Function *DMB = Intrinsic::getDeclaration(M, Intrinsic::arm_dmb);
+    Function *DMB = Intrinsic::getOrInsertDeclaration(M, Intrinsic::arm_dmb);
     // Only a full system barrier exists in the M-class architectures.
     Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
     Constant *CDomain = Builder.getInt32(Domain);
@@ -21417,7 +21417,7 @@ Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
   if (ValueTy->getPrimitiveSizeInBits() == 64) {
     Intrinsic::ID Int =
         IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
-    Function *Ldrex = Intrinsic::getDeclaration(M, Int);
+    Function *Ldrex = Intrinsic::getOrInsertDeclaration(M, Int);
 
     Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi");
 
@@ -21433,7 +21433,7 @@ Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
 
   Type *Tys[] = { Addr->getType() };
   Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
-  Function *Ldrex = Intrinsic::getDeclaration(M, Int, Tys);
+  Function *Ldrex = Intrinsic::getOrInsertDeclaration(M, Int, Tys);
   CallInst *CI = Builder.CreateCall(Ldrex, Addr);
 
   CI->addParamAttr(
@@ -21446,7 +21446,8 @@ void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
   if (!Subtarget->hasV7Ops())
     return;
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
-  Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::arm_clrex));
+  Builder.CreateCall(
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::arm_clrex));
 }
 
 Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
@@ -21461,7 +21462,7 @@ Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
   if (Val->getType()->getPrimitiveSizeInBits() == 64) {
     Intrinsic::ID Int =
         IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
-    Function *Strex = Intrinsic::getDeclaration(M, Int);
+    Function *Strex = Intrinsic::getOrInsertDeclaration(M, Int);
     Type *Int32Ty = Type::getInt32Ty(M->getContext());
 
     Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo");
@@ -21473,7 +21474,7 @@ Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
 
   Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
   Type *Tys[] = { Addr->getType() };
-  Function *Strex = Intrinsic::getDeclaration(M, Int, Tys);
+  Function *Strex = Intrinsic::getOrInsertDeclaration(M, Int, Tys);
 
   CallInst *CI = Builder.CreateCall(
       Strex, {Builder.CreateZExtOrBitCast(
@@ -21601,8 +21602,8 @@ bool ARMTargetLowering::lowerInterleavedLoad(
       static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
                                                 Intrinsic::arm_neon_vld3,
                                                 Intrinsic::arm_neon_vld4};
-      Function *VldnFunc =
-          Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
+      Function *VldnFunc = Intrinsic::getOrInsertDeclaration(
+          LI->getModule(), LoadInts[Factor - 2], Tys);
 
       SmallVector<Value *, 2> Ops;
       Ops.push_back(BaseAddr);
@@ -21617,7 +21618,7 @@ bool ARMTargetLowering::lowerInterleavedLoad(
       Type *PtrTy = Builder.getPtrTy(LI->getPointerAddressSpace());
       Type *Tys[] = {VecTy, PtrTy};
       Function *VldnFunc =
-          Intrinsic::getDeclaration(LI->getModule(), LoadInts, Tys);
+          Intrinsic::getOrInsertDeclaration(LI->getModule(), LoadInts, Tys);
 
       SmallVector<Value *, 2> Ops;
       Ops.push_back(BaseAddr);
@@ -21762,7 +21763,7 @@ bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
       Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace());
       Type *Tys[] = {PtrTy, SubVecTy};
 
-      Function *VstNFunc = Intrinsic::getDeclaration(
+      Function *VstNFunc = Intrinsic::getOrInsertDeclaration(
           SI->getModule(), StoreInts[Factor - 2], Tys);
 
       SmallVector<Value *, 6> Ops;
@@ -21778,7 +21779,7 @@ bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
       Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace());
       Type *Tys[] = {PtrTy, SubVecTy};
       Function *VstNFunc =
-          Intrinsic::getDeclaration(SI->getModule(), StoreInts, Tys);
+          Intrinsic::getOrInsertDeclaration(SI->getModule(), StoreInts, Tys);
 
       SmallVector<Value *, 6> Ops;
       Ops.push_back(BaseAddr);
diff --git a/llvm/lib/Target/ARM/ARMInstructionSelector.cpp b/llvm/lib/Target/ARM/ARMInstructionSelector.cpp
index 3b3c380e1e1b32a7770b8ff91e86ed51a9cfad24..ea18e6652c656f608fddd5556bd2b2c908fc14d6 100644
--- a/llvm/lib/Target/ARM/ARMInstructionSelector.cpp
+++ b/llvm/lib/Target/ARM/ARMInstructionSelector.cpp
@@ -14,6 +14,7 @@
 #include "ARMSubtarget.h"
 #include "ARMTargetMachine.h"
 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -1088,7 +1089,7 @@ bool ARMInstructionSelector::select(MachineInstr &I) {
     return selectGlobal(MIB, MRI);
   case G_STORE:
   case G_LOAD: {
-    const auto &MemOp = **I.memoperands_begin();
+    auto &MemOp = **I.memoperands_begin();
     if (MemOp.isAtomic()) {
       LLVM_DEBUG(dbgs() << "Atomic load/store not supported yet\n");
       return false;
@@ -1103,6 +1104,26 @@ bool ARMInstructionSelector::select(MachineInstr &I) {
     assert((ValSize != 64 || STI.hasVFP2Base()) &&
            "Don't know how to load/store 64-bit value without VFP");
 
+    if (auto *LoadMI = dyn_cast<GLoad>(&I)) {
+      Register PtrReg = LoadMI->getPointerReg();
+      MachineInstr *Ptr = MRI.getVRegDef(PtrReg);
+      if (Ptr->getOpcode() == TargetOpcode::G_CONSTANT_POOL) {
+        const MachineOperand &Index = Ptr->getOperand(1);
+        unsigned Opcode = Subtarget->isThumb() ? ARM::tLDRpci : ARM::LDRcp;
+
+        auto Instr = BuildMI(MBB, I, I.getDebugLoc(), TII.get(Opcode))
+                         .addDef(Reg)
+                         .add(Index)
+                         .addImm(0)
+                         .add(predOps(ARMCC::AL))
+                         .addMemOperand(&MemOp);
+        if (!constrainSelectedInstRegOperands(*Instr, TII, TRI, RBI))
+          return false;
+        I.eraseFromParent();
+        return true;
+      }
+    }
+
     const auto NewOpc = selectLoadStoreOpCode(I.getOpcode(), RegBank, ValSize);
     if (NewOpc == G_LOAD || NewOpc == G_STORE)
       return false;
diff --git a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
index 660f351bae64b71822b47ee55982b6ba3783ad84..452e908fdad98fd9374bb040e0eadab992d07b9a 100644
--- a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
@@ -29,7 +29,7 @@ static bool AEABI(const ARMSubtarget &ST) {
   return ST.isTargetAEABI() || ST.isTargetGNUAEABI() || ST.isTargetMuslAEABI();
 }
 
-ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
+ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) : ST(ST) {
   using namespace TargetOpcode;
 
   const LLT p0 = LLT::pointer(0, 32);
@@ -99,9 +99,11 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
       .minScalar(0, s32);
 
   getActionDefinitionsBuilder(G_CONSTANT)
-      .legalFor({s32, p0})
+      .customFor({s32, p0})
       .clampScalar(0, s32, s32);
 
+  getActionDefinitionsBuilder(G_CONSTANT_POOL).legalFor({p0});
+
   getActionDefinitionsBuilder(G_ICMP)
       .legalForCartesianProduct({s1}, {s32, p0})
       .minScalar(1, s32);
@@ -435,6 +437,13 @@ bool ARMLegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
     }
     break;
   }
+  case G_CONSTANT: {
+    const ConstantInt *ConstVal = MI.getOperand(1).getCImm();
+    uint64_t ImmVal = ConstVal->getZExtValue();
+    if (ConstantMaterializationCost(ImmVal, &ST) > 2 && !ST.genExecuteOnly())
+      return Helper.lowerConstant(MI) == LegalizerHelper::Legalized;
+    return true;
+  }
   case G_FCONSTANT: {
     // Convert to integer constants, while preserving the binary representation.
     auto AsInteger =
diff --git a/llvm/lib/Target/ARM/ARMLegalizerInfo.h b/llvm/lib/Target/ARM/ARMLegalizerInfo.h
index 9e10638233e6d1b69eb313432531185641609a21..f7edf0b1612c53bdca5947d8124023faf0426abe 100644
--- a/llvm/lib/Target/ARM/ARMLegalizerInfo.h
+++ b/llvm/lib/Target/ARM/ARMLegalizerInfo.h
@@ -59,6 +59,8 @@ private:
   // Get the libcall(s) corresponding to \p Predicate for operands of \p Size
   // bits.
   FCmpLibcallsList getFCmpLibcalls(CmpInst::Predicate, unsigned Size) const;
+
+  const ARMSubtarget &ST;
 };
 } // End llvm namespace.
 #endif
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index b151a0116a9c4110c1dba154ef0faaecf8bb275c..c7b6b3579120dc97ff6584eb0ba86929190f3a6c 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -2532,7 +2532,6 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
   bool RetVal = false;
 
   DenseMap<MachineInstr *, unsigned> MI2LocMap;
-  using MapIt = DenseMap<unsigned, SmallVector<MachineInstr *, 4>>::iterator;
   using Base2InstMap = DenseMap<unsigned, SmallVector<MachineInstr *, 4>>;
   using BaseVec = SmallVector<unsigned, 4>;
   Base2InstMap Base2LdsMap;
@@ -2571,9 +2570,9 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
       int Offset = getMemoryOpOffset(MI);
       bool StopHere = false;
       auto FindBases = [&](Base2InstMap &Base2Ops, BaseVec &Bases) {
-        MapIt BI = Base2Ops.find(Base);
-        if (BI == Base2Ops.end()) {
-          Base2Ops[Base].push_back(&MI);
+        auto [BI, Inserted] = Base2Ops.try_emplace(Base);
+        if (Inserted) {
+          BI->second.push_back(&MI);
           Bases.push_back(Base);
           return;
         }
diff --git a/llvm/lib/Target/ARM/ARMParallelDSP.cpp b/llvm/lib/Target/ARM/ARMParallelDSP.cpp
index 861d60d3bcce95190ac47b2566f8f163f20dc8e5..7804725ce77319269f998a0848bdd1ee6e27b904 100644
--- a/llvm/lib/Target/ARM/ARMParallelDSP.cpp
+++ b/llvm/lib/Target/ARM/ARMParallelDSP.cpp
@@ -630,13 +630,14 @@ void ARMParallelDSP::InsertParallelMACs(Reduction &R) {
     Value* Args[] = { WideLd0, WideLd1, Acc };
     Function *SMLAD = nullptr;
     if (Exchange)
-      SMLAD = Acc->getType()->isIntegerTy(32) ?
-        Intrinsic::getDeclaration(M, Intrinsic::arm_smladx) :
-        Intrinsic::getDeclaration(M, Intrinsic::arm_smlaldx);
+      SMLAD =
+          Acc->getType()->isIntegerTy(32)
+              ? Intrinsic::getOrInsertDeclaration(M, Intrinsic::arm_smladx)
+              : Intrinsic::getOrInsertDeclaration(M, Intrinsic::arm_smlaldx);
     else
-      SMLAD = Acc->getType()->isIntegerTy(32) ?
-        Intrinsic::getDeclaration(M, Intrinsic::arm_smlad) :
-        Intrinsic::getDeclaration(M, Intrinsic::arm_smlald);
+      SMLAD = Acc->getType()->isIntegerTy(32)
+                  ? Intrinsic::getOrInsertDeclaration(M, Intrinsic::arm_smlad)
+                  : Intrinsic::getOrInsertDeclaration(M, Intrinsic::arm_smlald);
 
     IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
                                 BasicBlock::iterator(InsertAfter));
diff --git a/llvm/lib/Target/ARM/ARMProcessors.td b/llvm/lib/Target/ARM/ARMProcessors.td
index ce767b2b968e1715beb4cdcda8fc3cffb529ce1a..08f62d12f4a9f1b6256c82b37dbdb0387a39cbd6 100644
--- a/llvm/lib/Target/ARM/ARMProcessors.td
+++ b/llvm/lib/Target/ARM/ARMProcessors.td
@@ -362,6 +362,16 @@ def : ProcessorModel<"cortex-m33", CortexM4Model,       [ARMv8mMainline,
                                                          FeatureHasNoBranchPredictor,
                                                          FeatureFixCMSE_CVE_2021_35465]>;
 
+def : ProcessorModel<"star-mc1", CortexM4Model,         [ARMv8mMainline,
+                                                         FeatureDSP,
+                                                         FeatureFPARMv8_D16_SP,
+                                                         FeaturePreferBranchAlign32,
+                                                         FeatureHasSlowFPVMLx,
+                                                         FeatureHasSlowFPVFMx,
+                                                         FeatureUseMISched,
+                                                         FeatureHasNoBranchPredictor,
+                                                         FeatureFixCMSE_CVE_2021_35465]>;
+
 def : ProcessorModel<"cortex-m35p", CortexM4Model,      [ARMv8mMainline,
                                                          FeatureDSP,
                                                          FeatureFPARMv8_D16_SP,
diff --git a/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp b/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp
index 447db18b8defa1ca3508088883f10f4d17765df3..0493fd9e317d5c3e512f5a3d135b2e37ed63dcfb 100644
--- a/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMRegisterBankInfo.cpp
@@ -329,6 +329,7 @@ ARMRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case G_CONSTANT:
   case G_FRAME_INDEX:
   case G_GLOBAL_VALUE:
+  case G_CONSTANT_POOL:
     OperandsMapping =
         getOperandsMapping({&ARM::ValueMappings[ARM::GPR3OpsIdx], nullptr});
     break;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index e68e7833327116ad86b3c252c1c4beeb44b80037..7917ddc17bdb812d97606bcb092c013a8046735c 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -344,7 +344,9 @@ public:
   }
   bool isTargetGNUAEABI() const {
     return (TargetTriple.getEnvironment() == Triple::GNUEABI ||
-            TargetTriple.getEnvironment() == Triple::GNUEABIHF) &&
+            TargetTriple.getEnvironment() == Triple::GNUEABIT64 ||
+            TargetTriple.getEnvironment() == Triple::GNUEABIHF ||
+            TargetTriple.getEnvironment() == Triple::GNUEABIHFT64) &&
            !isTargetDarwin() && !isTargetWindows();
   }
   bool isTargetMuslAEABI() const {
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 7553778c574033f8d6e79813b83ea5de62d5d36a..a58c63dcf762d1eb8f25cab7654306fd3fb5a7c0 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -241,7 +241,9 @@ ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT,
       Options.EABIVersion == EABI::Unknown) {
     // musl is compatible with glibc with regard to EABI version
     if ((TargetTriple.getEnvironment() == Triple::GNUEABI ||
+         TargetTriple.getEnvironment() == Triple::GNUEABIT64 ||
          TargetTriple.getEnvironment() == Triple::GNUEABIHF ||
+         TargetTriple.getEnvironment() == Triple::GNUEABIHFT64 ||
          TargetTriple.getEnvironment() == Triple::MuslEABI ||
          TargetTriple.getEnvironment() == Triple::MuslEABIHF ||
          TargetTriple.getEnvironment() == Triple::OpenHOS) &&
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h
index 69d8fa8ada649835ab786b7e18c70752d0084216..75ee50f0e93c885102f708949d6606720e76d7ee 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.h
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.h
@@ -64,6 +64,7 @@ public:
 
   bool isTargetHardFloat() const {
     return TargetTriple.getEnvironment() == Triple::GNUEABIHF ||
+           TargetTriple.getEnvironment() == Triple::GNUEABIHFT64 ||
            TargetTriple.getEnvironment() == Triple::MuslEABIHF ||
            TargetTriple.getEnvironment() == Triple::EABIHF ||
            (TargetTriple.isOSBinFormatMachO() &&
diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp
index e554e4d428d46f52678da9c8836b17e3786bb57f..60211db8a61ae383e24af9094b3cd134c468b962 100644
--- a/llvm/lib/Target/ARM/MVETailPredication.cpp
+++ b/llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -401,7 +401,7 @@ void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask,
   case 8:  VCTPID = Intrinsic::arm_mve_vctp16; break;
   case 16: VCTPID = Intrinsic::arm_mve_vctp8; break;
   }
-  Function *VCTP = Intrinsic::getDeclaration(M, VCTPID);
+  Function *VCTP = Intrinsic::getOrInsertDeclaration(M, VCTPID);
   Value *VCTPCall = Builder.CreateCall(VCTP, Processed);
   ActiveLaneMask->replaceAllUsesWith(VCTPCall);
 
diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
index 4be6220b358ba3b879f0ce2136aa593762500c29..39f38259a193b4c240db364cbe9f78f073527a79 100644
--- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
+++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
@@ -103,7 +103,7 @@ uint32_t BPFCoreSharedInfo::SeqNum;
 Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB,
                                                   Instruction *Input,
                                                   Instruction *Before) {
-  Function *Fn = Intrinsic::getDeclaration(
+  Function *Fn = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::bpf_passthrough, {Input->getType(), Input->getType()});
   Constant *SeqNumVal = ConstantInt::get(Type::getInt32Ty(BB->getContext()),
                                          BPFCoreSharedInfo::SeqNum++);
@@ -221,10 +221,9 @@ bool BPFAbstractMemberAccess::run(Function &F) {
 
 void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) {
   if (auto Ty = dyn_cast<DICompositeType>(CInfo.Metadata)) {
-    if (AnonRecords.find(Ty) != AnonRecords.end()) {
-      if (AnonRecords[Ty] != nullptr)
-        CInfo.Metadata = AnonRecords[Ty];
-    }
+    auto It = AnonRecords.find(Ty);
+    if (It != AnonRecords.end() && It->second != nullptr)
+      CInfo.Metadata = It->second;
   }
 }
 
@@ -234,18 +233,12 @@ void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy,
       ParentTy->getTag() != dwarf::DW_TAG_typedef)
     return;
 
-  if (AnonRecords.find(CTy) == AnonRecords.end()) {
-    AnonRecords[CTy] = ParentTy;
-    return;
-  }
-
+  auto [It, Inserted] = AnonRecords.try_emplace(CTy, ParentTy);
   // Two or more typedef's may point to the same anon record.
   // If this is the case, set the typedef DIType to be nullptr
   // to indicate the duplication case.
-  DIDerivedType *CurrTy = AnonRecords[CTy];
-  if (CurrTy == ParentTy)
-    return;
-  AnonRecords[CTy] = nullptr;
+  if (!Inserted && It->second != ParentTy)
+    It->second = nullptr;
 }
 
 void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy,
diff --git a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp
index 4ab0cbcc924779620025fc43dbc420d5af946925..4ca7bbe9c2a8c460822f337c85d8b785e0176e24 100644
--- a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp
+++ b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp
@@ -126,7 +126,7 @@ bool BPFAdjustOptImpl::adjustICmpToBuiltin() {
 
         Constant *Opcode =
             ConstantInt::get(Type::getInt32Ty(BB.getContext()), Op);
-        Function *Fn = Intrinsic::getDeclaration(
+        Function *Fn = Intrinsic::getOrInsertDeclaration(
             M, Intrinsic::bpf_compare, {Op0->getType(), ConstOp1->getType()});
         auto *NewInst = CallInst::Create(Fn, {Opcode, Op0, ConstOp1});
         NewInst->insertBefore(&I);
diff --git a/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
index 5d8339b4a44ceceabb64fb5b2f27cfacfc7d3e1f..9f7e3414beb8e3b81fe53eddd76444257185d8df 100644
--- a/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
+++ b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
@@ -163,7 +163,7 @@ static CallInst *makeIntrinsicCall(Module *M,
                                    ArrayRef<Type *> Types,
                                    ArrayRef<Value *> Args) {
 
-  Function *Fn = Intrinsic::getDeclaration(M, Intrinsic, Types);
+  Function *Fn = Intrinsic::getOrInsertDeclaration(M, Intrinsic, Types);
   return CallInst::Create(Fn, Args);
 }
 
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index c0f8d433833ee733f9357c2e0343e5ba4d919acc..99df485087207837a2a2961951ee42b07cf0d797 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -134,8 +134,8 @@ public:
   /// piecemeal way - we can add the casts in to avoid updating all of the uses
   /// or defs, and by the end all of the casts will be redundant.
   Value *createTmpHandleCast(Value *V, Type *Ty) {
-    Function *CastFn = Intrinsic::getDeclaration(&M, Intrinsic::dx_cast_handle,
-                                                 {Ty, V->getType()});
+    Function *CastFn = Intrinsic::getOrInsertDeclaration(
+        &M, Intrinsic::dx_cast_handle, {Ty, V->getType()});
     CallInst *Cast = OpBuilder.getIRB().CreateCall(CastFn, {V});
     CleanupCasts.push_back(Cast);
     return Cast;
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
index 05357de40e3a93e25c154f8ccd83ea064cd6789a..7c82f5e9f9a60469f5e08eab13ee36f6c09a6357 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -906,22 +906,25 @@ void HexagonFrameLowering::insertAllocframe(MachineBasicBlock &MBB,
   if (NumBytes >= ALLOCFRAME_MAX) {
     // Emit allocframe(#0).
     BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::S2_allocframe))
-      .addDef(SP)
-      .addReg(SP)
-      .addImm(0)
-      .addMemOperand(MMO);
+        .addDef(SP)
+        .addReg(SP)
+        .addImm(0)
+        .addMemOperand(MMO)
+        .setMIFlag(MachineInstr::FrameSetup);
 
     // Subtract the size from the stack pointer.
     Register SP = HRI.getStackRegister();
     BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::A2_addi), SP)
-      .addReg(SP)
-      .addImm(-int(NumBytes));
+        .addReg(SP)
+        .addImm(-int(NumBytes))
+        .setMIFlag(MachineInstr::FrameSetup);
   } else {
     BuildMI(MBB, InsertPt, dl, HII.get(Hexagon::S2_allocframe))
-      .addDef(SP)
-      .addReg(SP)
-      .addImm(NumBytes)
-      .addMemOperand(MMO);
+        .addDef(SP)
+        .addReg(SP)
+        .addImm(NumBytes)
+        .addMemOperand(MMO)
+        .setMIFlag(MachineInstr::FrameSetup);
   }
 }
 
diff --git a/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp b/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
index 3274f9162b543ac99aee039ed08eaea66dc39e72..65bbb1364488f757dfbd980ccf0b14bcbe906fa7 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
@@ -212,7 +212,7 @@ bool HexagonGenExtract::convert(Instruction *In) {
   Intrinsic::ID IntId = (BW == 32) ? Intrinsic::hexagon_S2_extractu
                                    : Intrinsic::hexagon_S2_extractup;
   Module *Mod = BB->getParent()->getParent();
-  Function *ExtF = Intrinsic::getDeclaration(Mod, IntId);
+  Function *ExtF = Intrinsic::getOrInsertDeclaration(Mod, IntId);
   Value *NewIn = IRB.CreateCall(ExtF, {BF, IRB.getInt32(W), IRB.getInt32(SR)});
   if (SL != 0)
     NewIn = IRB.CreateShl(NewIn, SL, CSL->getName());
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 856c952e785dac4226a7ad17e5ff7e29c7fa3f03..03c12f5ce44707d341fcbf42f35f85459004ce0b 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3865,7 +3865,7 @@ Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
   assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
   Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
                                    : Intrinsic::hexagon_L4_loadd_locked;
-  Function *Fn = Intrinsic::getDeclaration(M, IntID);
+  Function *Fn = Intrinsic::getOrInsertDeclaration(M, IntID);
 
   Value *Call = Builder.CreateCall(Fn, Addr, "larx");
 
@@ -3886,7 +3886,7 @@ Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
   assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
   Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
                                    : Intrinsic::hexagon_S4_stored_locked;
-  Function *Fn = Intrinsic::getDeclaration(M, IntID);
+  Function *Fn = Intrinsic::getOrInsertDeclaration(M, IntID);
 
   Val = Builder.CreateBitCast(Val, CastTy);
 
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index 4ef009c87a1e63d53094e3ed69782d7c1ad5f5ce..705e1f43851f7adf357b164ca0df401148b9fde4 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -1532,7 +1532,8 @@ Value *PolynomialMultiplyRecognize::generate(BasicBlock::iterator At,
       ParsedValues &PV) {
   IRBuilder<> B(&*At);
   Module *M = At->getParent()->getParent()->getParent();
-  Function *PMF = Intrinsic::getDeclaration(M, Intrinsic::hexagon_M4_pmpyw);
+  Function *PMF =
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::hexagon_M4_pmpyw);
 
   Value *P = PV.P, *Q = PV.Q, *P0 = P;
   unsigned IC = PV.IterCount;
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index f4e495266eae3fcd38cebc0f63159d4407bd3ca4..d2cfd3851e711debb2f321bbd0689d83142b8d9c 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -2390,8 +2390,8 @@ auto HexagonVectorCombine::vralignb(IRBuilderBase &Builder, Value *Lo,
     Type *Int64Ty = Type::getInt64Ty(F.getContext());
     Value *Lo64 = Builder.CreateBitCast(Lo, Int64Ty, "cst");
     Value *Hi64 = Builder.CreateBitCast(Hi, Int64Ty, "cst");
-    Function *FI = Intrinsic::getDeclaration(F.getParent(),
-                                             Intrinsic::hexagon_S2_valignrb);
+    Function *FI = Intrinsic::getOrInsertDeclaration(
+        F.getParent(), Intrinsic::hexagon_S2_valignrb);
     Value *Call = Builder.CreateCall(FI, {Hi64, Lo64, Amt}, "cup");
     return Builder.CreateBitCast(Call, Lo->getType(), "cst");
   }
@@ -2587,12 +2587,13 @@ auto HexagonVectorCombine::createHvxIntrinsic(IRBuilderBase &Builder,
     unsigned HwLen = HST.getVectorLength();
     Intrinsic::ID TC = HwLen == 64 ? Intrinsic::hexagon_V6_pred_typecast
                                    : Intrinsic::hexagon_V6_pred_typecast_128B;
-    Function *FI =
-        Intrinsic::getDeclaration(F.getParent(), TC, {DestTy, Val->getType()});
+    Function *FI = Intrinsic::getOrInsertDeclaration(F.getParent(), TC,
+                                                     {DestTy, Val->getType()});
     return Builder.CreateCall(FI, {Val}, "cup");
   };
 
-  Function *IntrFn = Intrinsic::getDeclaration(F.getParent(), IntID, ArgTys);
+  Function *IntrFn =
+      Intrinsic::getOrInsertDeclaration(F.getParent(), IntID, ArgTys);
   FunctionType *IntrTy = IntrFn->getFunctionType();
 
   SmallVector<Value *, 4> IntrArgs;
diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
index e3abb7eecc32b10975d3f798b8fa2c2c1e85c42f..ffc1a27aa6e18ff6f76e25e50f2cf721f52b32e7 100644
--- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
+++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
@@ -10,6 +10,7 @@
 #include "MCTargetDesc/LoongArchMCExpr.h"
 #include "MCTargetDesc/LoongArchMCTargetDesc.h"
 #include "MCTargetDesc/LoongArchMatInt.h"
+#include "MCTargetDesc/LoongArchTargetStreamer.h"
 #include "TargetInfo/LoongArchTargetInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstBuilder.h"
@@ -30,8 +31,16 @@ using namespace llvm;
 
 namespace {
 class LoongArchAsmParser : public MCTargetAsmParser {
+  SmallVector<FeatureBitset, 4> FeatureBitStack;
+
   SMLoc getLoc() const { return getParser().getTok().getLoc(); }
   bool is64Bit() const { return getSTI().hasFeature(LoongArch::Feature64Bit); }
+  LoongArchTargetStreamer &getTargetStreamer() {
+    assert(getParser().getStreamer().getTargetStreamer() &&
+           "do not have a target streamer");
+    MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
+    return static_cast<LoongArchTargetStreamer &>(TS);
+  }
 
   struct Inst {
     unsigned Opc;
@@ -60,6 +69,8 @@ class LoongArchAsmParser : public MCTargetAsmParser {
   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                       unsigned Kind) override;
 
+  ParseStatus parseDirective(AsmToken DirectiveID) override;
+
   bool generateImmOutOfRangeError(OperandVector &Operands, uint64_t ErrorInfo,
                                   int64_t Lower, int64_t Upper,
                                   const Twine &Msg);
@@ -81,6 +92,39 @@ class LoongArchAsmParser : public MCTargetAsmParser {
 
   bool parseOperand(OperandVector &Operands, StringRef Mnemonic);
 
+  bool parseDirectiveOption();
+
+  void setFeatureBits(uint64_t Feature, StringRef FeatureString) {
+    if (!(getSTI().hasFeature(Feature))) {
+      MCSubtargetInfo &STI = copySTI();
+      setAvailableFeatures(
+          ComputeAvailableFeatures(STI.ToggleFeature(FeatureString)));
+    }
+  }
+
+  void clearFeatureBits(uint64_t Feature, StringRef FeatureString) {
+    if (getSTI().hasFeature(Feature)) {
+      MCSubtargetInfo &STI = copySTI();
+      setAvailableFeatures(
+          ComputeAvailableFeatures(STI.ToggleFeature(FeatureString)));
+    }
+  }
+
+  void pushFeatureBits() {
+    FeatureBitStack.push_back(getSTI().getFeatureBits());
+  }
+
+  bool popFeatureBits() {
+    if (FeatureBitStack.empty())
+      return true;
+
+    FeatureBitset FeatureBits = FeatureBitStack.pop_back_val();
+    copySTI().setFeatureBits(FeatureBits);
+    setAvailableFeatures(ComputeAvailableFeatures(FeatureBits));
+
+    return false;
+  }
+
   // Helper to emit the sequence of instructions generated by the
   // "emitLoadAddress*" functions.
   void emitLAInstSeq(MCRegister DestReg, MCRegister TmpReg,
@@ -1748,6 +1792,70 @@ bool LoongArchAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   llvm_unreachable("Unknown match type detected!");
 }
 
+bool LoongArchAsmParser::parseDirectiveOption() {
+  MCAsmParser &Parser = getParser();
+  // Get the option token.
+  AsmToken Tok = Parser.getTok();
+
+  // At the moment only identifiers are supported.
+  if (parseToken(AsmToken::Identifier, "expected identifier"))
+    return true;
+
+  StringRef Option = Tok.getIdentifier();
+
+  if (Option == "push") {
+    if (Parser.parseEOL())
+      return true;
+
+    getTargetStreamer().emitDirectiveOptionPush();
+    pushFeatureBits();
+    return false;
+  }
+
+  if (Option == "pop") {
+    SMLoc StartLoc = Parser.getTok().getLoc();
+    if (Parser.parseEOL())
+      return true;
+
+    getTargetStreamer().emitDirectiveOptionPop();
+    if (popFeatureBits())
+      return Error(StartLoc, ".option pop with no .option push");
+
+    return false;
+  }
+
+  if (Option == "relax") {
+    if (Parser.parseEOL())
+      return true;
+
+    getTargetStreamer().emitDirectiveOptionRelax();
+    setFeatureBits(LoongArch::FeatureRelax, "relax");
+    return false;
+  }
+
+  if (Option == "norelax") {
+    if (Parser.parseEOL())
+      return true;
+
+    getTargetStreamer().emitDirectiveOptionNoRelax();
+    clearFeatureBits(LoongArch::FeatureRelax, "relax");
+    return false;
+  }
+
+  // Unknown option.
+  Warning(Parser.getTok().getLoc(),
+          "unknown option, expected 'push', 'pop', 'relax' or 'norelax'");
+  Parser.eatToEndOfStatement();
+  return false;
+}
+
+ParseStatus LoongArchAsmParser::parseDirective(AsmToken DirectiveID) {
+  if (DirectiveID.getString() == ".option")
+    return parseDirectiveOption();
+
+  return ParseStatus::NoMatch;
+}
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeLoongArchAsmParser() {
   RegisterMCAsmParser<LoongArchAsmParser> X(getTheLoongArch32Target());
   RegisterMCAsmParser<LoongArchAsmParser> Y(getTheLoongArch64Target());
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index bfafb33175210892cc6387cb3d4cceb13b45ea4d..8edca34624e9b2dac4b560a2cc028493e5192bc0 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -5808,7 +5808,7 @@ Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
   Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
   Type *Tys[] = {AlignedAddr->getType()};
   Function *MaskedCmpXchg =
-      Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
+      Intrinsic::getOrInsertDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
   Value *Result = Builder.CreateCall(
       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
   Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
@@ -5838,7 +5838,7 @@ Value *LoongArchTargetLowering::emitMaskedAtomicRMWIntrinsic(
   Value *Ordering =
       Builder.getIntN(GRLen, static_cast<uint64_t>(AI->getOrdering()));
   Type *Tys[] = {AlignedAddr->getType()};
-  Function *LlwOpScwLoop = Intrinsic::getDeclaration(
+  Function *LlwOpScwLoop = Intrinsic::getOrInsertDeclaration(
       AI->getModule(),
       getIntrinsicForMaskedAtomicRMWBinOp(GRLen, AI->getOperation()), Tys);
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
index eee85822398b14838d06b2955125a85aedf4f692..f2900797c6a2be4d9612d702d31d3e61187847c8 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
@@ -35,6 +35,11 @@ MCELFStreamer &LoongArchTargetELFStreamer::getStreamer() {
   return static_cast<MCELFStreamer &>(Streamer);
 }
 
+void LoongArchTargetELFStreamer::emitDirectiveOptionPush() {}
+void LoongArchTargetELFStreamer::emitDirectiveOptionPop() {}
+void LoongArchTargetELFStreamer::emitDirectiveOptionRelax() {}
+void LoongArchTargetELFStreamer::emitDirectiveOptionNoRelax() {}
+
 void LoongArchTargetELFStreamer::finish() {
   LoongArchTargetStreamer::finish();
   ELFObjectWriter &W = getStreamer().getWriter();
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h
index e220729d8923e243e483fce118c5ec604fd79b58..894ea9630714874ce537566a2c7e48d90fea9ccc 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h
@@ -19,6 +19,11 @@ public:
   MCELFStreamer &getStreamer();
   LoongArchTargetELFStreamer(MCStreamer &S, const MCSubtargetInfo &STI);
 
+  void emitDirectiveOptionPush() override;
+  void emitDirectiveOptionPop() override;
+  void emitDirectiveOptionRelax() override;
+  void emitDirectiveOptionNoRelax() override;
+
   void finish() override;
 };
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.cpp
index 595ce9fc815bf08b846feaa290fe8b43cd2b2c5f..62e900dc65babbad9606889114b9dfa2611f689c 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.cpp
@@ -87,6 +87,12 @@ createLoongArchObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) {
              : nullptr;
 }
 
+static MCTargetStreamer *
+createLoongArchAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS,
+                                 MCInstPrinter *InstPrint) {
+  return new LoongArchTargetAsmStreamer(S, OS);
+}
+
 namespace {
 
 class LoongArchMCInstrAnalysis : public MCInstrAnalysis {
@@ -212,5 +218,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeLoongArchTargetMC() {
     TargetRegistry::RegisterELFStreamer(*T, createLoongArchELFStreamer);
     TargetRegistry::RegisterObjectTargetStreamer(
         *T, createLoongArchObjectTargetStreamer);
+    TargetRegistry::RegisterAsmTargetStreamer(*T,
+                                              createLoongArchAsmTargetStreamer);
   }
 }
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.cpp
index 571178442055190b2e6c88ceaa2fdce5bc09af8a..8f9f0cb593b93b3ab39260f992eb8806d50c1e73 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.cpp
@@ -22,3 +22,29 @@ void LoongArchTargetStreamer::setTargetABI(LoongArchABI::ABI ABI) {
          "Improperly initialized target ABI");
   TargetABI = ABI;
 }
+
+void LoongArchTargetStreamer::emitDirectiveOptionPush() {}
+void LoongArchTargetStreamer::emitDirectiveOptionPop() {}
+void LoongArchTargetStreamer::emitDirectiveOptionRelax() {}
+void LoongArchTargetStreamer::emitDirectiveOptionNoRelax() {}
+
+// This part is for ascii assembly output.
+LoongArchTargetAsmStreamer::LoongArchTargetAsmStreamer(
+    MCStreamer &S, formatted_raw_ostream &OS)
+    : LoongArchTargetStreamer(S), OS(OS) {}
+
+void LoongArchTargetAsmStreamer::emitDirectiveOptionPush() {
+  OS << "\t.option\tpush\n";
+}
+
+void LoongArchTargetAsmStreamer::emitDirectiveOptionPop() {
+  OS << "\t.option\tpop\n";
+}
+
+void LoongArchTargetAsmStreamer::emitDirectiveOptionRelax() {
+  OS << "\t.option\trelax\n";
+}
+
+void LoongArchTargetAsmStreamer::emitDirectiveOptionNoRelax() {
+  OS << "\t.option\tnorelax\n";
+}
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.h
index d4b1b2a3e358dbd692abbb6fdf4a4dbae9b4c65f..96c324a3b6dc3c3fc5ddf848fef36cd68f0fc15b 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchTargetStreamer.h
@@ -12,6 +12,7 @@
 #include "LoongArch.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/FormattedStream.h"
 
 namespace llvm {
 class LoongArchTargetStreamer : public MCTargetStreamer {
@@ -21,6 +22,24 @@ public:
   LoongArchTargetStreamer(MCStreamer &S);
   void setTargetABI(LoongArchABI::ABI ABI);
   LoongArchABI::ABI getTargetABI() const { return TargetABI; }
+
+  virtual void emitDirectiveOptionPush();
+  virtual void emitDirectiveOptionPop();
+  virtual void emitDirectiveOptionRelax();
+  virtual void emitDirectiveOptionNoRelax();
+};
+
+// This part is for ascii assembly output.
+class LoongArchTargetAsmStreamer : public LoongArchTargetStreamer {
+  formatted_raw_ostream &OS;
+
+public:
+  LoongArchTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS);
+
+  void emitDirectiveOptionPush() override;
+  void emitDirectiveOptionPop() override;
+  void emitDirectiveOptionRelax() override;
+  void emitDirectiveOptionNoRelax() override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/NVPTX/NVPTX.td b/llvm/lib/Target/NVPTX/NVPTX.td
index bb4549a5e60782c7574572e4dfdbb7d68f36930a..9af8715ef52ae734b7c297e7555240857c1d0a62 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.td
+++ b/llvm/lib/Target/NVPTX/NVPTX.td
@@ -35,14 +35,14 @@ class FeaturePTX<int version>:
                     "Use PTX version " # version>;
 
 foreach sm = [20, 21, 30, 32, 35, 37, 50, 52, 53,
-              60, 61, 62, 70, 72, 75, 80, 86, 87, 89, 90] in
+              60, 61, 62, 70, 72, 75, 80, 86, 87, 89, 90, 100] in
   def SM#sm: FeatureSM<""#sm, !mul(sm, 10)>;
 
 def SM90a: FeatureSM<"90a", 901>;
 
 foreach version = [32, 40, 41, 42, 43, 50, 60, 61, 62, 63, 64, 65,
                    70, 71, 72, 73, 74, 75, 76, 77, 78,
-                   80, 81, 82, 83, 84, 85] in
+                   80, 81, 82, 83, 84, 85, 86] in
   def PTX#version: FeaturePTX<version>;
 
 //===----------------------------------------------------------------------===//
@@ -73,6 +73,7 @@ def : Proc<"sm_87", [SM87, PTX74]>;
 def : Proc<"sm_89", [SM89, PTX78]>;
 def : Proc<"sm_90", [SM90, PTX78]>;
 def : Proc<"sm_90a", [SM90a, PTX80]>;
+def : Proc<"sm_100", [SM100, PTX86]>;
 
 def NVPTXInstrInfo : InstrInfo {
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index d95f8f214be5575a60336e246e65fbcb824de6ab..57bc5fe0ac361c26d2824e2aa6177ae0e3247288 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -2332,23 +2332,20 @@ SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // Lower non-const v4i8 vector as byte-wise constructed i32, which allows us
     // to optimize calculation of constant parts.
     if (VT == MVT::v4i8) {
-      SDValue PRMT__10 = DAG.getNode(
-          NVPTXISD::PRMT, DL, MVT::v4i8,
-          {DAG.getAnyExtOrTrunc(Op->getOperand(0), DL, MVT::i32),
-           DAG.getAnyExtOrTrunc(Op->getOperand(1), DL, MVT::i32),
-           DAG.getConstant(0x3340, DL, MVT::i32),
-           DAG.getConstant(NVPTX::PTXPrmtMode::NONE, DL, MVT::i32)});
-      SDValue PRMT32__ = DAG.getNode(
-          NVPTXISD::PRMT, DL, MVT::v4i8,
-          {DAG.getAnyExtOrTrunc(Op->getOperand(2), DL, MVT::i32),
-           DAG.getAnyExtOrTrunc(Op->getOperand(3), DL, MVT::i32),
-           DAG.getConstant(0x4033, DL, MVT::i32),
-           DAG.getConstant(NVPTX::PTXPrmtMode::NONE, DL, MVT::i32)});
-      SDValue PRMT3210 = DAG.getNode(
-          NVPTXISD::PRMT, DL, MVT::v4i8,
-          {PRMT__10, PRMT32__, DAG.getConstant(0x5410, DL, MVT::i32),
-           DAG.getConstant(NVPTX::PTXPrmtMode::NONE, DL, MVT::i32)});
-      return DAG.getNode(ISD::BITCAST, DL, VT, PRMT3210);
+      SDValue C8 = DAG.getConstant(8, DL, MVT::i32);
+      SDValue E01 = DAG.getNode(
+          NVPTXISD::BFI, DL, MVT::i32,
+          DAG.getAnyExtOrTrunc(Op->getOperand(1), DL, MVT::i32),
+          DAG.getAnyExtOrTrunc(Op->getOperand(0), DL, MVT::i32), C8, C8);
+      SDValue E012 =
+          DAG.getNode(NVPTXISD::BFI, DL, MVT::i32,
+                      DAG.getAnyExtOrTrunc(Op->getOperand(2), DL, MVT::i32),
+                      E01, DAG.getConstant(16, DL, MVT::i32), C8);
+      SDValue E0123 =
+          DAG.getNode(NVPTXISD::BFI, DL, MVT::i32,
+                      DAG.getAnyExtOrTrunc(Op->getOperand(3), DL, MVT::i32),
+                      E012, DAG.getConstant(24, DL, MVT::i32), C8);
+      return DAG.getNode(ISD::BITCAST, DL, VT, E0123);
     }
     return Op;
   }
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 082546c4dd72f88690db60839c89f7ebba843858..4a184037add4cec9a751b3ea340227b9a7b20e35 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -282,7 +282,7 @@ static void convertToParamAS(Use *OldUse, Value *Param, bool HasCvtaParam,
           [](Value *Addr, Instruction *OriginalUser) -> Value * {
         PointerType *ReturnTy =
             PointerType::get(OriginalUser->getContext(), ADDRESS_SPACE_GENERIC);
-        Function *CvtToGen = Intrinsic::getDeclaration(
+        Function *CvtToGen = Intrinsic::getOrInsertDeclaration(
             OriginalUser->getModule(), Intrinsic::nvvm_ptr_param_to_gen,
             {ReturnTy, PointerType::get(OriginalUser->getContext(),
                                         ADDRESS_SPACE_PARAM)});
@@ -626,10 +626,10 @@ void NVPTXLowerArgs::handleByValParam(const NVPTXTargetMachine &TM,
     // Be sure to propagate alignment to this load; LLVM doesn't know that NVPTX
     // addrspacecast preserves alignment.  Since params are constant, this load
     // is definitely not volatile.
-    LoadInst *LI =
-        new LoadInst(StructType, ArgInParam, Arg->getName(),
-                     /*isVolatile=*/false, AllocA->getAlign(), FirstInst);
-    new StoreInst(LI, AllocA, FirstInst);
+    const auto ArgSize = *AllocA->getAllocationSize(DL);
+    IRBuilder<> IRB(&*FirstInst);
+    IRB.CreateMemCpy(AllocA, AllocA->getAlign(), ArgInParam, AllocA->getAlign(),
+                     ArgSize);
   }
 }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 2eb8b17f1b0f40fec694715de29a294fd291bb9a..7d04cf3dc51e673af1a31a8057b7fa6f59a997b0 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -445,7 +445,7 @@ void NVPTXPassConfig::addMachineSSAOptimization() {
 
   // This pass merges large allocas. StackSlotColoring is a different pass
   // which merges spill slots.
-  addPass(&StackColoringID);
+  addPass(&StackColoringLegacyID);
 
   // If the target requests it, assign local variables to stack slots relative
   // to one another and simplify frame index references where possible.
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 9a8ea8f87896adc8284b25566ddcde428afbb2ac..b141229dcfc7333755e550c808f5c7b22e1d6956 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -360,7 +360,8 @@ static Instruction *simplifyNvvmIntrinsic(IntrinsicInst *II, InstCombiner &IC) {
     // type argument, equal to that of the nvvm intrinsic's argument.
     Type *Tys[] = {II->getArgOperand(0)->getType()};
     return CallInst::Create(
-        Intrinsic::getDeclaration(II->getModule(), *Action.IID, Tys), Args);
+        Intrinsic::getOrInsertDeclaration(II->getModule(), *Action.IID, Tys),
+        Args);
   }
 
   // Simplify to target-generic binary op.
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index 2d62f34726e88c888ebb9719ed55414059463a6a..44c50827764b083f4676191c55b185d290b8b20d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -87,8 +87,9 @@ static void cacheAnnotationFromMD(const MDNode *MetadataNode,
       // assert: there can only exist one unique key value pair of
       // the form (string key, MDNode node). Operands of such a node
       // shall always be unsigned ints.
-      if (retval.find(Key) == retval.end()) {
-        readIntVecFromMDNode(VecMd, retval[Key]);
+      auto [It, Inserted] = retval.try_emplace(Key);
+      if (Inserted) {
+        readIntVecFromMDNode(VecMd, It->second);
         continue;
       }
     } else {
@@ -122,13 +123,7 @@ static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {
   if (tmp.empty()) // no annotations for this gv
     return;
 
-  if (AC.Cache.find(m) != AC.Cache.end())
-    AC.Cache[m][gv] = std::move(tmp);
-  else {
-    global_val_annot_t tmp1;
-    tmp1[gv] = std::move(tmp);
-    AC.Cache[m] = std::move(tmp1);
-  }
+  AC.Cache[m][gv] = std::move(tmp);
 }
 
 static std::optional<unsigned> findOneNVVMAnnotation(const GlobalValue *gv,
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index d9847a21489e639b3e3ae968eac6c4124c560b72..911d92f0c4846bbdcdfba489629be7febde4b253 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -12181,7 +12181,7 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
 
 static Instruction *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
-  Function *Func = Intrinsic::getDeclaration(M, Id);
+  Function *Func = Intrinsic::getOrInsertDeclaration(M, Id);
   return Builder.CreateCall(Func, {});
 }
 
@@ -12206,7 +12206,7 @@ Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
     // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
     if (isa<LoadInst>(Inst))
       return Builder.CreateCall(
-          Intrinsic::getDeclaration(
+          Intrinsic::getOrInsertDeclaration(
               Builder.GetInsertBlock()->getParent()->getParent(),
               Intrinsic::ppc_cfence, {Inst->getType()}),
           {Inst});
@@ -19005,7 +19005,7 @@ Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic(
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
   Type *ValTy = Incr->getType();
   assert(ValTy->getPrimitiveSizeInBits() == 128);
-  Function *RMW = Intrinsic::getDeclaration(
+  Function *RMW = Intrinsic::getOrInsertDeclaration(
       M, getIntrinsicForAtomicRMWBinOp128(AI->getOperation()));
   Type *Int64Ty = Type::getInt64Ty(M->getContext());
   Value *IncrLo = Builder.CreateTrunc(Incr, Int64Ty, "incr_lo");
@@ -19028,7 +19028,7 @@ Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
   Type *ValTy = CmpVal->getType();
   assert(ValTy->getPrimitiveSizeInBits() == 128);
   Function *IntCmpXchg =
-      Intrinsic::getDeclaration(M, Intrinsic::ppc_cmpxchg_i128);
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::ppc_cmpxchg_i128);
   Type *Int64Ty = Type::getInt64Ty(M->getContext());
   Value *CmpLo = Builder.CreateTrunc(CmpVal, Int64Ty, "cmp_lo");
   Value *CmpHi =
diff --git a/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp b/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp
index d10fe11bb5877b256b3a88c6dd5e733a89ec513f..9c2b58a47392f9d08923c16dc764c0331e474d5e 100644
--- a/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp
+++ b/llvm/lib/Target/PowerPC/PPCLowerMASSVEntries.cpp
@@ -123,7 +123,7 @@ bool PPCLowerMASSVEntries::handlePowSpecialCases(CallInst *CI, Function &Func,
         return false;
 
       CI->setCalledFunction(
-          Intrinsic::getDeclaration(&M, Intrinsic::pow, CI->getType()));
+          Intrinsic::getOrInsertDeclaration(&M, Intrinsic::pow, CI->getType()));
       return true;
     }
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index f9d0c053417905d26603355edd600716a892bffb..0b312c29061c9c1ddc5e9ea887159988be0936d7 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -3217,12 +3217,11 @@ bool RISCVAsmParser::parseDirectiveInsn(SMLoc L) {
     // Try parsing .insn [ length , ] value
     std::optional<int64_t> Length;
     int64_t Value = 0;
-    if (Parser.parseIntToken(
-            Value, "expected instruction format or an integer constant"))
+    if (Parser.parseAbsoluteExpression(Value))
       return true;
     if (Parser.parseOptionalToken(AsmToken::Comma)) {
       Length = Value;
-      if (Parser.parseIntToken(Value, "expected an integer constant"))
+      if (Parser.parseAbsoluteExpression(Value))
         return true;
 
       if (*Length == 0 || (*Length % 2) != 0)
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index c593d87bf5409baa02d292083693d9552f5f68ca..6005d8df2ff98b9e134aca203b2ff0a5af6f063c 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -59,6 +59,7 @@ add_llvm_target(RISCVCodeGen
   RISCVTargetObjectFile.cpp
   RISCVTargetTransformInfo.cpp
   RISCVVectorPeephole.cpp
+  RISCVVLOptimizer.cpp
   RISCVZacasABIFix.cpp
   GISel/RISCVCallLowering.cpp
   GISel/RISCVInstructionSelector.cpp
diff --git a/llvm/lib/Target/RISCV/EPIFoldBroadcast.cpp b/llvm/lib/Target/RISCV/EPIFoldBroadcast.cpp
index f2dd10b89401c59cb94712c1471d94bd4ed8c20a..6772ae97ee2ea84de3e85caebfadb58938bda924 100644
--- a/llvm/lib/Target/RISCV/EPIFoldBroadcast.cpp
+++ b/llvm/lib/Target/RISCV/EPIFoldBroadcast.cpp
@@ -241,7 +241,7 @@ bool EPIFoldBroadcast::foldBroadcasts(Function &F) {
       continue;
       break;
     }
-    Function *NewIntrinsic = Intrinsic::getDeclaration(
+    Function *NewIntrinsic = Intrinsic::getOrInsertDeclaration(
         F.getParent(), (Intrinsic::ID)EII->IntrinsicID, NewIntrinsicTypes);
 
     Instruction *CurrentCall = cast<Instruction>(FI.User);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index cfbd066d1c1401e689c6a760e8cf8b3300b581f7..efab9ad177797aa86adfc75fe2905349769e3337 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -81,7 +81,7 @@ public:
 
   /// Return binary encoding of operand. If the machine operand requires
   /// relocation, record the relocation and return zero.
-  unsigned getMachineOpValue(const MCInst &MI, const MCOperand &MO,
+  uint64_t getMachineOpValue(const MCInst &MI, const MCOperand &MO,
                              SmallVectorImpl<MCFixup> &Fixups,
                              const MCSubtargetInfo &STI) const;
 
@@ -413,7 +413,7 @@ void RISCVMCCodeEmitter::encodeInstruction(const MCInst &MI,
   ++MCNumEmitted; // Keep track of the # of mi's emitted.
 }
 
-unsigned
+uint64_t
 RISCVMCCodeEmitter::getMachineOpValue(const MCInst &MI, const MCOperand &MO,
                                       SmallVectorImpl<MCFixup> &Fixups,
                                       const MCSubtargetInfo &STI) const {
@@ -422,7 +422,7 @@ RISCVMCCodeEmitter::getMachineOpValue(const MCInst &MI, const MCOperand &MO,
     return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg());
 
   if (MO.isImm())
-    return static_cast<unsigned>(MO.getImm());
+    return MO.getImm();
 
   llvm_unreachable("Unhandled expression!");
   return 0;
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index 7e6c44326b17fd9c073b424bd97b260068f2e44d..2c36625c3d66300c1007d254ac3baa58b0591ea2 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -108,6 +108,9 @@ void initializeRISCVO0PreLegalizerCombinerPass(PassRegistry &);
 FunctionPass *createRISCVPreLegalizerCombiner();
 void initializeRISCVPreLegalizerCombinerPass(PassRegistry &);
 
+FunctionPass *createRISCVVLOptimizerPass();
+void initializeRISCVVLOptimizerPass(PassRegistry &);
+
 ModulePass *createEPIFMAContractionPass();
 void initializeEPIFMAContractionPass(PassRegistry &);
 } // namespace llvm
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 6d48b547efb90c263c0fb6eae66fb748bac00d78..e4adf310be49977b35fd9f62e09e2ce919301ced 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -305,7 +305,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::VASTART, MVT::Other, Custom);
   setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
 
-  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
+  if (!Subtarget.hasVendorXTHeadBb())
+    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 
   setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
 
@@ -416,13 +417,13 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::SELECT, XLenVT, Custom);
 
   static const unsigned FPLegalNodeTypes[] = {
-      ISD::FMINNUM,       ISD::FMAXNUM,       ISD::FMINIMUMNUM,
-      ISD::FMAXIMUMNUM,   ISD::LRINT,         ISD::LLRINT,
-      ISD::LROUND,        ISD::LLROUND,       ISD::STRICT_LRINT,
-      ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
-      ISD::STRICT_FMA,    ISD::STRICT_FADD,   ISD::STRICT_FSUB,
-      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,   ISD::STRICT_FSQRT,
-      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS};
+      ISD::FMINNUM,       ISD::FMAXNUM,        ISD::FMINIMUMNUM,
+      ISD::FMAXIMUMNUM,   ISD::LRINT,          ISD::LLRINT,
+      ISD::LROUND,        ISD::LLROUND,        ISD::STRICT_LRINT,
+      ISD::STRICT_LLRINT, ISD::STRICT_LROUND,  ISD::STRICT_LLROUND,
+      ISD::STRICT_FMA,    ISD::STRICT_FADD,    ISD::STRICT_FSUB,
+      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,    ISD::STRICT_FSQRT,
+      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
 
   static const unsigned FPLegalNodeTypesWithPromoteableResult[] = {
       ISD::LRINT,
@@ -461,7 +462,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
       ISD::SETCC,        ISD::FCEIL,         ISD::FFLOOR,
       ISD::FTRUNC,       ISD::FRINT,         ISD::FROUND,
-      ISD::FROUNDEVEN};
+      ISD::FROUNDEVEN,   ISD::FCANONICALIZE};
 
   if (Subtarget.hasStdExtZfbfmin()) {
     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
@@ -15546,8 +15547,10 @@ static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
   return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
 }
 
-// Looks for (sub (shl X, 8), X) where only bits 8, 16, 24, 32, etc. of X are
-// non-zero. Replace with orc.b.
+// Looks for (sub (shl X, 8-Y), (shr X, Y)) where the Y-th bit in each byte is
+// potentially set. It is fine for Y to be 0, meaning that (sub (shl X, 8), X)
+// is also valid. Replace with (orc.b X). For example, 0b0000_1000_0000_1000 is
+// valid with Y=3, while 0b0000_1000_0000_0100 is not.
 static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
   if (!Subtarget.hasStdExtZbb())
@@ -15561,18 +15564,44 @@ static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
-  if (N0.getOpcode() != ISD::SHL || N0.getOperand(0) != N1 || !N0.hasOneUse())
+  if (N0->getOpcode() != ISD::SHL)
+    return SDValue();
+
+  auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(N0.getOperand(1));
+  if (!ShAmtCLeft)
+    return SDValue();
+  unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
+
+  if (ShiftedAmount >= 8)
+    return SDValue();
+
+  SDValue LeftShiftOperand = N0->getOperand(0);
+  SDValue RightShiftOperand = N1;
+
+  if (ShiftedAmount != 0) { // Right operand must be a right shift.
+    if (N1->getOpcode() != ISD::SRL)
+      return SDValue();
+    auto *ShAmtCRight = dyn_cast<ConstantSDNode>(N1.getOperand(1));
+    if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
+      return SDValue();
+    RightShiftOperand = N1.getOperand(0);
+  }
+
+  // At least one shift should have a single use.
+  if (!N0.hasOneUse() && (ShiftedAmount == 0 || !N1.hasOneUse()))
     return SDValue();
 
-  auto *ShAmtC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
-  if (!ShAmtC || ShAmtC->getZExtValue() != 8)
+  if (LeftShiftOperand != RightShiftOperand)
     return SDValue();
 
-  APInt Mask = APInt::getSplat(VT.getSizeInBits(), APInt(8, 0xfe));
-  if (!DAG.MaskedValueIsZero(N1, Mask))
+  APInt Mask = APInt::getSplat(VT.getSizeInBits(), APInt(8, 0x1));
+  Mask <<= ShiftedAmount;
+  // Check that X has indeed the right shape (only the Y-th bit can be set in
+  // every byte).
+  if (!DAG.MaskedValueIsZero(LeftShiftOperand, ~Mask))
     return SDValue();
 
-  return DAG.getNode(RISCVISD::ORC_B, SDLoc(N), VT, N1);
+  return DAG.getNode(RISCVISD::ORC_B, SDLoc(N), VT, LeftShiftOperand);
 }
 
 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
@@ -22610,7 +22639,7 @@ RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
       return C_Other;
     }
   } else {
-    if (Constraint == "vr" || Constraint == "vm")
+    if (Constraint == "vr" || Constraint == "vd" || Constraint == "vm")
       return C_RegisterClass;
   }
   return TargetLowering::getConstraintType(Constraint);
@@ -22658,6 +22687,19 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
         return std::make_pair(0U, RC);
     }
+  } else if (Constraint == "vd") {
+    for (const auto *RC :
+         {&RISCV::VRNoV0RegClass, &RISCV::VRM2NoV0RegClass,
+          &RISCV::VRM4NoV0RegClass, &RISCV::VRM8NoV0RegClass,
+          &RISCV::VRN2M1NoV0RegClass, &RISCV::VRN3M1NoV0RegClass,
+          &RISCV::VRN4M1NoV0RegClass, &RISCV::VRN5M1NoV0RegClass,
+          &RISCV::VRN6M1NoV0RegClass, &RISCV::VRN7M1NoV0RegClass,
+          &RISCV::VRN8M1NoV0RegClass, &RISCV::VRN2M2NoV0RegClass,
+          &RISCV::VRN3M2NoV0RegClass, &RISCV::VRN4M2NoV0RegClass,
+          &RISCV::VRN2M4NoV0RegClass}) {
+      if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
+        return std::make_pair(0U, RC);
+    }
   } else if (Constraint == "vm") {
     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
       return std::make_pair(0U, &RISCV::VMV0RegClass);
@@ -23020,7 +23062,7 @@ Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
   Value *Ordering =
       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
   Type *Tys[] = {AlignedAddr->getType()};
-  Function *LrwOpScwLoop = Intrinsic::getDeclaration(
+  Function *LrwOpScwLoop = Intrinsic::getOrInsertDeclaration(
       AI->getModule(),
       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
 
@@ -23084,7 +23126,7 @@ Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
   }
   Type *Tys[] = {AlignedAddr->getType()};
   Function *MaskedCmpXchg =
-      Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
+      Intrinsic::getOrInsertDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
   Value *Result = Builder.CreateCall(
       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
   if (XLen == 64)
@@ -23582,7 +23624,7 @@ bool RISCVTargetLowering::preferScalarizeSplat(SDNode *N) const {
 static Value *useTpOffset(IRBuilderBase &IRB, unsigned Offset) {
   Module *M = IRB.GetInsertBlock()->getModule();
   Function *ThreadPointerFunc =
-      Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::thread_pointer);
   return IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
                                 IRB.CreateCall(ThreadPointerFunc), Offset);
 }
@@ -23699,9 +23741,9 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
 
   auto *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen());
 
-  Function *VlsegNFunc =
-      Intrinsic::getDeclaration(LI->getModule(), FixedVlsegIntrIds[Factor - 2],
-                                {VTy, LI->getPointerOperandType(), XLenTy});
+  Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
+      LI->getModule(), FixedVlsegIntrIds[Factor - 2],
+      {VTy, LI->getPointerOperandType(), XLenTy});
 
   Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
 
@@ -23753,9 +23795,9 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
 
   auto *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
 
-  Function *VssegNFunc =
-      Intrinsic::getDeclaration(SI->getModule(), FixedVssegIntrIds[Factor - 2],
-                                {VTy, SI->getPointerOperandType(), XLenTy});
+  Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
+      SI->getModule(), FixedVssegIntrIds[Factor - 2],
+      {VTy, SI->getPointerOperandType(), XLenTy});
 
   auto Mask = SVI->getShuffleMask();
   SmallVector<Value *, 10> Ops;
@@ -23800,7 +23842,7 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
   Type *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen());
 
   if (auto *FVTy = dyn_cast<FixedVectorType>(ResVTy)) {
-    Function *VlsegNFunc = Intrinsic::getDeclaration(
+    Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
         LI->getModule(), FixedVlsegIntrIds[Factor - 2],
         {ResVTy, LI->getPointerOperandType(), XLenTy});
     Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
@@ -23820,7 +23862,7 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
                                 NumElts * SEW / 8),
         Factor);
 
-    Function *VlsegNFunc = Intrinsic::getDeclaration(
+    Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
         LI->getModule(), IntrIds[Factor - 2], {VecTupTy, XLenTy});
     Value *VL = Constant::getAllOnesValue(XLenTy);
 
@@ -23830,7 +23872,7 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
 
     SmallVector<Type *, 2> AggrTypes{Factor, ResVTy};
     Return = PoisonValue::get(StructType::get(LI->getContext(), AggrTypes));
-    Function *VecExtractFunc = Intrinsic::getDeclaration(
+    Function *VecExtractFunc = Intrinsic::getOrInsertDeclaration(
         LI->getModule(), Intrinsic::riscv_tuple_extract, {ResVTy, VecTupTy});
     for (unsigned i = 0; i < Factor; ++i) {
       Value *VecExtract =
@@ -23866,7 +23908,7 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
   Type *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
 
   if (auto *FVTy = dyn_cast<FixedVectorType>(InVTy)) {
-    Function *VssegNFunc = Intrinsic::getDeclaration(
+    Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
         SI->getModule(), FixedVssegIntrIds[Factor - 2],
         {InVTy, SI->getPointerOperandType(), XLenTy});
     Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
@@ -23887,12 +23929,12 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
                                 NumElts * SEW / 8),
         Factor);
 
-    Function *VssegNFunc = Intrinsic::getDeclaration(
+    Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
         SI->getModule(), IntrIds[Factor - 2], {VecTupTy, XLenTy});
 
     Value *VL = Constant::getAllOnesValue(XLenTy);
 
-    Function *VecInsertFunc = Intrinsic::getDeclaration(
+    Function *VecInsertFunc = Intrinsic::getOrInsertDeclaration(
         SI->getModule(), Intrinsic::riscv_tuple_insert, {VecTupTy, InVTy});
     Value *StoredVal = PoisonValue::get(VecTupTy);
     for (unsigned i = 0; i < Factor; ++i)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
index 7d742322b42969cf99d5e561bf7b65e719a10121..e8c4860fd3e55ee707a6e6584d7bf4534c9a7ae3 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -279,9 +279,9 @@ class Bcz<bits<3> funct3, string OpcodeStr,
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class Shift_right<bits<2> funct2, string OpcodeStr, RegisterClass cls,
                   Operand ImmOpnd>
-    : RVInst16CB<0b100, 0b01, (outs cls:$rs1_wb), (ins cls:$rs1, ImmOpnd:$imm),
+    : RVInst16CB<0b100, 0b01, (outs cls:$rd), (ins cls:$rs1, ImmOpnd:$imm),
                  OpcodeStr, "$rs1, $imm"> {
-  let Constraints = "$rs1 = $rs1_wb";
+  let Constraints = "$rs1 = $rd";
   let Inst{12} = imm{5};
   let Inst{11-10} = funct2;
   let Inst{6-2} = imm{4-0};
@@ -401,9 +401,11 @@ def C_SD : CStore_rri<0b111, "c.sd", GPRC, uimm8_lsb000>,
   let Inst{6-5} = imm{7-6};
 }
 
-let rd = 0, imm = 0, hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def C_NOP : RVInst16CI<0b000, 0b01, (outs), (ins), "c.nop", "">,
             Sched<[WriteNop]> {
+  let rd = 0;
+  let imm = 0;
   let Inst{6-2} = 0;
 }
 
@@ -477,10 +479,10 @@ def C_SRAI : Shift_right<0b01, "c.srai", GPRC, uimmlog2xlennonzero>,
              Sched<[WriteShiftImm, ReadShiftImm]>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-def C_ANDI : RVInst16CB<0b100, 0b01, (outs GPRC:$rs1_wb), (ins GPRC:$rs1, simm6:$imm),
+def C_ANDI : RVInst16CB<0b100, 0b01, (outs GPRC:$rd), (ins GPRC:$rs1, simm6:$imm),
                         "c.andi", "$rs1, $imm">,
              Sched<[WriteIALU, ReadIALU]> {
-  let Constraints = "$rs1 = $rs1_wb";
+  let Constraints = "$rs1 = $rd";
   let Inst{12} = imm{5};
   let Inst{11-10} = 0b10;
   let Inst{6-2} = imm{4-0};
@@ -580,11 +582,11 @@ def C_JALR : RVInst16CR<0b1001, 0b10, (outs), (ins GPRNoX0:$rs1),
                         "c.jalr", "$rs1">, Sched<[WriteJalr, ReadJalr]>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-def C_ADD : RVInst16CR<0b1001, 0b10, (outs GPRNoX0:$rs1_wb),
+def C_ADD : RVInst16CR<0b1001, 0b10, (outs GPRNoX0:$rd),
                        (ins GPRNoX0:$rs1, GPRNoX0:$rs2),
                        "c.add", "$rs1, $rs2">,
             Sched<[WriteIALU, ReadIALU, ReadIALU]> {
-  let Constraints = "$rs1 = $rs1_wb";
+  let Constraints = "$rs1 = $rd";
 }
 
 let Predicates = [HasStdExtCOrZcd, HasStdExtD] in
@@ -639,9 +641,9 @@ def C_UNIMP : RVInst16<(outs), (ins), "c.unimp", "", [], InstFormatOther>,
 let Predicates = [HasStdExtCOrZca, HasRVCHints], hasSideEffects = 0, mayLoad = 0,
     mayStore = 0 in {
 
-let rd = 0 in
 def C_NOP_HINT : RVInst16CI<0b000, 0b01, (outs), (ins simm6nonzero:$imm),
                             "c.nop", "$imm">, Sched<[WriteNop]> {
+  let rd = 0;
   let Inst{6-2} = imm{4-0};
 }
 
@@ -678,11 +680,11 @@ def C_MV_HINT : RVInst16CR<0b1000, 0b10, (outs GPRX0:$rs1), (ins GPRNoX0:$rs2),
   let DecoderMethod = "decodeRVCInstrRdRs2";
 }
 
-def C_ADD_HINT : RVInst16CR<0b1001, 0b10, (outs GPRX0:$rs1_wb),
+def C_ADD_HINT : RVInst16CR<0b1001, 0b10, (outs GPRX0:$rd),
                             (ins GPRX0:$rs1, GPRNoX0:$rs2),
                             "c.add", "$rs1, $rs2">,
                  Sched<[WriteIALU, ReadIALU, ReadIALU]> {
-  let Constraints = "$rs1 = $rs1_wb";
+  let Constraints = "$rs1 = $rd";
   let Inst{11-7} = 0;
   let DecoderMethod = "decodeRVCInstrRdRs1Rs2";
 }
@@ -705,23 +707,23 @@ def C_SLLI64_HINT : RVInst16CI<0b000, 0b10, (outs GPR:$rd_wb), (ins GPR:$rd),
   let Inst{12} = 0;
 }
 
-def C_SRLI64_HINT : RVInst16CI<0b100, 0b01, (outs GPRC:$rd_wb),
-                               (ins GPRC:$rd),
-                               "c.srli64", "$rd">,
+def C_SRLI64_HINT : RVInst16CB<0b100, 0b01, (outs GPRC:$rd),
+                               (ins GPRC:$rs1),
+                               "c.srli64", "$rs1">,
                     Sched<[WriteShiftImm, ReadShiftImm]> {
-  let Constraints = "$rd = $rd_wb";
+  let Constraints = "$rs1 = $rd";
   let Inst{6-2} = 0;
-  let Inst{11-10} = 0;
+  let Inst{11-10} = 0b00;
   let Inst{12} = 0;
 }
 
-def C_SRAI64_HINT : RVInst16CI<0b100, 0b01, (outs GPRC:$rd_wb),
-                               (ins GPRC:$rd),
-                               "c.srai64", "$rd">,
+def C_SRAI64_HINT : RVInst16CB<0b100, 0b01, (outs GPRC:$rd),
+                               (ins GPRC:$rs1),
+                               "c.srai64", "$rs1">,
                     Sched<[WriteShiftImm, ReadShiftImm]> {
-  let Constraints = "$rd = $rd_wb";
+  let Constraints = "$rs1 = $rd";
   let Inst{6-2} = 0;
-  let Inst{11-10} = 1;
+  let Inst{11-10} = 0b01;
   let Inst{12} = 0;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
index 1f19c585798045ad4c56c400c645fcec8636541b..74e54fd90a423b74ee1e0c8b7ef5bcf7eff05416 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -398,6 +398,7 @@ foreach Ext = DExts in {
   defm : PatFprFpr_m<fmaximumnum, FMAX_D, Ext>;
   defm : PatFprFpr_m<riscv_fmin, FMIN_D, Ext>;
   defm : PatFprFpr_m<riscv_fmax, FMAX_D, Ext>;
+  def : Pat<(f64 (fcanonicalize FPR64:$rs1)), (FMIN_D $rs1, $rs1)>;
 }
 
 /// Setcc
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index a989678363af60d88a3f8e3ab3a5a41b752149c4..4c58af0b379add5ad550ad3d6eaeff1ff9c43763 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -623,6 +623,7 @@ foreach Ext = FExts in {
   defm : PatFprFpr_m<fmaximumnum, FMAX_S, Ext>;
   defm : PatFprFpr_m<riscv_fmin, FMIN_S, Ext>;
   defm : PatFprFpr_m<riscv_fmax, FMAX_S, Ext>;
+  def : Pat<(f32 (fcanonicalize FPR32:$rs1)), (FMIN_S $rs1, $rs1)>;
 }
 
 /// Setcc
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index 851e817c50125041d7f3cb6bc51649a40b2ae7f1..5068d0be0fb49bd2df474a897d085e12d61f4ca5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -305,6 +305,7 @@ class VPseudoVC_V_XVV<Operand OpClass, VReg RDClass, VReg RS2Class,
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let hasSideEffects = 0;
+  let Constraints = "$rd = $rs3";
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
index 51123180d47c699337f9c2f0c3605ab2ea3e46a8..0d3127e0d5abeb36aaa107570eb62427443f68fd 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
@@ -369,6 +369,7 @@ foreach Ext = ZfhExts in {
   defm : PatFprFpr_m<fmaximumnum, FMAX_H, Ext>;
   defm : PatFprFpr_m<riscv_fmin, FMIN_H, Ext>;
   defm : PatFprFpr_m<riscv_fmax, FMAX_H, Ext>;
+  def : Pat<(f16 (fcanonicalize FPR16:$rs1)), (FMIN_H $rs1, $rs1)>;
 }
 
 /// Setcc
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 5cc7aa1432ec4f2af50a739bb386af93298bfb9b..69ebbf2eb25d32e7cb520245b66c26fde13bf1c4 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -97,9 +97,9 @@ static cl::opt<bool>
                            cl::desc("Enable the loop data prefetch pass"),
                            cl::init(true));
 
-static cl::opt<bool> EnableMISchedLoadClustering(
-    "riscv-misched-load-clustering", cl::Hidden,
-    cl::desc("Enable load clustering in the machine scheduler"),
+static cl::opt<bool> EnableMISchedLoadStoreClustering(
+    "riscv-misched-load-store-clustering", cl::Hidden,
+    cl::desc("Enable load and store clustering in the machine scheduler"),
     cl::init(true));
 
 static cl::opt<bool> EnableVSETVLIAfterRVVRegAlloc(
@@ -107,6 +107,11 @@ static cl::opt<bool> EnableVSETVLIAfterRVVRegAlloc(
     cl::desc("Insert vsetvls after vector register allocation"),
     cl::init(true));
 
+static cl::opt<bool>
+    EnableVLOptimizer("riscv-enable-vl-optimizer",
+                      cl::desc("Enable the RISC-V VL Optimizer pass"),
+                      cl::init(false), cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
@@ -126,6 +131,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVPreRAExpandPseudoPass(*PR);
   initializeRISCVExpandPseudoPass(*PR);
   initializeRISCVVectorPeepholePass(*PR);
+  initializeRISCVVLOptimizerPass(*PR);
   initializeRISCVInsertVSETVLIPass(*PR);
   initializeRISCVInsertReadWriteCSRPass(*PR);
   initializeRISCVInsertWriteVXRMPass(*PR);
@@ -360,10 +366,12 @@ public:
   ScheduleDAGInstrs *
   createMachineScheduler(MachineSchedContext *C) const override {
     ScheduleDAGMILive *DAG = nullptr;
-    if (EnableMISchedLoadClustering) {
+    if (EnableMISchedLoadStoreClustering) {
       DAG = createGenericSchedLive(C);
       DAG->addMutation(createLoadClusterDAGMutation(
           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+      DAG->addMutation(createStoreClusterDAGMutation(
+          DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
     }
     return DAG;
   }
@@ -574,8 +582,11 @@ void RISCVPassConfig::addMachineSSAOptimization() {
 
 void RISCVPassConfig::addPreRegAlloc() {
   addPass(createRISCVPreRAExpandPseudoPass());
-  if (TM->getOptLevel() != CodeGenOptLevel::None)
+  if (TM->getOptLevel() != CodeGenOptLevel::None) {
     addPass(createRISCVMergeBaseOffsetOptPass());
+    if (EnableVLOptimizer)
+      addPass(createRISCVVLOptimizerPass());
+  }
 
   addPass(createRISCVInsertReadWriteCSRPass());
   addPass(createRISCVInsertWriteVXRMPass());
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 680d2bb2c16e9d2b10235fbdfea5e0cb99aede80..2864902c86dfd28087380323d53ee987ab9c7db2 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -418,6 +418,12 @@ public:
     return TLI->isLegalStridedLoadStore(DataTypeVT, Alignment);
   }
 
+  bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+                                    Align Alignment, unsigned AddrSpace) {
+    return TLI->isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace,
+                                             DL);
+  }
+
   bool isLegalMaskedCompressStore(Type *DataTy, Align Alignment);
 
   bool isVScaleKnownToBeAPowerOfTwo() const {
diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1a9084f8b6cb2bef2e38108e26c51aac185854f9
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -0,0 +1,844 @@
+//===-------------- RISCVVLOptimizer.cpp - VL Optimizer -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This pass reduces the VL where possible at the MI level, before VSETVLI
+// instructions are inserted.
+//
+// The purpose of this optimization is to make the VL argument, for instructions
+// that have a VL argument, as small as possible. This is implemented by
+// visiting each instruction in reverse order and checking that if it has a VL
+// argument, whether the VL can be reduced.
+//
+//===---------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVMachineFunctionInfo.h"
+#include "RISCVSubtarget.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "riscv-vl-optimizer"
+#define PASS_NAME "RISC-V VL Optimizer"
+
+namespace {
+
+class RISCVVLOptimizer : public MachineFunctionPass {
+  const MachineRegisterInfo *MRI;
+  const MachineDominatorTree *MDT;
+
+public:
+  static char ID;
+
+  RISCVVLOptimizer() : MachineFunctionPass(ID) {}
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<MachineDominatorTreeWrapperPass>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  StringRef getPassName() const override { return PASS_NAME; }
+
+private:
+  bool checkUsers(std::optional<Register> &CommonVL, MachineInstr &MI);
+  bool tryReduceVL(MachineInstr &MI);
+  bool isCandidate(const MachineInstr &MI) const;
+};
+
+} // end anonymous namespace
+
+char RISCVVLOptimizer::ID = 0;
+INITIALIZE_PASS_BEGIN(RISCVVLOptimizer, DEBUG_TYPE, PASS_NAME, false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_END(RISCVVLOptimizer, DEBUG_TYPE, PASS_NAME, false, false)
+
+FunctionPass *llvm::createRISCVVLOptimizerPass() {
+  return new RISCVVLOptimizer();
+}
+
+/// Return true if R is a physical or virtual vector register, false otherwise.
+static bool isVectorRegClass(Register R, const MachineRegisterInfo *MRI) {
+  if (R.isPhysical())
+    return RISCV::VRRegClass.contains(R);
+  const TargetRegisterClass *RC = MRI->getRegClass(R);
+  return RISCVRI::isVRegClass(RC->TSFlags);
+}
+
+/// Represents the EMUL and EEW of a MachineOperand.
+struct OperandInfo {
+  enum class State {
+    Unknown,
+    Known,
+  } S;
+
+  // Represent as 1,2,4,8, ... and fractional indicator. This is because
+  // EMUL can take on values that don't map to RISCVII::VLMUL values exactly.
+  // For example, a mask operand can have an EMUL less than MF8.
+  std::optional<std::pair<unsigned, bool>> EMUL;
+
+  unsigned Log2EEW;
+
+  OperandInfo(RISCVII::VLMUL EMUL, unsigned Log2EEW)
+      : S(State::Known), EMUL(RISCVVType::decodeVLMUL(EMUL)), Log2EEW(Log2EEW) {
+  }
+
+  OperandInfo(std::pair<unsigned, bool> EMUL, unsigned Log2EEW)
+      : S(State::Known), EMUL(EMUL), Log2EEW(Log2EEW) {}
+
+  OperandInfo() : S(State::Unknown) {}
+
+  bool isUnknown() const { return S == State::Unknown; }
+  bool isKnown() const { return S == State::Known; }
+
+  static bool EMULAndEEWAreEqual(const OperandInfo &A, const OperandInfo &B) {
+    assert(A.isKnown() && B.isKnown() && "Both operands must be known");
+
+    return A.Log2EEW == B.Log2EEW && A.EMUL->first == B.EMUL->first &&
+           A.EMUL->second == B.EMUL->second;
+  }
+
+  void print(raw_ostream &OS) const {
+    if (isUnknown()) {
+      OS << "Unknown";
+      return;
+    }
+    assert(EMUL && "Expected EMUL to have value");
+    OS << "EMUL: ";
+    if (EMUL->second)
+      OS << "m";
+    OS << "f" << EMUL->first;
+    OS << ", EEW: " << (1 << Log2EEW);
+  }
+};
+
+LLVM_ATTRIBUTE_UNUSED
+static raw_ostream &operator<<(raw_ostream &OS, const OperandInfo &OI) {
+  OI.print(OS);
+  return OS;
+}
+
+namespace llvm {
+namespace RISCVVType {
+/// Return the RISCVII::VLMUL that is two times VLMul.
+/// Precondition: VLMul is not LMUL_RESERVED or LMUL_8.
+static RISCVII::VLMUL twoTimesVLMUL(RISCVII::VLMUL VLMul) {
+  switch (VLMul) {
+  case RISCVII::VLMUL::LMUL_F8:
+    return RISCVII::VLMUL::LMUL_F4;
+  case RISCVII::VLMUL::LMUL_F4:
+    return RISCVII::VLMUL::LMUL_F2;
+  case RISCVII::VLMUL::LMUL_F2:
+    return RISCVII::VLMUL::LMUL_1;
+  case RISCVII::VLMUL::LMUL_1:
+    return RISCVII::VLMUL::LMUL_2;
+  case RISCVII::VLMUL::LMUL_2:
+    return RISCVII::VLMUL::LMUL_4;
+  case RISCVII::VLMUL::LMUL_4:
+    return RISCVII::VLMUL::LMUL_8;
+  case RISCVII::VLMUL::LMUL_8:
+  default:
+    llvm_unreachable("Could not multiply VLMul by 2");
+  }
+}
+
+/// Return EMUL = (EEW / SEW) * LMUL where EEW comes from Log2EEW and LMUL and
+/// SEW are from the TSFlags of MI.
+static std::pair<unsigned, bool>
+getEMULEqualsEEWDivSEWTimesLMUL(unsigned Log2EEW, const MachineInstr &MI) {
+  RISCVII::VLMUL MIVLMUL = RISCVII::getLMul(MI.getDesc().TSFlags);
+  auto [MILMUL, MILMULIsFractional] = RISCVVType::decodeVLMUL(MIVLMUL);
+  unsigned MILog2SEW =
+      MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
+  unsigned MISEW = 1 << MILog2SEW;
+
+  unsigned EEW = 1 << Log2EEW;
+  // Calculate (EEW/SEW)*LMUL preserving fractions less than 1. Use GCD
+  // to put fraction in simplest form.
+  unsigned Num = EEW, Denom = MISEW;
+  int GCD = MILMULIsFractional ? std::gcd(Num, Denom * MILMUL)
+                               : std::gcd(Num * MILMUL, Denom);
+  Num = MILMULIsFractional ? Num / GCD : Num * MILMUL / GCD;
+  Denom = MILMULIsFractional ? Denom * MILMUL / GCD : Denom / GCD;
+  return std::make_pair(Num > Denom ? Num : Denom, Denom > Num);
+}
+} // end namespace RISCVVType
+} // end namespace llvm
+
+/// Dest has EEW=SEW and EMUL=LMUL. Source EEW=SEW/Factor (i.e. F2 => EEW/2).
+/// Source has EMUL=(EEW/SEW)*LMUL. LMUL and SEW comes from TSFlags of MI.
+static OperandInfo getIntegerExtensionOperandInfo(unsigned Factor,
+                                                  const MachineInstr &MI,
+                                                  const MachineOperand &MO) {
+  RISCVII::VLMUL MIVLMul = RISCVII::getLMul(MI.getDesc().TSFlags);
+  unsigned MILog2SEW =
+      MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
+
+  if (MO.getOperandNo() == 0)
+    return OperandInfo(MIVLMul, MILog2SEW);
+
+  unsigned MISEW = 1 << MILog2SEW;
+  unsigned EEW = MISEW / Factor;
+  unsigned Log2EEW = Log2_32(EEW);
+
+  return OperandInfo(RISCVVType::getEMULEqualsEEWDivSEWTimesLMUL(Log2EEW, MI),
+                     Log2EEW);
+}
+
+/// Check whether MO is a mask operand of MI.
+static bool isMaskOperand(const MachineInstr &MI, const MachineOperand &MO,
+                          const MachineRegisterInfo *MRI) {
+
+  if (!MO.isReg() || !isVectorRegClass(MO.getReg(), MRI))
+    return false;
+
+  const MCInstrDesc &Desc = MI.getDesc();
+  return Desc.operands()[MO.getOperandNo()].RegClass == RISCV::VMV0RegClassID;
+}
+
+/// Return the OperandInfo for MO, which is an operand of MI.
+static OperandInfo getOperandInfo(const MachineInstr &MI,
+                                  const MachineOperand &MO,
+                                  const MachineRegisterInfo *MRI) {
+  const RISCVVPseudosTable::PseudoInfo *RVV =
+      RISCVVPseudosTable::getPseudoInfo(MI.getOpcode());
+  assert(RVV && "Could not find MI in PseudoTable");
+
+  // MI has a VLMUL and SEW associated with it. The RVV specification defines
+  // the LMUL and SEW of each operand and definition in relation to MI.VLMUL and
+  // MI.SEW.
+  RISCVII::VLMUL MIVLMul = RISCVII::getLMul(MI.getDesc().TSFlags);
+  unsigned MILog2SEW =
+      MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
+
+  const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(MI.getDesc());
+
+  // We bail out early for instructions that have passthru with non NoRegister,
+  // which means they are using TU policy. We are not interested in these
+  // since they must preserve the entire register content.
+  if (HasPassthru && MO.getOperandNo() == MI.getNumExplicitDefs() &&
+      (MO.getReg() != RISCV::NoRegister))
+    return {};
+
+  bool IsMODef = MO.getOperandNo() == 0;
+
+  // All mask operands have EEW=1, EMUL=(EEW/SEW)*LMUL
+  if (isMaskOperand(MI, MO, MRI))
+    return OperandInfo(RISCVVType::getEMULEqualsEEWDivSEWTimesLMUL(0, MI), 0);
+
+  // switch against BaseInstr to reduce number of cases that need to be
+  // considered.
+  switch (RVV->BaseInstr) {
+
+  // 6. Configuration-Setting Instructions
+  // Configuration setting instructions do not read or write vector registers
+  case RISCV::VSETIVLI:
+  case RISCV::VSETVL:
+  case RISCV::VSETVLI:
+    llvm_unreachable("Configuration setting instructions do not read or write "
+                     "vector registers");
+
+  // 11. Vector Integer Arithmetic Instructions
+  // 11.1. Vector Single-Width Integer Add and Subtract
+  case RISCV::VADD_VI:
+  case RISCV::VADD_VV:
+  case RISCV::VADD_VX:
+  case RISCV::VSUB_VV:
+  case RISCV::VSUB_VX:
+  case RISCV::VRSUB_VI:
+  case RISCV::VRSUB_VX:
+  // 11.5. Vector Bitwise Logical Instructions
+  // 11.6. Vector Single-Width Shift Instructions
+  // EEW=SEW. EMUL=LMUL.
+  case RISCV::VAND_VI:
+  case RISCV::VAND_VV:
+  case RISCV::VAND_VX:
+  case RISCV::VOR_VI:
+  case RISCV::VOR_VV:
+  case RISCV::VOR_VX:
+  case RISCV::VXOR_VI:
+  case RISCV::VXOR_VV:
+  case RISCV::VXOR_VX:
+  case RISCV::VSLL_VI:
+  case RISCV::VSLL_VV:
+  case RISCV::VSLL_VX:
+  case RISCV::VSRL_VI:
+  case RISCV::VSRL_VV:
+  case RISCV::VSRL_VX:
+  case RISCV::VSRA_VI:
+  case RISCV::VSRA_VV:
+  case RISCV::VSRA_VX:
+  // 11.9. Vector Integer Min/Max Instructions
+  // EEW=SEW. EMUL=LMUL.
+  case RISCV::VMINU_VV:
+  case RISCV::VMINU_VX:
+  case RISCV::VMIN_VV:
+  case RISCV::VMIN_VX:
+  case RISCV::VMAXU_VV:
+  case RISCV::VMAXU_VX:
+  case RISCV::VMAX_VV:
+  case RISCV::VMAX_VX:
+  // 11.10. Vector Single-Width Integer Multiply Instructions
+  // Source and Dest EEW=SEW and EMUL=LMUL.
+  case RISCV::VMUL_VV:
+  case RISCV::VMUL_VX:
+  case RISCV::VMULH_VV:
+  case RISCV::VMULH_VX:
+  case RISCV::VMULHU_VV:
+  case RISCV::VMULHU_VX:
+  case RISCV::VMULHSU_VV:
+  case RISCV::VMULHSU_VX:
+  // 11.11. Vector Integer Divide Instructions
+  // EEW=SEW. EMUL=LMUL.
+  case RISCV::VDIVU_VV:
+  case RISCV::VDIVU_VX:
+  case RISCV::VDIV_VV:
+  case RISCV::VDIV_VX:
+  case RISCV::VREMU_VV:
+  case RISCV::VREMU_VX:
+  case RISCV::VREM_VV:
+  case RISCV::VREM_VX:
+  // 11.13. Vector Single-Width Integer Multiply-Add Instructions
+  // EEW=SEW. EMUL=LMUL.
+  case RISCV::VMACC_VV:
+  case RISCV::VMACC_VX:
+  case RISCV::VNMSAC_VV:
+  case RISCV::VNMSAC_VX:
+  case RISCV::VMADD_VV:
+  case RISCV::VMADD_VX:
+  case RISCV::VNMSUB_VV:
+  case RISCV::VNMSUB_VX:
+  // 11.15. Vector Integer Merge Instructions
+  // EEW=SEW and EMUL=LMUL, except the mask operand has EEW=1 and EMUL=
+  // (EEW/SEW)*LMUL. Mask operand is handled before this switch.
+  case RISCV::VMERGE_VIM:
+  case RISCV::VMERGE_VVM:
+  case RISCV::VMERGE_VXM:
+  // 11.16. Vector Integer Move Instructions
+  // 12. Vector Fixed-Point Arithmetic Instructions
+  // 12.1. Vector Single-Width Saturating Add and Subtract
+  // 12.2. Vector Single-Width Averaging Add and Subtract
+  // EEW=SEW. EMUL=LMUL.
+  case RISCV::VMV_V_I:
+  case RISCV::VMV_V_V:
+  case RISCV::VMV_V_X:
+  case RISCV::VSADDU_VI:
+  case RISCV::VSADDU_VV:
+  case RISCV::VSADDU_VX:
+  case RISCV::VSADD_VI:
+  case RISCV::VSADD_VV:
+  case RISCV::VSADD_VX:
+  case RISCV::VSSUBU_VV:
+  case RISCV::VSSUBU_VX:
+  case RISCV::VSSUB_VV:
+  case RISCV::VSSUB_VX:
+  case RISCV::VAADDU_VV:
+  case RISCV::VAADDU_VX:
+  case RISCV::VAADD_VV:
+  case RISCV::VAADD_VX:
+  case RISCV::VASUBU_VV:
+  case RISCV::VASUBU_VX:
+  case RISCV::VASUB_VV:
+  case RISCV::VASUB_VX:
+  // 12.4. Vector Single-Width Scaling Shift Instructions
+  // EEW=SEW. EMUL=LMUL.
+  case RISCV::VSSRL_VI:
+  case RISCV::VSSRL_VV:
+  case RISCV::VSSRL_VX:
+  case RISCV::VSSRA_VI:
+  case RISCV::VSSRA_VV:
+  case RISCV::VSSRA_VX:
+  // 16. Vector Permutation Instructions
+  // 16.1. Integer Scalar Move Instructions
+  // 16.2. Floating-Point Scalar Move Instructions
+  // EMUL=LMUL. EEW=SEW.
+  case RISCV::VMV_X_S:
+  case RISCV::VMV_S_X:
+  case RISCV::VFMV_F_S:
+  case RISCV::VFMV_S_F:
+  // 16.3. Vector Slide Instructions
+  // EMUL=LMUL. EEW=SEW.
+  case RISCV::VSLIDEUP_VI:
+  case RISCV::VSLIDEUP_VX:
+  case RISCV::VSLIDEDOWN_VI:
+  case RISCV::VSLIDEDOWN_VX:
+  case RISCV::VSLIDE1UP_VX:
+  case RISCV::VFSLIDE1UP_VF:
+  case RISCV::VSLIDE1DOWN_VX:
+  case RISCV::VFSLIDE1DOWN_VF:
+  // 16.4. Vector Register Gather Instructions
+  // EMUL=LMUL. EEW=SEW. For mask operand, EMUL=1 and EEW=1.
+  case RISCV::VRGATHER_VI:
+  case RISCV::VRGATHER_VV:
+  case RISCV::VRGATHER_VX:
+  // 16.5. Vector Compress Instruction
+  // EMUL=LMUL. EEW=SEW.
+  case RISCV::VCOMPRESS_VM:
+    return OperandInfo(MIVLMul, MILog2SEW);
+
+  // 11.2. Vector Widening Integer Add/Subtract
+  // Def uses EEW=2*SEW and EMUL=2*LMUL. Operands use EEW=SEW and EMUL=LMUL.
+  case RISCV::VWADDU_VV:
+  case RISCV::VWADDU_VX:
+  case RISCV::VWSUBU_VV:
+  case RISCV::VWSUBU_VX:
+  case RISCV::VWADD_VV:
+  case RISCV::VWADD_VX:
+  case RISCV::VWSUB_VV:
+  case RISCV::VWSUB_VX:
+  case RISCV::VWSLL_VI:
+  // 11.12. Vector Widening Integer Multiply Instructions
+  // Source and Destination EMUL=LMUL. Destination EEW=2*SEW. Source EEW=SEW.
+  case RISCV::VWMUL_VV:
+  case RISCV::VWMUL_VX:
+  case RISCV::VWMULSU_VV:
+  case RISCV::VWMULSU_VX:
+  case RISCV::VWMULU_VV:
+  case RISCV::VWMULU_VX: {
+    unsigned Log2EEW = IsMODef ? MILog2SEW + 1 : MILog2SEW;
+    RISCVII::VLMUL EMUL =
+        IsMODef ? RISCVVType::twoTimesVLMUL(MIVLMul) : MIVLMul;
+    return OperandInfo(EMUL, Log2EEW);
+  }
+
+  // Def and Op1 uses EEW=2*SEW and EMUL=2*LMUL. Op2 uses EEW=SEW and EMUL=LMUL
+  case RISCV::VWADDU_WV:
+  case RISCV::VWADDU_WX:
+  case RISCV::VWSUBU_WV:
+  case RISCV::VWSUBU_WX:
+  case RISCV::VWADD_WV:
+  case RISCV::VWADD_WX:
+  case RISCV::VWSUB_WV:
+  case RISCV::VWSUB_WX:
+  // 11.14. Vector Widening Integer Multiply-Add Instructions
+  // Destination EEW=2*SEW and EMUL=2*LMUL. Source EEW=SEW and EMUL=LMUL.
+  // Even though the add is a 2*SEW addition, the operands of the add are the
+  // Dest which is 2*SEW and the result of the multiply which is 2*SEW.
+  case RISCV::VWMACCU_VV:
+  case RISCV::VWMACCU_VX:
+  case RISCV::VWMACC_VV:
+  case RISCV::VWMACC_VX:
+  case RISCV::VWMACCSU_VV:
+  case RISCV::VWMACCSU_VX:
+  case RISCV::VWMACCUS_VX: {
+    bool IsOp1 = HasPassthru ? MO.getOperandNo() == 2 : MO.getOperandNo() == 1;
+    bool TwoTimes = IsMODef || IsOp1;
+    unsigned Log2EEW = TwoTimes ? MILog2SEW + 1 : MILog2SEW;
+    RISCVII::VLMUL EMUL =
+        TwoTimes ? RISCVVType::twoTimesVLMUL(MIVLMul) : MIVLMul;
+    return OperandInfo(EMUL, Log2EEW);
+  }
+
+  // 11.3. Vector Integer Extension
+  case RISCV::VZEXT_VF2:
+  case RISCV::VSEXT_VF2:
+    return getIntegerExtensionOperandInfo(2, MI, MO);
+  case RISCV::VZEXT_VF4:
+  case RISCV::VSEXT_VF4:
+    return getIntegerExtensionOperandInfo(4, MI, MO);
+  case RISCV::VZEXT_VF8:
+  case RISCV::VSEXT_VF8:
+    return getIntegerExtensionOperandInfo(8, MI, MO);
+
+  // 11.7. Vector Narrowing Integer Right Shift Instructions
+  // Destination EEW=SEW and EMUL=LMUL, Op 1 has EEW=2*SEW EMUL=2*LMUL. Op2 has
+  // EEW=SEW EMUL=LMUL.
+  case RISCV::VNSRL_WX:
+  case RISCV::VNSRL_WI:
+  case RISCV::VNSRL_WV:
+  case RISCV::VNSRA_WI:
+  case RISCV::VNSRA_WV:
+  case RISCV::VNSRA_WX:
+  // 12.5. Vector Narrowing Fixed-Point Clip Instructions
+  // Destination and Op1 EEW=SEW and EMUL=LMUL. Op2 EEW=2*SEW and EMUL=2*LMUL
+  case RISCV::VNCLIPU_WI:
+  case RISCV::VNCLIPU_WV:
+  case RISCV::VNCLIPU_WX:
+  case RISCV::VNCLIP_WI:
+  case RISCV::VNCLIP_WV:
+  case RISCV::VNCLIP_WX: {
+    bool IsOp1 = HasPassthru ? MO.getOperandNo() == 2 : MO.getOperandNo() == 1;
+    bool TwoTimes = IsOp1;
+    unsigned Log2EEW = TwoTimes ? MILog2SEW + 1 : MILog2SEW;
+    RISCVII::VLMUL EMUL =
+        TwoTimes ? RISCVVType::twoTimesVLMUL(MIVLMul) : MIVLMul;
+    return OperandInfo(EMUL, Log2EEW);
+  }
+
+  default:
+    return {};
+  }
+}
+
+/// Return true if this optimization should consider MI for VL reduction. This
+/// white-list approach simplifies this optimization for instructions that may
+/// have more complex semantics with relation to how it uses VL.
+static bool isSupportedInstr(const MachineInstr &MI) {
+  const RISCVVPseudosTable::PseudoInfo *RVV =
+      RISCVVPseudosTable::getPseudoInfo(MI.getOpcode());
+
+  if (!RVV)
+    return false;
+
+  switch (RVV->BaseInstr) {
+  // 11.1. Vector Single-Width Integer Add and Subtract
+  case RISCV::VADD_VI:
+  case RISCV::VADD_VV:
+  case RISCV::VADD_VX:
+  case RISCV::VSUB_VV:
+  case RISCV::VSUB_VX:
+  case RISCV::VRSUB_VI:
+  case RISCV::VRSUB_VX:
+  // 11.2. Vector Widening Integer Add/Subtract
+  case RISCV::VWADDU_VV:
+  case RISCV::VWADDU_VX:
+  case RISCV::VWSUBU_VV:
+  case RISCV::VWSUBU_VX:
+  case RISCV::VWADD_VV:
+  case RISCV::VWADD_VX:
+  case RISCV::VWSUB_VV:
+  case RISCV::VWSUB_VX:
+  case RISCV::VWADDU_WV:
+  case RISCV::VWADDU_WX:
+  case RISCV::VWSUBU_WV:
+  case RISCV::VWSUBU_WX:
+  case RISCV::VWADD_WV:
+  case RISCV::VWADD_WX:
+  case RISCV::VWSUB_WV:
+  case RISCV::VWSUB_WX:
+  // 11.3. Vector Integer Extension
+  case RISCV::VZEXT_VF2:
+  case RISCV::VSEXT_VF2:
+  case RISCV::VZEXT_VF4:
+  case RISCV::VSEXT_VF4:
+  case RISCV::VZEXT_VF8:
+  case RISCV::VSEXT_VF8:
+  // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
+  // FIXME: Add support for 11.4 instructions
+  // 11.5. Vector Bitwise Logical Instructions
+  // FIXME: Add support for 11.5 instructions
+  // 11.6. Vector Single-Width Shift Instructions
+  // FIXME: Add support for 11.6 instructions
+  case RISCV::VSLL_VI:
+  // 11.7. Vector Narrowing Integer Right Shift Instructions
+  // FIXME: Add support for 11.7 instructions
+  case RISCV::VNSRL_WI:
+  // 11.8 Vector Integer Compare Instructions
+  // FIXME: Add support for 11.8 instructions
+  // 11.9. Vector Integer Min/Max Instructions
+  case RISCV::VMINU_VV:
+  case RISCV::VMINU_VX:
+  case RISCV::VMIN_VV:
+  case RISCV::VMIN_VX:
+  case RISCV::VMAXU_VV:
+  case RISCV::VMAXU_VX:
+  case RISCV::VMAX_VV:
+  case RISCV::VMAX_VX:
+  // 11.10. Vector Single-Width Integer Multiply Instructions
+  case RISCV::VMUL_VV:
+  case RISCV::VMUL_VX:
+  case RISCV::VMULH_VV:
+  case RISCV::VMULH_VX:
+  case RISCV::VMULHU_VV:
+  case RISCV::VMULHU_VX:
+  case RISCV::VMULHSU_VV:
+  case RISCV::VMULHSU_VX:
+  // 11.11. Vector Integer Divide Instructions
+  case RISCV::VDIVU_VV:
+  case RISCV::VDIVU_VX:
+  case RISCV::VDIV_VV:
+  case RISCV::VDIV_VX:
+  case RISCV::VREMU_VV:
+  case RISCV::VREMU_VX:
+  case RISCV::VREM_VV:
+  case RISCV::VREM_VX:
+  // 11.12. Vector Widening Integer Multiply Instructions
+  // FIXME: Add support for 11.12 instructions
+  // 11.13. Vector Single-Width Integer Multiply-Add Instructions
+  // FIXME: Add support for 11.13 instructions
+  // 11.14. Vector Widening Integer Multiply-Add Instructions
+  // FIXME: Add support for 11.14 instructions
+  case RISCV::VWMACC_VX:
+  case RISCV::VWMACCU_VX:
+  // 11.15. Vector Integer Merge Instructions
+  // FIXME: Add support for 11.15 instructions
+  // 11.16. Vector Integer Move Instructions
+  // FIXME: Add support for 11.16 instructions
+  case RISCV::VMV_V_I:
+  case RISCV::VMV_V_X:
+
+  // Vector Crypto
+  case RISCV::VWSLL_VI:
+    return true;
+  }
+
+  return false;
+}
+
+/// Return true if MO is a vector operand but is used as a scalar operand.
+static bool isVectorOpUsedAsScalarOp(MachineOperand &MO) {
+  MachineInstr *MI = MO.getParent();
+  const RISCVVPseudosTable::PseudoInfo *RVV =
+      RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
+
+  if (!RVV)
+    return false;
+
+  switch (RVV->BaseInstr) {
+  // Reductions only use vs1[0] of vs1
+  case RISCV::VREDAND_VS:
+  case RISCV::VREDMAX_VS:
+  case RISCV::VREDMAXU_VS:
+  case RISCV::VREDMIN_VS:
+  case RISCV::VREDMINU_VS:
+  case RISCV::VREDOR_VS:
+  case RISCV::VREDSUM_VS:
+  case RISCV::VREDXOR_VS:
+  case RISCV::VWREDSUM_VS:
+  case RISCV::VWREDSUMU_VS:
+  case RISCV::VFREDMAX_VS:
+  case RISCV::VFREDMIN_VS:
+  case RISCV::VFREDOSUM_VS:
+  case RISCV::VFREDUSUM_VS:
+  case RISCV::VFWREDOSUM_VS:
+  case RISCV::VFWREDUSUM_VS: {
+    bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(MI->getDesc());
+    return HasPassthru ? MO.getOperandNo() == 2 : MO.getOperandNo() == 3;
+  }
+  default:
+    return false;
+  }
+}
+
+/// Return true if MI may read elements past VL.
+static bool mayReadPastVL(const MachineInstr &MI) {
+  const RISCVVPseudosTable::PseudoInfo *RVV =
+      RISCVVPseudosTable::getPseudoInfo(MI.getOpcode());
+  if (!RVV)
+    return true;
+
+  switch (RVV->BaseInstr) {
+  // vslidedown instructions may read elements past VL. They are handled
+  // according to current tail policy.
+  case RISCV::VSLIDEDOWN_VI:
+  case RISCV::VSLIDEDOWN_VX:
+  case RISCV::VSLIDE1DOWN_VX:
+  case RISCV::VFSLIDE1DOWN_VF:
+
+  // vrgather instructions may read the source vector at any index < VLMAX,
+  // regardless of VL.
+  case RISCV::VRGATHER_VI:
+  case RISCV::VRGATHER_VV:
+  case RISCV::VRGATHER_VX:
+  case RISCV::VRGATHEREI16_VV:
+    return true;
+
+  default:
+    return false;
+  }
+}
+
+bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
+  const MCInstrDesc &Desc = MI.getDesc();
+  if (!RISCVII::hasVLOp(Desc.TSFlags) || !RISCVII::hasSEWOp(Desc.TSFlags))
+    return false;
+  if (MI.getNumDefs() != 1)
+    return false;
+
+  unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
+  const MachineOperand &VLOp = MI.getOperand(VLOpNum);
+  if (!VLOp.isImm() || VLOp.getImm() != RISCV::VLMaxSentinel)
+    return false;
+
+  // Some instructions that produce vectors have semantics that make it more
+  // difficult to determine whether the VL can be reduced. For example, some
+  // instructions, such as reductions, may write lanes past VL to a scalar
+  // register. Other instructions, such as some loads or stores, may write
+  // lower lanes using data from higher lanes. There may be other complex
+  // semantics not mentioned here that make it hard to determine whether
+  // the VL can be optimized. As a result, a white-list of supported
+  // instructions is used. Over time, more instructions cam be supported
+  // upon careful examination of their semantics under the logic in this
+  // optimization.
+  // TODO: Use a better approach than a white-list, such as adding
+  // properties to instructions using something like TSFlags.
+  if (!isSupportedInstr(MI)) {
+    LLVM_DEBUG(dbgs() << "Not a candidate due to unsupported instruction\n");
+    return false;
+  }
+
+  LLVM_DEBUG(dbgs() << "Found a candidate for VL reduction: " << MI << "\n");
+  return true;
+}
+
+bool RISCVVLOptimizer::checkUsers(std::optional<Register> &CommonVL,
+                                  MachineInstr &MI) {
+  // FIXME: Avoid visiting each user for each time we visit something on the
+  // worklist, combined with an extra visit from the outer loop. Restructure
+  // along lines of an instcombine style worklist which integrates the outer
+  // pass.
+  bool CanReduceVL = true;
+  for (auto &UserOp : MRI->use_operands(MI.getOperand(0).getReg())) {
+    const MachineInstr &UserMI = *UserOp.getParent();
+    LLVM_DEBUG(dbgs() << "  Checking user: " << UserMI << "\n");
+
+    // Instructions like reductions may use a vector register as a scalar
+    // register. In this case, we should treat it like a scalar register which
+    // does not impact the decision on whether to optimize VL.
+    if (isVectorOpUsedAsScalarOp(UserOp)) {
+      [[maybe_unused]] Register R = UserOp.getReg();
+      [[maybe_unused]] const TargetRegisterClass *RC = MRI->getRegClass(R);
+      assert(RISCV::VRRegClass.hasSubClassEq(RC) &&
+             "Expect LMUL 1 register class for vector as scalar operands!");
+      LLVM_DEBUG(dbgs() << "    Use this operand as a scalar operand\n");
+      continue;
+    }
+
+    if (mayReadPastVL(UserMI)) {
+      LLVM_DEBUG(dbgs() << "    Abort because used by unsafe instruction\n");
+      CanReduceVL = false;
+      break;
+    }
+
+    // Tied operands might pass through.
+    if (UserOp.isTied()) {
+      LLVM_DEBUG(dbgs() << "    Abort because user used as tied operand\n");
+      CanReduceVL = false;
+      break;
+    }
+
+    const MCInstrDesc &Desc = UserMI.getDesc();
+    if (!RISCVII::hasVLOp(Desc.TSFlags) || !RISCVII::hasSEWOp(Desc.TSFlags)) {
+      LLVM_DEBUG(dbgs() << "    Abort due to lack of VL or SEW, assume that"
+                           " use VLMAX\n");
+      CanReduceVL = false;
+      break;
+    }
+
+    unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
+    const MachineOperand &VLOp = UserMI.getOperand(VLOpNum);
+    // Looking for a register VL that isn't X0.
+    if (!VLOp.isReg() || VLOp.getReg() == RISCV::X0) {
+      LLVM_DEBUG(dbgs() << "    Abort due to user uses X0 as VL.\n");
+      CanReduceVL = false;
+      break;
+    }
+
+    if (!CommonVL) {
+      CommonVL = VLOp.getReg();
+    } else if (*CommonVL != VLOp.getReg()) {
+      LLVM_DEBUG(dbgs() << "    Abort because users have different VL\n");
+      CanReduceVL = false;
+      break;
+    }
+
+    // The SEW and LMUL of destination and source registers need to match.
+
+    // We know that MI DEF is a vector register, because that was the guard
+    // to call this function.
+    assert(isVectorRegClass(UserMI.getOperand(0).getReg(), MRI) &&
+           "Expected DEF and USE to be vector registers");
+
+    OperandInfo ConsumerInfo = getOperandInfo(UserMI, UserOp, MRI);
+    OperandInfo ProducerInfo = getOperandInfo(MI, MI.getOperand(0), MRI);
+    if (ConsumerInfo.isUnknown() || ProducerInfo.isUnknown() ||
+        !OperandInfo::EMULAndEEWAreEqual(ConsumerInfo, ProducerInfo)) {
+      LLVM_DEBUG(dbgs() << "    Abort due to incompatible or unknown "
+                           "information for EMUL or EEW.\n");
+      LLVM_DEBUG(dbgs() << "      ConsumerInfo is: " << ConsumerInfo << "\n");
+      LLVM_DEBUG(dbgs() << "      ProducerInfo is: " << ProducerInfo << "\n");
+      CanReduceVL = false;
+      break;
+    }
+  }
+  return CanReduceVL;
+}
+
+bool RISCVVLOptimizer::tryReduceVL(MachineInstr &OrigMI) {
+  SetVector<MachineInstr *> Worklist;
+  Worklist.insert(&OrigMI);
+
+  bool MadeChange = false;
+  while (!Worklist.empty()) {
+    MachineInstr &MI = *Worklist.pop_back_val();
+    LLVM_DEBUG(dbgs() << "Trying to reduce VL for " << MI << "\n");
+
+    std::optional<Register> CommonVL;
+    bool CanReduceVL = true;
+    if (isVectorRegClass(MI.getOperand(0).getReg(), MRI))
+      CanReduceVL = checkUsers(CommonVL, MI);
+
+    if (!CanReduceVL || !CommonVL)
+      continue;
+
+    if (!CommonVL->isVirtual()) {
+      LLVM_DEBUG(
+          dbgs() << "    Abort due to new VL is not virtual register.\n");
+      continue;
+    }
+
+    const MachineInstr *VLMI = MRI->getVRegDef(*CommonVL);
+    if (!MDT->dominates(VLMI, &MI))
+      continue;
+
+    // All our checks passed. We can reduce VL.
+    LLVM_DEBUG(dbgs() << "    Reducing VL for: " << MI << "\n");
+    unsigned VLOpNum = RISCVII::getVLOpNum(MI.getDesc());
+    MachineOperand &VLOp = MI.getOperand(VLOpNum);
+    VLOp.ChangeToRegister(*CommonVL, false);
+    MadeChange = true;
+
+    // Now add all inputs to this instruction to the worklist.
+    for (auto &Op : MI.operands()) {
+      if (!Op.isReg() || !Op.isUse() || !Op.getReg().isVirtual())
+        continue;
+
+      if (!isVectorRegClass(Op.getReg(), MRI))
+        continue;
+
+      MachineInstr *DefMI = MRI->getVRegDef(Op.getReg());
+
+      if (!isCandidate(*DefMI))
+        continue;
+
+      Worklist.insert(DefMI);
+    }
+  }
+
+  return MadeChange;
+}
+
+bool RISCVVLOptimizer::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  MRI = &MF.getRegInfo();
+  MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+
+  const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
+  if (!ST.hasVInstructions())
+    return false;
+
+  bool MadeChange = false;
+  for (MachineBasicBlock &MBB : MF) {
+    // Visit instructions in reverse order.
+    for (auto &MI : make_range(MBB.rbegin(), MBB.rend())) {
+      if (!isCandidate(MI))
+        continue;
+
+      MadeChange |= tryReduceVL(MI);
+    }
+  }
+
+  return MadeChange;
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 1872b238d1077a46a4bb13df18e10711dca7b25a..ecf9b6ddae1fc3d6ede782fdda3cc4a5b3a70be2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -353,11 +353,11 @@ static void lowerExpectAssume(IntrinsicInst *II) {
   // We need to lower this into a builtin and then the builtin into a SPIR-V
   // instruction.
   if (II->getIntrinsicID() == Intrinsic::assume) {
-    Function *F = Intrinsic::getDeclaration(
+    Function *F = Intrinsic::getOrInsertDeclaration(
         II->getModule(), Intrinsic::SPVIntrinsics::spv_assume);
     II->setCalledFunction(F);
   } else if (II->getIntrinsicID() == Intrinsic::expect) {
-    Function *F = Intrinsic::getDeclaration(
+    Function *F = Intrinsic::getOrInsertDeclaration(
         II->getModule(), Intrinsic::SPVIntrinsics::spv_expect,
         {II->getOperand(0)->getType()});
     II->setCalledFunction(F);
@@ -372,12 +372,12 @@ static bool toSpvOverloadedIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID,
                                      ArrayRef<unsigned> OpNos) {
   Function *F = nullptr;
   if (OpNos.empty()) {
-    F = Intrinsic::getDeclaration(II->getModule(), NewID);
+    F = Intrinsic::getOrInsertDeclaration(II->getModule(), NewID);
   } else {
     SmallVector<Type *, 4> Tys;
     for (unsigned OpNo : OpNos)
       Tys.push_back(II->getOperand(OpNo)->getType());
-    F = Intrinsic::getDeclaration(II->getModule(), NewID, Tys);
+    F = Intrinsic::getOrInsertDeclaration(II->getModule(), NewID, Tys);
   }
   II->setCalledFunction(F);
   return true;
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index abf414fd1ffb71ea2a3649d8624c985090e0eb11..c7626434efac674e2c0aa5051be40899bd869489 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -8340,11 +8340,11 @@ static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
     if (MI->getOperand(4).getImm() == (CCValid ^ CCMask))
       std::swap(TrueReg, FalseReg);
 
-    if (RegRewriteTable.contains(TrueReg))
-      TrueReg = RegRewriteTable[TrueReg].first;
+    if (auto It = RegRewriteTable.find(TrueReg); It != RegRewriteTable.end())
+      TrueReg = It->second.first;
 
-    if (RegRewriteTable.contains(FalseReg))
-      FalseReg = RegRewriteTable[FalseReg].second;
+    if (auto It = RegRewriteTable.find(FalseReg); It != RegRewriteTable.end())
+      FalseReg = It->second.second;
 
     DebugLoc DL = MI->getDebugLoc();
     BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg)
diff --git a/llvm/lib/Target/SystemZ/SystemZTDC.cpp b/llvm/lib/Target/SystemZ/SystemZTDC.cpp
index f62afb8ddfcfae4754390403e52991ae2f682400..345327e880ecd5d34e542bd7793bd82a08b7a2a5 100644
--- a/llvm/lib/Target/SystemZ/SystemZTDC.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTDC.cpp
@@ -366,8 +366,8 @@ bool SystemZTDCPass::runOnFunction(Function &F) {
       if (!Worthy)
         continue;
       // Call the intrinsic, compare result with 0.
-      Function *TDCFunc =
-          Intrinsic::getDeclaration(&M, Intrinsic::s390_tdc, V->getType());
+      Function *TDCFunc = Intrinsic::getOrInsertDeclaration(
+          &M, Intrinsic::s390_tdc, V->getType());
       IRBuilder<> IRB(I);
       Value *MaskVal = ConstantInt::get(Type::getInt64Ty(Ctx), Mask);
       Instruction *TDC = IRB.CreateCall(TDCFunc, {V, MaskVal});
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index c040e560be605f2c07f2c0678ab5dc7bd948af30..493e391964ac14b5c23380b502b3651d91251bb7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -575,8 +575,9 @@ Function *WebAssemblyLowerEmscriptenEHSjLj::getInvokeWrapper(CallBase *CI) {
   FunctionType *CalleeFTy = CI->getFunctionType();
 
   std::string Sig = getSignature(CalleeFTy);
-  if (InvokeWrappers.contains(Sig))
-    return InvokeWrappers[Sig];
+  auto It = InvokeWrappers.find(Sig);
+  if (It != InvokeWrappers.end())
+    return It->second;
 
   // Put the pointer to the callee as first argument
   ArgTys.push_back(PointerType::getUnqual(CalleeFTy));
@@ -1016,7 +1017,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
 
       // wasm.catch() will be lowered down to wasm 'catch' instruction in
       // instruction selection.
-      CatchF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_catch);
+      CatchF = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::wasm_catch);
       // Type for struct __WasmLongjmpArgs
       LongjmpArgsTy = StructType::get(Int8PtrTy, // env
                                       Int32Ty    // val
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerRefTypesIntPtrConv.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerRefTypesIntPtrConv.cpp
index 2594430d1d8f3a9d6d01dbfc421887b038a72c95..c61aa5eff4a7082dc24697b158c96bf047b2a5f9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerRefTypesIntPtrConv.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerRefTypesIntPtrConv.cpp
@@ -72,7 +72,7 @@ bool WebAssemblyLowerRefTypesIntPtrConv::runOnFunction(Function &F) {
     I->replaceAllUsesWith(U);
 
     Function *TrapIntrin =
-        Intrinsic::getDeclaration(F.getParent(), Intrinsic::debugtrap);
+        Intrinsic::getOrInsertDeclaration(F.getParent(), Intrinsic::debugtrap);
     CallInst::Create(TrapIntrin, {}, "", I->getIterator());
 
     worklist.insert(&*I);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 73765f8fa0092cbc82949713c084a17623b2f48d..3fe6ccf1c608e1e235519fa32c3b2501bee56a17 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -133,7 +133,10 @@ WebAssemblyTargetMachine::WebAssemblyTargetMachine(
   // WebAssembly type-checks instructions, but a noreturn function with a return
   // type that doesn't match the context will cause a check failure. So we lower
   // LLVM 'unreachable' to ISD::TRAP and then lower that to WebAssembly's
-  // 'unreachable' instructions which is meant for that case.
+  // 'unreachable' instructions which is meant for that case. Formerly, we also
+  // needed to add checks to SP failure emission in the instruction selection
+  // backends, but this has since been tied to TrapUnreachable and is no longer
+  // necessary.
   this->Options.TrapUnreachable = true;
   this->Options.NoTrapAfterNoreturn = false;
 
@@ -202,8 +205,7 @@ public:
   bool runOnModule(Module &M) override {
     FeatureBitset Features = coalesceFeatures(M);
 
-    std::string FeatureStr =
-        getFeatureString(Features, WasmTM->getTargetFeatureString());
+    std::string FeatureStr = getFeatureString(Features);
     WasmTM->setTargetFeatureString(FeatureStr);
     for (auto &F : M)
       replaceFeatures(F, FeatureStr);
@@ -241,17 +243,14 @@ private:
     return Features;
   }
 
-  static std::string getFeatureString(const FeatureBitset &Features,
-                                      StringRef TargetFS) {
+  static std::string getFeatureString(const FeatureBitset &Features) {
     std::string Ret;
     for (const SubtargetFeatureKV &KV : WebAssemblyFeatureKV) {
       if (Features[KV.Value])
         Ret += (StringRef("+") + KV.Key + ",").str();
+      else
+        Ret += (StringRef("-") + KV.Key + ",").str();
     }
-    SubtargetFeatures TF{TargetFS};
-    for (std::string const &F : TF.getFeatures())
-      if (!SubtargetFeatures::isEnabled(F))
-        Ret += F + ",";
     return Ret;
   }
 
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7a6d20c6a121b6a798f843e4ad239dfbedcf2fc2..e70fbd8b23f2ab74f04021c453ebaea7ec4e1a55 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -30143,6 +30143,39 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
     return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});
   }
 
+  // If we're shifting (per-lane) uniform vXi8 constants, we can use PSHUFB to
+  // look up the pre-computed shift values.
+  if ((VT == MVT::v16i8 && Subtarget.hasSSSE3()) ||
+      (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
+      (VT == MVT::v64i8 && Subtarget.hasBWI())) {
+    unsigned NumElts = VT.getVectorNumElements();
+    unsigned NumLanes = VT.getSizeInBits() / 128u;
+    unsigned NumEltsPerLane = NumElts / NumLanes;
+    SmallVector<APInt, 16> LUT;
+    for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
+      unsigned LoElt = Lane * NumEltsPerLane;
+      APInt EltMask = APInt::getBitsSet(NumElts, LoElt, LoElt + NumEltsPerLane);
+      KnownBits KnownLane = DAG.computeKnownBits(R, EltMask);
+      if (!KnownLane.isConstant())
+        break;
+      const APInt &LaneSplat = KnownLane.getConstant();
+      for (unsigned I = 0; I != 8; ++I) {
+        if (Opc == ISD::SHL)
+          LUT.push_back(LaneSplat.shl(I));
+        else if (Opc == ISD::SRL)
+          LUT.push_back(LaneSplat.lshr(I));
+        else if (Opc == ISD::SRA)
+          LUT.push_back(LaneSplat.ashr(I));
+      }
+      LUT.append(8, APInt::getZero(8));
+    }
+    if (LUT.size() == NumElts) {
+      APInt Undefs = APInt::getSplat(NumElts, APInt(16, 0xFF00));
+      SDValue Mask = getConstVector(LUT, Undefs, VT, DAG, dl);
+      return DAG.getNode(X86ISD::PSHUFB, dl, VT, Mask, Amt);
+    }
+  }
+
   // It's worth extending once and using the vXi16/vXi32 shifts for smaller
   // types, but without AVX512 the extra overheads to get from vXi8 to vXi32
   // make the existing SSE solution better.
@@ -31163,12 +31196,14 @@ void X86TargetLowering::emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const {
   if (BitTested.second == ConstantBit || BitTested.second == NotConstantBit) {
     auto *C = cast<ConstantInt>(I->getOperand(I->getOperand(0) == AI ? 1 : 0));
 
-    BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_C, AI->getType());
+    BitTest = Intrinsic::getOrInsertDeclaration(AI->getModule(), IID_C,
+                                                AI->getType());
 
     unsigned Imm = llvm::countr_zero(C->getZExtValue());
     Result = Builder.CreateCall(BitTest, {Addr, Builder.getInt8(Imm)});
   } else {
-    BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_I, AI->getType());
+    BitTest = Intrinsic::getOrInsertDeclaration(AI->getModule(), IID_I,
+                                                AI->getType());
 
     assert(BitTested.second == ShiftBit || BitTested.second == NotShiftBit);
 
@@ -31328,7 +31363,7 @@ void X86TargetLowering::emitCmpArithAtomicRMWIntrinsic(
     break;
   }
   Function *CmpArith =
-      Intrinsic::getDeclaration(AI->getModule(), IID, AI->getType());
+      Intrinsic::getOrInsertDeclaration(AI->getModule(), IID, AI->getType());
   Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
                                           PointerType::getUnqual(Ctx));
   Value *Call = Builder.CreateCall(
@@ -31444,7 +31479,7 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
     return nullptr;
 
   Function *MFence =
-      llvm::Intrinsic::getDeclaration(M, Intrinsic::x86_sse2_mfence);
+      llvm::Intrinsic::getOrInsertDeclaration(M, Intrinsic::x86_sse2_mfence);
   Builder.CreateCall(MFence, {});
 
   // Finally we can emit the atomic load.
@@ -43804,7 +43839,8 @@ bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
 
   switch (Op.getOpcode()) {
   case X86ISD::PSHUFD:
-  case X86ISD::VPERMILPI: {
+  case X86ISD::VPERMILPI:
+  case X86ISD::VPERMV3: {
     SmallVector<int, 8> Mask;
     SmallVector<SDValue, 2> Ops;
     if (getTargetShuffleMask(Op, true, Ops, Mask)) {
@@ -43848,6 +43884,7 @@ bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
     return false;
   case X86ISD::PSHUFD:
   case X86ISD::VPERMILPI:
+  case X86ISD::VPERMV3:
   case X86ISD::UNPCKH:
   case X86ISD::UNPCKL:
     return false;
@@ -50000,6 +50037,28 @@ static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {
          (VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit()));
 }
 
+/// Folds (and X, (or Y, ~Z)) --> (and X, ~(and ~Y, Z))
+/// This undoes the inverse fold performed in InstCombine
+static SDValue combineAndNotOrIntoAndNotAnd(SDNode *N, SelectionDAG &DAG) {
+
+  using namespace llvm::SDPatternMatch;
+  MVT VT = N->getSimpleValueType(0);
+  SDLoc DL(N);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  if (!TLI.hasAndNot(SDValue(N, 0)))
+    return SDValue();
+
+  SDValue X, Y, Z;
+  if (sd_match(
+          N, m_And(m_Value(X), m_OneUse(m_Or(m_Value(Y), m_Not(m_Value(Z)))))))
+    return DAG.getNode(
+        ISD::AND, DL, VT, X,
+        DAG.getNOT(DL, DAG.getNode(ISD::AND, DL, VT, DAG.getNOT(DL, Y, VT), Z),
+                   VT));
+
+  return SDValue();
+}
+
 // This function recognizes cases where X86 bzhi instruction can replace and
 // 'and-load' sequence.
 // In case of loading integer value from an array of constants which is defined
@@ -50491,6 +50550,9 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
     return R;
 
+  if (SDValue R = combineAndNotOrIntoAndNotAnd(N, DAG))
+    return R;
+
   // fold (and (mul x, c1), c2) -> (mul x, (and c1, c2))
   // iff c2 is all/no bits mask - i.e. a select-with-zero mask.
   // TODO: Handle PMULDQ/PMULUDQ/VPMADDWD/VPMADDUBSW?
@@ -53961,6 +54023,7 @@ static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
   MVT VT = N->getSimpleValueType(0);
   int NumElts = VT.getVectorNumElements();
   unsigned EltSizeInBits = VT.getScalarSizeInBits();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDLoc DL(N);
 
   // ANDNP(undef, x) -> 0
@@ -53984,6 +54047,18 @@ static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
   if (SDValue Not = IsNOT(N0, DAG))
     return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1);
 
+  // On AVX512 targets, attempt to reverse foldVSelectToSignBitSplatMask.
+  // to make use of predicated selects.
+  // ANDN(SEXT(SETCC()),X) -> SELECT(NOT(SETCC()),X,0)
+  if (DCI.isAfterLegalizeDAG() && N0.getOpcode() == ISD::SIGN_EXTEND) {
+    SDValue Src = N0.getOperand(0);
+    EVT SrcVT = Src.getValueType();
+    if (Src.getOpcode() == ISD::SETCC && SrcVT.getScalarType() == MVT::i1 &&
+        TLI.isTypeLegal(SrcVT) && N0.hasOneUse() && Src.hasOneUse())
+      return DAG.getSelect(DL, VT, DAG.getNOT(DL, Src, SrcVT), N1,
+                           getZeroVector(VT, Subtarget, DAG, DL));
+  }
+
   // Constant Folding
   APInt Undefs0, Undefs1;
   SmallVector<APInt> EltBits0, EltBits1;
@@ -54050,7 +54125,6 @@ static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
     std::tie(Bits0, Elts0) = GetDemandedMasks(N1);
     std::tie(Bits1, Elts1) = GetDemandedMasks(N0, true);
 
-    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     if (TLI.SimplifyDemandedVectorElts(N0, Elts0, DCI) ||
         TLI.SimplifyDemandedVectorElts(N1, Elts1, DCI) ||
         TLI.SimplifyDemandedBits(N0, Bits0, Elts0, DCI) ||
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 77139f38c977bbce5192d9c5fa36d514c95564fb..c4374984da4b9eaa514b0dfa936491be89d389ae 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -1876,7 +1876,8 @@ static Value *simplifyX86extrq(IntrinsicInst &II, Value *Op0,
     if (II.getIntrinsicID() == Intrinsic::x86_sse4a_extrq) {
       Value *Args[] = {Op0, CILength, CIIndex};
       Module *M = II.getModule();
-      Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi);
+      Function *F =
+          Intrinsic::getOrInsertDeclaration(M, Intrinsic::x86_sse4a_extrqi);
       return Builder.CreateCall(F, Args);
     }
   }
@@ -1975,7 +1976,8 @@ static Value *simplifyX86insertq(IntrinsicInst &II, Value *Op0, Value *Op1,
 
     Value *Args[] = {Op0, Op1, CILength, CIIndex};
     Module *M = II.getModule();
-    Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
+    Function *F =
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::x86_sse4a_insertqi);
     return Builder.CreateCall(F, Args);
   }
 
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index 613722b398f4469201332537c77d206c176dca7d..9cf700d6b65b0595df86293c8ab8942ba2fabff3 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -140,14 +140,16 @@ bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) {
       MachineInstr *NewMI =
           addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc)), TileSS)
               .addReg(SrcReg, getKillRegState(SrcMO.isKill()));
-      MachineOperand &MO = NewMI->getOperand(2);
-      MO.setReg(GR64Cand ? GR64Cand : X86::RAX);
-      MO.setIsKill(true);
+      MachineOperand *MO = &NewMI->getOperand(X86::AddrIndexReg);
+      MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
       // tileloadd (%sp, %idx), %tmm
       Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
 #undef GET_EGPR_IF_ENABLED
       NewMI = addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc), DstReg),
                                 TileSS);
+      MO = &NewMI->getOperand(1 + X86::AddrIndexReg);
+      MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
+      MO->setIsKill(true);
       if (!GR64Cand) {
         // restore %rax
         // mov (%sp) %rax
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
index 5bbfabcbd67bc60f2cbcc2c54e169cb2c25ce934..e88702caa9a52b0f67f0ffcf2ee6afedc3af018d 100644
--- a/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -278,7 +278,7 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
     IntrinsicNumElts = 16;
   }
 
-  Function *PSADBWFn = Intrinsic::getDeclaration(Op->getModule(), IID);
+  Function *PSADBWFn = Intrinsic::getOrInsertDeclaration(Op->getModule(), IID);
 
   if (NumElts < 16) {
     // Pad input with zeroes.
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index fc2a1e34b711ef5b4cadcc9a35e03f57134b944f..ceb87a62dba01140c9ff95cfec88ab18cc78363f 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -242,7 +242,10 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
           OL),
       TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) {
   // On PS4/PS5, the "return address" of a 'noreturn' call must still be within
-  // the calling function, and TrapUnreachable is an easy way to get that.
+  // the calling function. Note that this also includes __stack_chk_fail,
+  // so there was some target-specific logic in the instruction selectors
+  // to handle that. That code has since been generalized, so the only thing
+  // needed is to set TrapUnreachable here.
   if (TT.isPS() || TT.isOSBinFormatMachO()) {
     this->Options.TrapUnreachable = true;
     this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
diff --git a/llvm/lib/Target/X86/X86WinEHState.cpp b/llvm/lib/Target/X86/X86WinEHState.cpp
index 963d613ddbfe7d3fbe360f4f0a31c23df3dd13af..05fc6f13129f24e7a225c1c60419475e04cb3f7a 100644
--- a/llvm/lib/Target/X86/X86WinEHState.cpp
+++ b/llvm/lib/Target/X86/X86WinEHState.cpp
@@ -334,7 +334,7 @@ void WinEHStatePass::emitExceptionRegistrationRecord(Function *F) {
     if (UseStackGuard) {
       Value *Val = Builder.CreateLoad(Int32Ty, Cookie);
       Value *FrameAddr = Builder.CreateCall(
-          Intrinsic::getDeclaration(
+          Intrinsic::getOrInsertDeclaration(
               TheModule, Intrinsic::frameaddress,
               Builder.getPtrTy(
                   TheModule->getDataLayout().getAllocaAddrSpace())),
@@ -370,7 +370,7 @@ void WinEHStatePass::emitExceptionRegistrationRecord(Function *F) {
 
 Value *WinEHStatePass::emitEHLSDA(IRBuilder<> &Builder, Function *F) {
   return Builder.CreateCall(
-      Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_lsda), F);
+      Intrinsic::getOrInsertDeclaration(TheModule, Intrinsic::x86_seh_lsda), F);
 }
 
 /// Generate a thunk that puts the LSDA of ParentFunc in EAX and then calls
@@ -624,17 +624,17 @@ void WinEHStatePass::addStateStores(Function &F, WinEHFuncInfo &FuncInfo) {
   // that it can recover the original frame pointer.
   IRBuilder<> Builder(RegNode->getNextNode());
   Value *RegNodeI8 = Builder.CreateBitCast(RegNode, Builder.getPtrTy());
-  Builder.CreateCall(
-      Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_ehregnode),
-      {RegNodeI8});
+  Builder.CreateCall(Intrinsic::getOrInsertDeclaration(
+                         TheModule, Intrinsic::x86_seh_ehregnode),
+                     {RegNodeI8});
 
   if (EHGuardNode) {
     IRBuilder<> Builder(EHGuardNode->getNextNode());
     Value *EHGuardNodeI8 =
         Builder.CreateBitCast(EHGuardNode, Builder.getPtrTy());
-    Builder.CreateCall(
-        Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_ehguard),
-        {EHGuardNodeI8});
+    Builder.CreateCall(Intrinsic::getOrInsertDeclaration(
+                           TheModule, Intrinsic::x86_seh_ehguard),
+                       {EHGuardNodeI8});
   }
 
   // Calculate state numbers.
diff --git a/llvm/lib/Target/XCore/XCoreLowerThreadLocal.cpp b/llvm/lib/Target/XCore/XCoreLowerThreadLocal.cpp
index 95962d1a0a240fc79a96b5623891dc33d43cbf37..3604774ddf35bfadd402f862cc1d228c0b8e52c2 100644
--- a/llvm/lib/Target/XCore/XCoreLowerThreadLocal.cpp
+++ b/llvm/lib/Target/XCore/XCoreLowerThreadLocal.cpp
@@ -157,8 +157,8 @@ bool XCoreLowerThreadLocal::lowerGlobal(GlobalVariable *GV) {
   for (User *U : Users) {
     Instruction *Inst = cast<Instruction>(U);
     IRBuilder<> Builder(Inst);
-    Function *GetID = Intrinsic::getDeclaration(GV->getParent(),
-                                                Intrinsic::xcore_getid);
+    Function *GetID = Intrinsic::getOrInsertDeclaration(GV->getParent(),
+                                                        Intrinsic::xcore_getid);
     Value *ThreadID = Builder.CreateCall(GetID, {});
     Value *Addr = Builder.CreateInBoundsGEP(NewGV->getValueType(), NewGV,
                                             {Builder.getInt64(0), ThreadID});
diff --git a/llvm/lib/TargetParser/ARMTargetParser.cpp b/llvm/lib/TargetParser/ARMTargetParser.cpp
index f073bf66f20bac7e44a676b210a4a6dfa82b3702..9bcfa6ca62c97f66423fe31bae449a2b0ad721e3 100644
--- a/llvm/lib/TargetParser/ARMTargetParser.cpp
+++ b/llvm/lib/TargetParser/ARMTargetParser.cpp
@@ -556,7 +556,9 @@ StringRef ARM::computeDefaultTargetABI(const Triple &TT, StringRef CPU) {
   switch (TT.getEnvironment()) {
   case Triple::Android:
   case Triple::GNUEABI:
+  case Triple::GNUEABIT64:
   case Triple::GNUEABIHF:
+  case Triple::GNUEABIHFT64:
   case Triple::MuslEABI:
   case Triple::MuslEABIHF:
   case Triple::OpenHOS:
@@ -637,6 +639,7 @@ StringRef ARM::getARMCPUForArch(const llvm::Triple &Triple, StringRef MArch) {
     switch (Triple.getEnvironment()) {
     case llvm::Triple::EABIHF:
     case llvm::Triple::GNUEABIHF:
+    case llvm::Triple::GNUEABIHFT64:
     case llvm::Triple::MuslEABIHF:
       return "arm1176jzf-s";
     default:
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 6e1f4b6052bda8b9aa5193c4121a1658d75a04eb..9834aaacba18d0aad59cd090f2f560e091c44707 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -342,6 +342,12 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
     }
   }
 
+  if (Implementer == "0x63") { // Arm China.
+    return StringSwitch<const char *>(Part)
+        .Case("0x132", "star-mc1")
+        .Default("generic");
+  }
+
   if (Implementer == "0x6d") { // Microsoft Corporation.
     // The Microsoft Azure Cobalt 100 CPU is handled as a Neoverse N2.
     return StringSwitch<const char *>(Part)
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 229cf782e52edb37502a86345d5c1aa47463dad5..b271f29d265cfe0d8fe2ac03f289f7b659006869 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -312,10 +312,13 @@ StringRef Triple::getEnvironmentTypeName(EnvironmentType Kind) {
   case EABI: return "eabi";
   case EABIHF: return "eabihf";
   case GNU: return "gnu";
+  case GNUT64: return "gnut64";
   case GNUABI64: return "gnuabi64";
   case GNUABIN32: return "gnuabin32";
   case GNUEABI: return "gnueabi";
+  case GNUEABIT64: return "gnueabit64";
   case GNUEABIHF: return "gnueabihf";
+  case GNUEABIHFT64: return "gnueabihft64";
   case GNUF32: return "gnuf32";
   case GNUF64: return "gnuf64";
   case GNUSF: return "gnusf";
@@ -692,7 +695,9 @@ static Triple::EnvironmentType parseEnvironment(StringRef EnvironmentName) {
       .StartsWith("eabi", Triple::EABI)
       .StartsWith("gnuabin32", Triple::GNUABIN32)
       .StartsWith("gnuabi64", Triple::GNUABI64)
+      .StartsWith("gnueabihft64", Triple::GNUEABIHFT64)
       .StartsWith("gnueabihf", Triple::GNUEABIHF)
+      .StartsWith("gnueabit64", Triple::GNUEABIT64)
       .StartsWith("gnueabi", Triple::GNUEABI)
       .StartsWith("gnuf32", Triple::GNUF32)
       .StartsWith("gnuf64", Triple::GNUF64)
@@ -700,6 +705,7 @@ static Triple::EnvironmentType parseEnvironment(StringRef EnvironmentName) {
       .StartsWith("gnux32", Triple::GNUX32)
       .StartsWith("gnu_ilp32", Triple::GNUILP32)
       .StartsWith("code16", Triple::CODE16)
+      .StartsWith("gnut64", Triple::GNUT64)
       .StartsWith("gnu", Triple::GNU)
       .StartsWith("android", Triple::Android)
       .StartsWith("muslabin32", Triple::MuslABIN32)
@@ -919,7 +925,6 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) {
   case Triple::mips64:
   case Triple::mips64el:
   case Triple::mips:
-  case Triple::mipsel:
   case Triple::msp430:
   case Triple::nvptx64:
   case Triple::nvptx:
@@ -944,6 +949,11 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) {
   case Triple::xtensa:
     return Triple::ELF;
 
+  case Triple::mipsel:
+    if (T.isOSWindows())
+      return Triple::COFF;
+    return Triple::ELF;
+
   case Triple::ppc64:
   case Triple::ppc:
     if (T.isOSAIX())
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 9943c3cbb9fc7d3bc56f414e31c348a7a34b9e4b..898d55fab2b00d5ec1c9107bda2c3ab1132f55a9 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -172,7 +172,8 @@ static bool foldGuardedFunnelShift(Instruction &I, const DominatorTree &DT) {
   //   %cond = phi i32 [ %fsh, %FunnelBB ], [ %ShVal0, %GuardBB ]
   // -->
   // llvm.fshl.i32(i32 %ShVal0, i32 %ShVal1, i32 %ShAmt)
-  Function *F = Intrinsic::getDeclaration(Phi.getModule(), IID, Phi.getType());
+  Function *F =
+      Intrinsic::getOrInsertDeclaration(Phi.getModule(), IID, Phi.getType());
   Phi.replaceAllUsesWith(Builder.CreateCall(F, {ShVal0, ShVal1, ShAmt}));
   return true;
 }
@@ -331,7 +332,7 @@ static bool tryToRecognizePopCount(Instruction &I) {
                                 m_SpecificInt(Mask55)))) {
           LLVM_DEBUG(dbgs() << "Recognized popcount intrinsic\n");
           IRBuilder<> Builder(&I);
-          Function *Func = Intrinsic::getDeclaration(
+          Function *Func = Intrinsic::getOrInsertDeclaration(
               I.getModule(), Intrinsic::ctpop, I.getType());
           I.replaceAllUsesWith(Builder.CreateCall(Func, {Root}));
           ++NumPopCountRecognized;
@@ -398,8 +399,8 @@ static bool tryToFPToSat(Instruction &I, TargetTransformInfo &TTI) {
     return false;
 
   IRBuilder<> Builder(&I);
-  Function *Fn = Intrinsic::getDeclaration(I.getModule(), Intrinsic::fptosi_sat,
-                                           {SatTy, FpTy});
+  Function *Fn = Intrinsic::getOrInsertDeclaration(
+      I.getModule(), Intrinsic::fptosi_sat, {SatTy, FpTy});
   Value *Sat = Builder.CreateCall(Fn, In);
   I.replaceAllUsesWith(Builder.CreateSExt(Sat, IntTy));
   return true;
@@ -431,7 +432,7 @@ static bool foldSqrt(CallInst *Call, LibFunc Func, TargetTransformInfo &TTI,
     IRBuilderBase::FastMathFlagGuard Guard(Builder);
     Builder.setFastMathFlags(Call->getFastMathFlags());
 
-    Function *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::sqrt, Ty);
+    Function *Sqrt = Intrinsic::getOrInsertDeclaration(M, Intrinsic::sqrt, Ty);
     Value *NewSqrt = Builder.CreateCall(Sqrt, Arg, "sqrt");
     Call->replaceAllUsesWith(NewSqrt);
 
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index 1c45bcd7f6a837c597187bd88b6b8f355da21479..45b9767657c66a259e41924e9af67f3b2680bdcd 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -52,7 +52,8 @@ coro::LowererBase::LowererBase(Module &M)
 CallInst *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
                                            Instruction *InsertPt) {
   auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
-  auto *Fn = Intrinsic::getDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
+  auto *Fn =
+      Intrinsic::getOrInsertDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
 
   assert(Index >= CoroSubFnInst::IndexFirst &&
          Index < CoroSubFnInst::IndexLast &&
@@ -183,7 +184,7 @@ void coro::suppressCoroAllocs(LLVMContext &Context,
 static CoroSaveInst *createCoroSave(CoroBeginInst *CoroBegin,
                                     CoroSuspendInst *SuspendInst) {
   Module *M = SuspendInst->getModule();
-  auto *Fn = Intrinsic::getDeclaration(M, Intrinsic::coro_save);
+  auto *Fn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::coro_save);
   auto *SaveInst = cast<CoroSaveInst>(
       CallInst::Create(Fn, CoroBegin, "", SuspendInst->getIterator()));
   assert(!SuspendInst->getCoroSave());
diff --git a/llvm/lib/Transforms/IPO/CrossDSOCFI.cpp b/llvm/lib/Transforms/IPO/CrossDSOCFI.cpp
index 91d445dfc4c73439ce055aba87072f93562c814c..9e5d9ea31af6c4cf2cb6899ce1ba00fae85f2155 100644
--- a/llvm/lib/Transforms/IPO/CrossDSOCFI.cpp
+++ b/llvm/lib/Transforms/IPO/CrossDSOCFI.cpp
@@ -125,7 +125,8 @@ void CrossDSOCFI::buildCFICheck(Module &M) {
     ConstantInt *CaseTypeId = ConstantInt::get(Type::getInt64Ty(Ctx), TypeId);
     BasicBlock *TestBB = BasicBlock::Create(Ctx, "test", F);
     IRBuilder<> IRBTest(TestBB);
-    Function *BitsetTestFn = Intrinsic::getDeclaration(&M, Intrinsic::type_test);
+    Function *BitsetTestFn =
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::type_test);
 
     Value *Test = IRBTest.CreateCall(
         BitsetTestFn, {&Addr, MetadataAsValue::get(
diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 5e7d2c3c713d328ca4b2277519a0df6f06bff3b9..5ade0db343f278e43c07299d940b053a9081c34f 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -43,7 +43,9 @@
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/Utils/CallPromotionUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/Instrumentation.h"
 #include <deque>
 #include <sstream>
 #include <unordered_map>
@@ -449,9 +451,10 @@ public:
     }
   };
 
-  /// Helpers to remove callee edges that have allocation type None (due to not
+  /// Helpers to remove edges that have allocation type None (due to not
   /// carrying any context ids) after transformations.
   void removeNoneTypeCalleeEdges(ContextNode *Node);
+  void removeNoneTypeCallerEdges(ContextNode *Node);
   void
   recursivelyRemoveNoneTypeCalleeEdges(ContextNode *Node,
                                        DenseSet<const ContextNode *> &Visited);
@@ -483,6 +486,14 @@ protected:
   /// multiple different callee target functions.
   void handleCallsitesWithMultipleTargets();
 
+  // Try to partition calls on the given node (already placed into the AllCalls
+  // array) by callee function, creating new copies of Node as needed to hold
+  // calls with different callees, and moving the callee edges appropriately.
+  // Returns true if partitioning was successful.
+  bool partitionCallsByCallee(
+      ContextNode *Node, ArrayRef<CallInfo> AllCalls,
+      std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode);
+
   /// Save lists of calls with MemProf metadata in each function, for faster
   /// iteration.
   MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
@@ -563,6 +574,12 @@ private:
   calleesMatch(CallTy Call, EdgeIter &EI,
                MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap);
 
+  // Return the callee function of the given call, or nullptr if it can't be
+  // determined
+  const FuncTy *getCalleeFunc(CallTy Call) {
+    return static_cast<DerivedCCG *>(this)->getCalleeFunc(Call);
+  }
+
   /// Returns true if the given call targets the given function, or if we were
   /// able to identify the call chain through intermediate tail calls (in which
   /// case FoundCalleeChain will be populated).
@@ -670,6 +687,14 @@ private:
                                      bool NewClone = false,
                                      DenseSet<uint32_t> ContextIdsToMove = {});
 
+  /// Change the caller of the edge at the given callee edge iterator to be
+  /// NewCaller, performing the necessary context id and allocation type
+  /// updates. The iterator is updated as the edge is removed from the list of
+  /// callee edges in the original caller. This is similar to the above
+  /// moveEdgeToExistingCalleeClone, but a simplified version of it as we always
+  /// move the given edge and all of its context ids.
+  void moveCalleeEdgeToNewCaller(EdgeIter &CalleeEdgeI, ContextNode *NewCaller);
+
   /// Recursively perform cloning on the graph for the given Node and its
   /// callers, in order to uniquely identify the allocation behavior of an
   /// allocation given its context. The context ids of the allocation being
@@ -731,6 +756,7 @@ private:
                               Instruction *>;
 
   uint64_t getStackId(uint64_t IdOrIndex) const;
+  const Function *getCalleeFunc(Instruction *Call);
   bool calleeMatchesFunc(
       Instruction *Call, const Function *Func, const Function *CallerFunc,
       std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain);
@@ -808,6 +834,7 @@ private:
                               IndexCall>;
 
   uint64_t getStackId(uint64_t IdOrIndex) const;
+  const FunctionSummary *getCalleeFunc(IndexCall &Call);
   bool calleeMatchesFunc(
       IndexCall &Call, const FunctionSummary *Func,
       const FunctionSummary *CallerFunc,
@@ -1003,6 +1030,20 @@ void CallsiteContextGraph<
   }
 }
 
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+void CallsiteContextGraph<
+    DerivedCCG, FuncTy, CallTy>::removeNoneTypeCallerEdges(ContextNode *Node) {
+  for (auto EI = Node->CallerEdges.begin(); EI != Node->CallerEdges.end();) {
+    auto Edge = *EI;
+    if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
+      assert(Edge->ContextIds.empty());
+      Edge->Caller->eraseCalleeEdge(Edge.get());
+      EI = Node->CallerEdges.erase(EI);
+    } else
+      ++EI;
+  }
+}
+
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
 CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
@@ -1784,6 +1825,10 @@ static std::string getMemProfFuncName(Twine Base, unsigned CloneNo) {
   return (Base + MemProfCloneSuffix + Twine(CloneNo)).str();
 }
 
+static bool isMemProfClone(const Function &F) {
+  return F.getName().contains(MemProfCloneSuffix);
+}
+
 std::string ModuleCallsiteContextGraph::getLabel(const Function *Func,
                                                  const Instruction *Call,
                                                  unsigned CloneNo) const {
@@ -2025,6 +2070,21 @@ void CallsiteContextGraph<DerivedCCG, FuncTy,
     AllCalls.push_back(Node->Call);
     AllCalls.insert(AllCalls.end(), Node->MatchingCalls.begin(),
                     Node->MatchingCalls.end());
+
+    // First see if we can partition the calls by callee function, creating new
+    // nodes to host each set of calls calling the same callees. This is
+    // necessary for support indirect calls with ThinLTO, for which we
+    // synthesized CallsiteInfo records for each target. They will all have the
+    // same callsite stack ids and would be sharing a context node at this
+    // point. We need to perform separate cloning for each, which will be
+    // applied along with speculative devirtualization in the ThinLTO backends
+    // as needed. Note this does not currently support looking through tail
+    // calls, it is unclear if we need that for indirect call targets.
+    // First partition calls by callee func. Map indexed by func, value is
+    // struct with list of matching calls, assigned node.
+    if (partitionCallsByCallee(Node, AllCalls, NewCallToNode))
+      continue;
+
     auto It = AllCalls.begin();
     // Iterate through the calls until we find the first that matches.
     for (; It != AllCalls.end(); ++It) {
@@ -2098,6 +2158,130 @@ void CallsiteContextGraph<DerivedCCG, FuncTy,
     NonAllocationCallToContextNodeMap[Call] = Node;
 }
 
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::partitionCallsByCallee(
+    ContextNode *Node, ArrayRef<CallInfo> AllCalls,
+    std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode) {
+  // Struct to keep track of all the calls having the same callee function,
+  // and the node we eventually assign to them. Eventually we will record the
+  // context node assigned to this group of calls.
+  struct CallsWithSameCallee {
+    std::vector<CallInfo> Calls;
+    ContextNode *Node = nullptr;
+  };
+
+  // First partition calls by callee function. Build map from each function
+  // to the list of matching calls.
+  DenseMap<const FuncTy *, CallsWithSameCallee> CalleeFuncToCallInfo;
+  for (auto ThisCall : AllCalls) {
+    auto *F = getCalleeFunc(ThisCall.call());
+    if (F)
+      CalleeFuncToCallInfo[F].Calls.push_back(ThisCall);
+  }
+
+  // Next, walk through all callee edges. For each callee node, get its
+  // containing function and see if it was recorded in the above map (meaning we
+  // have at least one matching call). Build another map from each callee node
+  // with a matching call to the structure instance created above containing all
+  // the calls.
+  DenseMap<ContextNode *, CallsWithSameCallee *> CalleeNodeToCallInfo;
+  for (const auto &Edge : Node->CalleeEdges) {
+    if (!Edge->Callee->hasCall())
+      continue;
+    const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
+    if (CalleeFuncToCallInfo.contains(ProfiledCalleeFunc))
+      CalleeNodeToCallInfo[Edge->Callee] =
+          &CalleeFuncToCallInfo[ProfiledCalleeFunc];
+  }
+
+  // If there are entries in the second map, then there were no matching
+  // calls/callees, nothing to do here. Return so we can go to the handling that
+  // looks through tail calls.
+  if (CalleeNodeToCallInfo.empty())
+    return false;
+
+  // Walk through all callee edges again. Any and all callee edges that didn't
+  // match any calls (callee not in the CalleeNodeToCallInfo map) are moved to a
+  // new caller node (UnmatchedCalleesNode) which gets a null call so that it is
+  // ignored during cloning. If it is in the map, then we use the node recorded
+  // in that entry (creating it if needed), and move the callee edge to it.
+  // The first callee will use the original node instead of creating a new one.
+  // Note that any of the original calls on this node (in AllCalls) that didn't
+  // have a callee function automatically get dropped from the node as part of
+  // this process.
+  ContextNode *UnmatchedCalleesNode = nullptr;
+  // Track whether we already assigned original node to a callee.
+  bool UsedOrigNode = false;
+  assert(NodeToCallingFunc[Node]);
+  for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) {
+    auto Edge = *EI;
+    if (!Edge->Callee->hasCall()) {
+      ++EI;
+      continue;
+    }
+
+    // Will be updated below to point to whatever (caller) node this callee edge
+    // should be moved to.
+    ContextNode *CallerNodeToUse = nullptr;
+
+    // Handle the case where there were no matching calls first. Move this
+    // callee edge to the UnmatchedCalleesNode, creating it if needed.
+    if (!CalleeNodeToCallInfo.contains(Edge->Callee)) {
+      if (!UnmatchedCalleesNode)
+        UnmatchedCalleesNode =
+            createNewNode(/*IsAllocation=*/false, NodeToCallingFunc[Node]);
+      CallerNodeToUse = UnmatchedCalleesNode;
+    } else {
+      // Look up the information recorded for this callee node, and use the
+      // recorded caller node (creating it if needed).
+      auto *Info = CalleeNodeToCallInfo[Edge->Callee];
+      if (!Info->Node) {
+        // If we haven't assigned any callees to the original node use it.
+        if (!UsedOrigNode) {
+          Info->Node = Node;
+          // Clear the set of matching calls which will be updated below.
+          Node->MatchingCalls.clear();
+          UsedOrigNode = true;
+        } else
+          Info->Node =
+              createNewNode(/*IsAllocation=*/false, NodeToCallingFunc[Node]);
+        assert(!Info->Calls.empty());
+        // The first call becomes the primary call for this caller node, and the
+        // rest go in the matching calls list.
+        Info->Node->setCall(Info->Calls.front());
+        Info->Node->MatchingCalls.insert(Info->Node->MatchingCalls.end(),
+                                         Info->Calls.begin() + 1,
+                                         Info->Calls.end());
+        // Save the primary call to node correspondence so that we can update
+        // the NonAllocationCallToContextNodeMap, which is being iterated in the
+        // caller of this function.
+        NewCallToNode.push_back({Info->Node->Call, Info->Node});
+      }
+      CallerNodeToUse = Info->Node;
+    }
+
+    // Don't need to move edge if we are using the original node;
+    if (CallerNodeToUse == Node) {
+      ++EI;
+      continue;
+    }
+
+    moveCalleeEdgeToNewCaller(EI, CallerNodeToUse);
+  }
+  // Now that we are done moving edges, clean up any caller edges that ended
+  // up with no type or context ids. During moveCalleeEdgeToNewCaller all
+  // caller edges from Node are replicated onto the new callers, and it
+  // simplifies the handling to leave them until we have moved all
+  // edges/context ids.
+  for (auto &I : CalleeNodeToCallInfo)
+    removeNoneTypeCallerEdges(I.second->Node);
+  if (UnmatchedCalleesNode)
+    removeNoneTypeCallerEdges(UnmatchedCalleesNode);
+  removeNoneTypeCallerEdges(Node);
+
+  return true;
+}
+
 uint64_t ModuleCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
   // In the Module (IR) case this is already the Id.
   return IdOrIndex;
@@ -2276,6 +2460,17 @@ bool ModuleCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
   return FoundSingleCalleeChain;
 }
 
+const Function *ModuleCallsiteContextGraph::getCalleeFunc(Instruction *Call) {
+  auto *CB = dyn_cast<CallBase>(Call);
+  if (!CB->getCalledOperand() || CB->isIndirectCall())
+    return nullptr;
+  auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
+  auto *Alias = dyn_cast<GlobalAlias>(CalleeVal);
+  if (Alias)
+    return dyn_cast<Function>(Alias->getAliasee());
+  return dyn_cast<Function>(CalleeVal);
+}
+
 bool ModuleCallsiteContextGraph::calleeMatchesFunc(
     Instruction *Call, const Function *Func, const Function *CallerFunc,
     std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain) {
@@ -2411,6 +2606,15 @@ bool IndexCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
   return FoundSingleCalleeChain;
 }
 
+const FunctionSummary *
+IndexCallsiteContextGraph::getCalleeFunc(IndexCall &Call) {
+  ValueInfo Callee =
+      dyn_cast_if_present<CallsiteInfo *>(Call.getBase())->Callee;
+  if (Callee.getSummaryList().empty())
+    return nullptr;
+  return dyn_cast<FunctionSummary>(Callee.getSummaryList()[0]->getBaseObject());
+}
+
 bool IndexCallsiteContextGraph::calleeMatchesFunc(
     IndexCall &Call, const FunctionSummary *Func,
     const FunctionSummary *CallerFunc,
@@ -2871,6 +3075,81 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
   }
 }
 
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
+    moveCalleeEdgeToNewCaller(EdgeIter &CalleeEdgeI, ContextNode *NewCaller) {
+  auto Edge = *CalleeEdgeI;
+
+  ContextNode *OldCaller = Edge->Caller;
+
+  // We might already have an edge to the new caller. If one exists we will
+  // reuse it.
+  auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(Edge->Callee);
+
+  CalleeEdgeI = OldCaller->CalleeEdges.erase(CalleeEdgeI);
+  if (ExistingEdgeToNewCaller) {
+    // Since we already have an edge to NewCaller, simply move the ids
+    // onto it, and remove the existing Edge.
+    ExistingEdgeToNewCaller->getContextIds().insert(
+        Edge->getContextIds().begin(), Edge->getContextIds().end());
+    ExistingEdgeToNewCaller->AllocTypes |= Edge->AllocTypes;
+    Edge->ContextIds.clear();
+    Edge->AllocTypes = (uint8_t)AllocationType::None;
+    Edge->Callee->eraseCallerEdge(Edge.get());
+  } else {
+    // Otherwise just reconnect Edge to NewCaller.
+    Edge->Caller = NewCaller;
+    NewCaller->CalleeEdges.push_back(Edge);
+    // Don't need to update Edge's context ids since we are simply
+    // reconnecting it.
+  }
+  // In either case, need to update the alloc types on New Caller.
+  NewCaller->AllocTypes |= Edge->AllocTypes;
+
+  // Now walk the old caller node's caller edges and move Edge's context ids
+  // over to the corresponding edge into the node (which is created here if
+  // this is a newly created node). We can tell whether this is a newly created
+  // node by seeing if it has any caller edges yet.
+#ifndef NDEBUG
+  bool IsNewNode = NewCaller->CallerEdges.empty();
+#endif
+  for (auto &OldCallerEdge : OldCaller->CallerEdges) {
+    // The context ids moving to the new caller are the subset of this edge's
+    // context ids and the context ids on the callee edge being moved.
+    DenseSet<uint32_t> EdgeContextIdsToMove =
+        set_intersection(OldCallerEdge->getContextIds(), Edge->getContextIds());
+    set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
+    OldCallerEdge->AllocTypes =
+        computeAllocType(OldCallerEdge->getContextIds());
+    // In this function we expect that any pre-existing node already has edges
+    // from the same callers as the old node. That should be true in the current
+    // use case, where we will remove None-type edges after copying over all
+    // caller edges from the callee.
+    assert(IsNewNode || NewCaller->findEdgeFromCaller(OldCallerEdge->Caller));
+    auto NewEdge = std::make_shared<ContextEdge>(
+        NewCaller, OldCallerEdge->Caller,
+        computeAllocType(EdgeContextIdsToMove), EdgeContextIdsToMove);
+    NewCaller->CallerEdges.push_back(NewEdge);
+    NewEdge->Caller->CalleeEdges.push_back(NewEdge);
+  }
+  // Recompute the node alloc type now that its caller edges have been
+  // updated (since we will compute from those edges).
+  OldCaller->AllocTypes = OldCaller->computeAllocType();
+  // OldCaller alloc type should be None iff its context id set is now empty.
+  assert((OldCaller->AllocTypes == (uint8_t)AllocationType::None) ==
+         OldCaller->emptyContextIds());
+  if (VerifyCCG) {
+    checkNode<DerivedCCG, FuncTy, CallTy>(OldCaller, /*CheckEdges=*/false);
+    checkNode<DerivedCCG, FuncTy, CallTy>(NewCaller, /*CheckEdges=*/false);
+    for (const auto &OldCallerEdge : OldCaller->CallerEdges)
+      checkNode<DerivedCCG, FuncTy, CallTy>(OldCallerEdge->Caller,
+                                            /*CheckEdges=*/false);
+    for (const auto &NewCallerEdge : NewCaller->CallerEdges)
+      checkNode<DerivedCCG, FuncTy, CallTy>(NewCallerEdge->Caller,
+                                            /*CheckEdges=*/false);
+  }
+}
+
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     recursivelyRemoveNoneTypeCalleeEdges(
@@ -3789,14 +4068,22 @@ static ValueInfo findValueInfoForFunc(const Function &F, const Module &M,
   return TheFnVI;
 }
 
+bool MemProfContextDisambiguation::initializeIndirectCallPromotionInfo(
+    Module &M) {
+  ICallAnalysis = std::make_unique<ICallPromotionAnalysis>();
+  Symtab = std::make_unique<InstrProfSymtab>();
+  if (Error E = Symtab->create(M, /*InLTO=*/true)) {
+    std::string SymtabFailure = toString(std::move(E));
+    M.getContext().emitError("Failed to create symtab: " + SymtabFailure);
+    return false;
+  }
+  return true;
+}
+
 bool MemProfContextDisambiguation::applyImport(Module &M) {
   assert(ImportSummary);
   bool Changed = false;
 
-  auto IsMemProfClone = [](const Function &F) {
-    return F.getName().contains(MemProfCloneSuffix);
-  };
-
   // We also need to clone any aliases that reference cloned functions, because
   // the modified callsites may invoke via the alias. Keep track of the aliases
   // for each function.
@@ -3808,8 +4095,11 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
       FuncToAliasMap[F].insert(&A);
   }
 
+  if (!initializeIndirectCallPromotionInfo(M))
+    return false;
+
   for (auto &F : M) {
-    if (F.isDeclaration() || IsMemProfClone(F))
+    if (F.isDeclaration() || isMemProfClone(F))
       continue;
 
     OptimizationRemarkEmitter ORE(&F);
@@ -3844,9 +4134,7 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
       // Perform cloning if not yet done.
       CloneFuncIfNeeded(/*NumClones=*/StackNode.Clones.size());
 
-      // Should have skipped indirect calls via mayHaveMemprofSummary.
-      assert(CalledFunction);
-      assert(!IsMemProfClone(*CalledFunction));
+      assert(!isMemProfClone(*CalledFunction));
 
       // Update the calls per the summary info.
       // Save orig name since it gets updated in the first iteration
@@ -3933,6 +4221,9 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
       MapTailCallCalleeVIToCallsite.insert({Callsite.Callee, Callsite});
     }
 
+    // Keeps track of needed ICP for the function.
+    SmallVector<ICallAnalysisData> ICallAnalysisInfo;
+
     // Assume for now that the instructions are in the exact same order
     // as when the summary was created, but confirm this is correct by
     // matching the stack ids.
@@ -4075,24 +4366,49 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
                      << ore::NV("Attribute", AllocTypeString));
           }
         } else if (!CallsiteContext.empty()) {
-          // Consult the next callsite node.
-          assert(SI != FS->callsites().end());
-          auto &StackNode = *(SI++);
-
+          if (!CalledFunction) {
 #ifndef NDEBUG
-          // Sanity check that the stack ids match between the summary and
-          // instruction metadata.
-          auto StackIdIndexIter = StackNode.StackIdIndices.begin();
-          for (auto StackId : CallsiteContext) {
-            assert(StackIdIndexIter != StackNode.StackIdIndices.end());
-            assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
-                   StackId);
-            StackIdIndexIter++;
+            // We should have skipped inline assembly calls.
+            auto *CI = dyn_cast<CallInst>(CB);
+            assert(!CI || !CI->isInlineAsm());
+#endif
+            // We should have skipped direct calls via a Constant.
+            assert(CalledValue && !isa<Constant>(CalledValue));
+
+            // This is an indirect call, see if we have profile information and
+            // whether any clones were recorded for the profiled targets (that
+            // we synthesized CallsiteInfo summary records for when building the
+            // index).
+            auto NumClones =
+                recordICPInfo(CB, FS->callsites(), SI, ICallAnalysisInfo);
+
+            // Perform cloning if not yet done. This is done here in case
+            // we don't need to do ICP, but might need to clone this
+            // function as it is the target of other cloned calls.
+            if (NumClones)
+              CloneFuncIfNeeded(NumClones);
           }
+
+          else {
+            // Consult the next callsite node.
+            assert(SI != FS->callsites().end());
+            auto &StackNode = *(SI++);
+
+#ifndef NDEBUG
+            // Sanity check that the stack ids match between the summary and
+            // instruction metadata.
+            auto StackIdIndexIter = StackNode.StackIdIndices.begin();
+            for (auto StackId : CallsiteContext) {
+              assert(StackIdIndexIter != StackNode.StackIdIndices.end());
+              assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
+                     StackId);
+              StackIdIndexIter++;
+            }
 #endif
 
-          CloneCallsite(StackNode, CB, CalledFunction);
-        } else if (CB->isTailCall()) {
+            CloneCallsite(StackNode, CB, CalledFunction);
+          }
+        } else if (CB->isTailCall() && CalledFunction) {
           // Locate the synthesized callsite info for the callee VI, if any was
           // created, and use that for cloning.
           ValueInfo CalleeVI =
@@ -4108,11 +4424,175 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
         I.setMetadata(LLVMContext::MD_callsite, nullptr);
       }
     }
+
+    // Now do any promotion required for cloning.
+    performICP(M, FS->callsites(), VMaps, ICallAnalysisInfo, ORE);
   }
 
   return Changed;
 }
 
+unsigned MemProfContextDisambiguation::recordICPInfo(
+    CallBase *CB, ArrayRef<CallsiteInfo> AllCallsites,
+    ArrayRef<CallsiteInfo>::iterator &SI,
+    SmallVector<ICallAnalysisData> &ICallAnalysisInfo) {
+  // First see if we have profile information for this indirect call.
+  uint32_t NumCandidates;
+  uint64_t TotalCount;
+  auto CandidateProfileData =
+      ICallAnalysis->getPromotionCandidatesForInstruction(CB, TotalCount,
+                                                          NumCandidates);
+  if (CandidateProfileData.empty())
+    return 0;
+
+  // Iterate through all of the candidate profiled targets along with the
+  // CallsiteInfo summary records synthesized for them when building the index,
+  // and see if any are cloned and/or refer to clones.
+  bool ICPNeeded = false;
+  unsigned NumClones = 0;
+  size_t CallsiteInfoStartIndex = std::distance(AllCallsites.begin(), SI);
+  for (const auto &Candidate : CandidateProfileData) {
+#ifndef NDEBUG
+    auto CalleeValueInfo =
+#endif
+        ImportSummary->getValueInfo(Candidate.Value);
+    // We might not have a ValueInfo if this is a distributed
+    // ThinLTO backend and decided not to import that function.
+    assert(!CalleeValueInfo || SI->Callee == CalleeValueInfo);
+    assert(SI != AllCallsites.end());
+    auto &StackNode = *(SI++);
+    // See if any of the clones of the indirect callsite for this
+    // profiled target should call a cloned version of the profiled
+    // target. We only need to do the ICP here if so.
+    ICPNeeded |= llvm::any_of(StackNode.Clones,
+                              [](unsigned CloneNo) { return CloneNo != 0; });
+    // Every callsite in the same function should have been cloned the same
+    // number of times.
+    assert(!NumClones || NumClones == StackNode.Clones.size());
+    NumClones = StackNode.Clones.size();
+  }
+  if (!ICPNeeded)
+    return NumClones;
+  // Save information for ICP, which is performed later to avoid messing up the
+  // current function traversal.
+  ICallAnalysisInfo.push_back({CB, CandidateProfileData.vec(), NumCandidates,
+                               TotalCount, CallsiteInfoStartIndex});
+  return NumClones;
+}
+
+void MemProfContextDisambiguation::performICP(
+    Module &M, ArrayRef<CallsiteInfo> AllCallsites,
+    ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps,
+    ArrayRef<ICallAnalysisData> ICallAnalysisInfo,
+    OptimizationRemarkEmitter &ORE) {
+  // Now do any promotion required for cloning. Specifically, for each
+  // recorded ICP candidate (which was only recorded because one clone of that
+  // candidate should call a cloned target), we perform ICP (speculative
+  // devirtualization) for each clone of the callsite, and update its callee
+  // to the appropriate clone. Note that the ICP compares against the original
+  // version of the target, which is what is in the vtable.
+  for (auto &Info : ICallAnalysisInfo) {
+    auto *CB = Info.CB;
+    auto CallsiteIndex = Info.CallsiteInfoStartIndex;
+    auto TotalCount = Info.TotalCount;
+    unsigned NumPromoted = 0;
+    unsigned NumClones = 0;
+
+    for (auto &Candidate : Info.CandidateProfileData) {
+      auto &StackNode = AllCallsites[CallsiteIndex++];
+
+      // All calls in the same function must have the same number of clones.
+      assert(!NumClones || NumClones == StackNode.Clones.size());
+      NumClones = StackNode.Clones.size();
+
+      // See if the target is in the module. If it wasn't imported, it is
+      // possible that this profile could have been collected on a different
+      // target (or version of the code), and we need to be conservative
+      // (similar to what is done in the ICP pass).
+      Function *TargetFunction = Symtab->getFunction(Candidate.Value);
+      if (TargetFunction == nullptr || TargetFunction->isDeclaration()) {
+        ORE.emit([&]() {
+          return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", CB)
+                 << "Memprof cannot promote indirect call: target with md5sum "
+                 << ore::NV("target md5sum", Candidate.Value) << " not found";
+        });
+        // FIXME: See if we can use the new declaration importing support to
+        // at least get the declarations imported for this case. Hot indirect
+        // targets should have been imported normally, however.
+        continue;
+      }
+
+      // Check if legal to promote
+      const char *Reason = nullptr;
+      if (!isLegalToPromote(*CB, TargetFunction, &Reason)) {
+        ORE.emit([&]() {
+          return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToPromote", CB)
+                 << "Memprof cannot promote indirect call to "
+                 << ore::NV("TargetFunction", TargetFunction)
+                 << " with count of " << ore::NV("TotalCount", TotalCount)
+                 << ": " << Reason;
+        });
+        continue;
+      }
+
+      assert(!isMemProfClone(*TargetFunction));
+
+      // Handle each call clone, applying ICP so that each clone directly
+      // calls the specified callee clone, guarded by the appropriate ICP
+      // check.
+      CallBase *CBClone = CB;
+      for (unsigned J = 0; J < NumClones; J++) {
+        // Copy 0 is the original function.
+        if (J > 0)
+          CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
+        // We do the promotion using the original name, so that the comparison
+        // is against the name in the vtable. Then just below, change the new
+        // direct call to call the cloned function.
+        auto &DirectCall =
+            pgo::promoteIndirectCall(*CBClone, TargetFunction, Candidate.Count,
+                                     TotalCount, isSamplePGO, &ORE);
+        auto *TargetToUse = TargetFunction;
+        // Call original if this version calls the original version of its
+        // callee.
+        if (StackNode.Clones[J]) {
+          TargetToUse =
+              cast<Function>(M.getOrInsertFunction(
+                                  getMemProfFuncName(TargetFunction->getName(),
+                                                     StackNode.Clones[J]),
+                                  TargetFunction->getFunctionType())
+                                 .getCallee());
+        }
+        DirectCall.setCalledFunction(TargetToUse);
+        ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofCall", CBClone)
+                 << ore::NV("Call", CBClone) << " in clone "
+                 << ore::NV("Caller", CBClone->getFunction())
+                 << " promoted and assigned to call function clone "
+                 << ore::NV("Callee", TargetToUse));
+      }
+
+      // Update TotalCount (all clones should get same count above)
+      TotalCount -= Candidate.Count;
+      NumPromoted++;
+    }
+    // Adjust the MD.prof metadata for all clones, now that we have the new
+    // TotalCount and the number promoted.
+    CallBase *CBClone = CB;
+    for (unsigned J = 0; J < NumClones; J++) {
+      // Copy 0 is the original function.
+      if (J > 0)
+        CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
+      // First delete the old one.
+      CBClone->setMetadata(LLVMContext::MD_prof, nullptr);
+      // If all promoted, we don't need the MD.prof metadata.
+      // Otherwise we need update with the un-promoted records back.
+      if (TotalCount != 0)
+        annotateValueSite(
+            M, *CBClone, ArrayRef(Info.CandidateProfileData).slice(NumPromoted),
+            TotalCount, IPVK_IndirectCallTarget, Info.NumCandidates);
+    }
+  }
+}
+
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::process() {
   if (DumpCCG) {
@@ -4179,8 +4659,8 @@ bool MemProfContextDisambiguation::processModule(
 }
 
 MemProfContextDisambiguation::MemProfContextDisambiguation(
-    const ModuleSummaryIndex *Summary)
-    : ImportSummary(Summary) {
+    const ModuleSummaryIndex *Summary, bool isSamplePGO)
+    : ImportSummary(Summary), isSamplePGO(isSamplePGO) {
   if (ImportSummary) {
     // The MemProfImportSummary should only be used for testing ThinLTO
     // distributed backend handling via opt, in which case we don't have a
diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
index d84856f71c9de6474c6505af35c667d73a202698..543987d5981bab8ab21d2b948f231acdf3b5ffa3 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
@@ -401,7 +401,7 @@ void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) {
     assert(Builder.GetInsertPoint() != BB->end() &&
            "Cannot get the probing point");
     Function *ProbeFn =
-        llvm::Intrinsic::getDeclaration(M, Intrinsic::pseudoprobe);
+        llvm::Intrinsic::getOrInsertDeclaration(M, Intrinsic::pseudoprobe);
     Value *Args[] = {Builder.getInt64(Guid), Builder.getInt64(Index),
                      Builder.getInt32(0),
                      Builder.getInt64(PseudoProbeFullDistributionFactor)};
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 36a1841b363463897d2444be2a74104523e7a39f..59f986b4ca2664c427289343fbb0f1e7c26f49cc 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -856,7 +856,7 @@ void llvm::updatePublicTypeTestCalls(Module &M,
     return;
   if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) {
     Function *TypeTestFunc =
-        Intrinsic::getDeclaration(&M, Intrinsic::type_test);
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::type_test);
     for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) {
       auto *CI = cast<CallInst>(U.getUser());
       auto *NewCI = CallInst::Create(
@@ -1187,7 +1187,8 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
         Instruction *ThenTerm =
             SplitBlockAndInsertIfThen(Cond, &CB, /*Unreachable=*/false);
         Builder.SetInsertPoint(ThenTerm);
-        Function *TrapFn = Intrinsic::getDeclaration(&M, Intrinsic::debugtrap);
+        Function *TrapFn =
+            Intrinsic::getOrInsertDeclaration(&M, Intrinsic::debugtrap);
         auto *CallTrap = Builder.CreateCall(TrapFn);
         CallTrap->setDebugLoc(CB.getDebugLoc());
       }
@@ -1434,8 +1435,8 @@ void DevirtModule::tryICallBranchFunnel(
   }
 
   BasicBlock *BB = BasicBlock::Create(M.getContext(), "", JT, nullptr);
-  Function *Intr =
-      Intrinsic::getDeclaration(&M, llvm::Intrinsic::icall_branch_funnel, {});
+  Function *Intr = Intrinsic::getOrInsertDeclaration(
+      &M, llvm::Intrinsic::icall_branch_funnel, {});
 
   auto *CI = CallInst::Create(Intr, JTArgs, "", BB);
   CI->setTailCallKind(CallInst::TCK_MustTail);
@@ -2026,7 +2027,8 @@ void DevirtModule::scanTypeTestUsers(
 }
 
 void DevirtModule::scanTypeCheckedLoadUsers(Function *TypeCheckedLoadFunc) {
-  Function *TypeTestFunc = Intrinsic::getDeclaration(&M, Intrinsic::type_test);
+  Function *TypeTestFunc =
+      Intrinsic::getOrInsertDeclaration(&M, Intrinsic::type_test);
 
   for (Use &U : llvm::make_early_inc_range(TypeCheckedLoadFunc->uses())) {
     auto *CI = dyn_cast<CallInst>(U.getUser());
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index e5c3a20e1a6487f4d7eebb7ac1c339c107ac55ba..21588aca51275845839084a5a1270120593063fd 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1232,7 +1232,8 @@ static Instruction *foldToUnsignedSaturatedAdd(BinaryOperator &I) {
   assert(I.getOpcode() == Instruction::Add && "Expecting add instruction");
   Type *Ty = I.getType();
   auto getUAddSat = [&]() {
-    return Intrinsic::getDeclaration(I.getModule(), Intrinsic::uadd_sat, Ty);
+    return Intrinsic::getOrInsertDeclaration(I.getModule(), Intrinsic::uadd_sat,
+                                             Ty);
   };
 
   // add (umin X, ~Y), Y --> uaddsat X, Y
@@ -2127,7 +2128,7 @@ static Instruction *foldSubOfMinMax(BinaryOperator &I,
   if (match(Op0, m_c_Add(m_Specific(X), m_Specific(Y))) &&
       (Op0->hasOneUse() || Op1->hasOneUse())) {
     Intrinsic::ID InvID = getInverseMinMaxIntrinsic(MinMax->getIntrinsicID());
-    Function *F = Intrinsic::getDeclaration(I.getModule(), InvID, Ty);
+    Function *F = Intrinsic::getOrInsertDeclaration(I.getModule(), InvID, Ty);
     return CallInst::Create(F, {X, Y});
   }
 
@@ -2150,7 +2151,7 @@ static Instruction *foldSubOfMinMax(BinaryOperator &I,
   if (MinMax->isSigned() && match(Y, m_ZeroInt()) &&
       match(X, m_NSWSub(m_Specific(Op0), m_Value(Z)))) {
     Intrinsic::ID InvID = getInverseMinMaxIntrinsic(MinMax->getIntrinsicID());
-    Function *F = Intrinsic::getDeclaration(I.getModule(), InvID, Ty);
+    Function *F = Intrinsic::getOrInsertDeclaration(I.getModule(), InvID, Ty);
     return CallInst::Create(F, {Op0, Z});
   }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 688601a8ffa543d7f3df7a74a199243f8fff48ba..64bee4ab974ede5f02c424f30fc6b7afc2bef204 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -935,9 +935,11 @@ static Value *foldSignedTruncationCheck(ICmpInst *ICmp0, ICmpInst *ICmp1,
 
 /// Fold (icmp eq ctpop(X) 1) | (icmp eq X 0) into (icmp ult ctpop(X) 2) and
 /// fold (icmp ne ctpop(X) 1) & (icmp ne X 0) into (icmp ugt ctpop(X) 1).
-/// Also used for logical and/or, must be poison safe.
+/// Also used for logical and/or, must be poison safe if range attributes are
+/// dropped.
 static Value *foldIsPowerOf2OrZero(ICmpInst *Cmp0, ICmpInst *Cmp1, bool IsAnd,
-                                   InstCombiner::BuilderTy &Builder) {
+                                   InstCombiner::BuilderTy &Builder,
+                                   InstCombinerImpl &IC) {
   CmpInst::Predicate Pred0, Pred1;
   Value *X;
   if (!match(Cmp0, m_ICmp(Pred0, m_Intrinsic<Intrinsic::ctpop>(m_Value(X)),
@@ -945,19 +947,29 @@ static Value *foldIsPowerOf2OrZero(ICmpInst *Cmp0, ICmpInst *Cmp1, bool IsAnd,
       !match(Cmp1, m_ICmp(Pred1, m_Specific(X), m_ZeroInt())))
     return nullptr;
 
-  Value *CtPop = Cmp0->getOperand(0);
-  if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_NE)
+  auto *CtPop = cast<Instruction>(Cmp0->getOperand(0));
+  if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_NE) {
+    // Drop range attributes and re-infer them in the next iteration.
+    CtPop->dropPoisonGeneratingAnnotations();
+    IC.addToWorklist(CtPop);
     return Builder.CreateICmpUGT(CtPop, ConstantInt::get(CtPop->getType(), 1));
-  if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_EQ)
+  }
+  if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_EQ) {
+    // Drop range attributes and re-infer them in the next iteration.
+    CtPop->dropPoisonGeneratingAnnotations();
+    IC.addToWorklist(CtPop);
     return Builder.CreateICmpULT(CtPop, ConstantInt::get(CtPop->getType(), 2));
+  }
 
   return nullptr;
 }
 
 /// Reduce a pair of compares that check if a value has exactly 1 bit set.
-/// Also used for logical and/or, must be poison safe.
+/// Also used for logical and/or, must be poison safe if range attributes are
+/// dropped.
 static Value *foldIsPowerOf2(ICmpInst *Cmp0, ICmpInst *Cmp1, bool JoinedByAnd,
-                             InstCombiner::BuilderTy &Builder) {
+                             InstCombiner::BuilderTy &Builder,
+                             InstCombinerImpl &IC) {
   // Handle 'and' / 'or' commutation: make the equality check the first operand.
   if (JoinedByAnd && Cmp1->getPredicate() == ICmpInst::ICMP_NE)
     std::swap(Cmp0, Cmp1);
@@ -971,7 +983,10 @@ static Value *foldIsPowerOf2(ICmpInst *Cmp0, ICmpInst *Cmp1, bool JoinedByAnd,
       match(Cmp1, m_SpecificICmp(ICmpInst::ICMP_ULT,
                                  m_Intrinsic<Intrinsic::ctpop>(m_Specific(X)),
                                  m_SpecificInt(2)))) {
-    Value *CtPop = Cmp1->getOperand(0);
+    auto *CtPop = cast<Instruction>(Cmp1->getOperand(0));
+    // Drop range attributes and re-infer them in the next iteration.
+    CtPop->dropPoisonGeneratingAnnotations();
+    IC.addToWorklist(CtPop);
     return Builder.CreateICmpEQ(CtPop, ConstantInt::get(CtPop->getType(), 1));
   }
   // (X == 0) || (ctpop(X) u> 1) --> ctpop(X) != 1
@@ -980,7 +995,10 @@ static Value *foldIsPowerOf2(ICmpInst *Cmp0, ICmpInst *Cmp1, bool JoinedByAnd,
       match(Cmp1, m_SpecificICmp(ICmpInst::ICMP_UGT,
                                  m_Intrinsic<Intrinsic::ctpop>(m_Specific(X)),
                                  m_SpecificInt(1)))) {
-    Value *CtPop = Cmp1->getOperand(0);
+    auto *CtPop = cast<Instruction>(Cmp1->getOperand(0));
+    // Drop range attributes and re-infer them in the next iteration.
+    CtPop->dropPoisonGeneratingAnnotations();
+    IC.addToWorklist(CtPop);
     return Builder.CreateICmpNE(CtPop, ConstantInt::get(CtPop->getType(), 1));
   }
   return nullptr;
@@ -2261,7 +2279,8 @@ foldBitwiseLogicWithIntrinsics(BinaryOperator &I,
         Builder.CreateBinOp(I.getOpcode(), X->getOperand(0), Y->getOperand(0));
     Value *NewOp1 =
         Builder.CreateBinOp(I.getOpcode(), X->getOperand(1), Y->getOperand(1));
-    Function *F = Intrinsic::getDeclaration(I.getModule(), IID, I.getType());
+    Function *F =
+        Intrinsic::getOrInsertDeclaration(I.getModule(), IID, I.getType());
     return CallInst::Create(F, {NewOp0, NewOp1, X->getOperand(2)});
   }
   case Intrinsic::bswap:
@@ -2272,7 +2291,8 @@ foldBitwiseLogicWithIntrinsics(BinaryOperator &I,
           : ConstantInt::get(I.getType(), IID == Intrinsic::bswap
                                               ? RHSC->byteSwap()
                                               : RHSC->reverseBits()));
-    Function *F = Intrinsic::getDeclaration(I.getModule(), IID, I.getType());
+    Function *F =
+        Intrinsic::getOrInsertDeclaration(I.getModule(), IID, I.getType());
     return CallInst::Create(F, {NewOp0});
   }
   default:
@@ -3048,7 +3068,8 @@ InstCombinerImpl::convertOrOfShiftsToFunnelShift(Instruction &Or) {
 static Instruction *matchFunnelShift(Instruction &Or, InstCombinerImpl &IC) {
   if (auto Opt = IC.convertOrOfShiftsToFunnelShift(Or)) {
     auto [IID, FShiftArgs] = *Opt;
-    Function *F = Intrinsic::getDeclaration(Or.getModule(), IID, Or.getType());
+    Function *F =
+        Intrinsic::getOrInsertDeclaration(Or.getModule(), IID, Or.getType());
     return CallInst::Create(F, FShiftArgs);
   }
 
@@ -3087,7 +3108,7 @@ static Instruction *matchOrConcat(Instruction &Or,
     Value *NewUpper = Builder.CreateZExt(Hi, Ty);
     NewUpper = Builder.CreateShl(NewUpper, HalfWidth);
     Value *BinOp = Builder.CreateOr(NewLower, NewUpper);
-    Function *F = Intrinsic::getDeclaration(Or.getModule(), id, Ty);
+    Function *F = Intrinsic::getOrInsertDeclaration(Or.getModule(), id, Ty);
     return Builder.CreateCall(F, BinOp);
   };
 
@@ -3351,9 +3372,9 @@ Value *InstCombinerImpl::foldAndOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
                                              /*IsLogical*/ false, Builder, Q))
     return V;
 
-  if (Value *V = foldIsPowerOf2OrZero(LHS, RHS, IsAnd, Builder))
+  if (Value *V = foldIsPowerOf2OrZero(LHS, RHS, IsAnd, Builder, *this))
     return V;
-  if (Value *V = foldIsPowerOf2OrZero(RHS, LHS, IsAnd, Builder))
+  if (Value *V = foldIsPowerOf2OrZero(RHS, LHS, IsAnd, Builder, *this))
     return V;
 
   // TODO: One of these directions is fine with logical and/or, the other could
@@ -3375,7 +3396,7 @@ Value *InstCombinerImpl::foldAndOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
     if (Value *V = foldSignedTruncationCheck(LHS, RHS, I, Builder))
       return V;
 
-  if (Value *V = foldIsPowerOf2(LHS, RHS, IsAnd, Builder))
+  if (Value *V = foldIsPowerOf2(LHS, RHS, IsAnd, Builder, *this))
     return V;
 
   if (Value *V = foldPowerOf2AndShiftedMask(LHS, RHS, IsAnd, Builder))
@@ -4795,7 +4816,8 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
             match(II->getArgOperand(1), m_One()) &&
             isKnownToBeAPowerOfTwo(II->getArgOperand(0), /*OrZero */ true)) {
           IID = (IID == Intrinsic::ctlz) ? Intrinsic::cttz : Intrinsic::ctlz;
-          Function *F = Intrinsic::getDeclaration(II->getModule(), IID, Ty);
+          Function *F =
+              Intrinsic::getOrInsertDeclaration(II->getModule(), IID, Ty);
           return CallInst::Create(F, {II->getArgOperand(0), Builder.getTrue()});
         }
       }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index e6da1e72a31ad681a7940a1c35070b95561dd5b6..11196268cdd09fc0a23c9cd4a553302c8428a08e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -488,7 +488,8 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {
   // cttz(bitreverse(x)) -> ctlz(x)
   if (match(Op0, m_BitReverse(m_Value(X)))) {
     Intrinsic::ID ID = IsTZ ? Intrinsic::ctlz : Intrinsic::cttz;
-    Function *F = Intrinsic::getDeclaration(II.getModule(), ID, II.getType());
+    Function *F =
+        Intrinsic::getOrInsertDeclaration(II.getModule(), ID, II.getType());
     return CallInst::Create(F, {X, II.getArgOperand(1)});
   }
 
@@ -647,7 +648,7 @@ static Instruction *foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC) {
   if (Op0->hasOneUse() &&
       match(Op0, m_c_Or(m_Value(X), m_Neg(m_Deferred(X))))) {
     Function *F =
-        Intrinsic::getDeclaration(II.getModule(), Intrinsic::cttz, Ty);
+        Intrinsic::getOrInsertDeclaration(II.getModule(), Intrinsic::cttz, Ty);
     auto *Cttz = IC.Builder.CreateCall(F, {X, IC.Builder.getFalse()});
     auto *Bw = ConstantInt::get(Ty, APInt(BitWidth, BitWidth));
     return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Bw, Cttz));
@@ -657,7 +658,7 @@ static Instruction *foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC) {
   if (match(Op0,
             m_c_And(m_Not(m_Value(X)), m_Add(m_Deferred(X), m_AllOnes())))) {
     Function *F =
-        Intrinsic::getDeclaration(II.getModule(), Intrinsic::cttz, Ty);
+        Intrinsic::getOrInsertDeclaration(II.getModule(), Intrinsic::cttz, Ty);
     return CallInst::Create(F, {X, IC.Builder.getFalse()});
   }
 
@@ -1181,7 +1182,8 @@ Instruction *InstCombinerImpl::matchSAddSubSat(IntrinsicInst &MinMax1) {
     return nullptr;
 
   // Finally create and return the sat intrinsic, truncated to the new type
-  Function *F = Intrinsic::getDeclaration(MinMax1.getModule(), IntrinsicID, NewTy);
+  Function *F = Intrinsic::getOrInsertDeclaration(MinMax1.getModule(),
+                                                  IntrinsicID, NewTy);
   Value *AT = Builder.CreateTrunc(AddSub->getOperand(0), NewTy);
   Value *BT = Builder.CreateTrunc(AddSub->getOperand(1), NewTy);
   Value *Sat = Builder.CreateCall(F, {AT, BT});
@@ -1286,8 +1288,8 @@ reassociateMinMaxWithConstantInOperand(IntrinsicInst *II,
     return nullptr;
 
   // max (max X, C), Y --> max (max X, Y), C
-  Function *MinMax =
-      Intrinsic::getDeclaration(II->getModule(), MinMaxID, II->getType());
+  Function *MinMax = Intrinsic::getOrInsertDeclaration(II->getModule(),
+                                                       MinMaxID, II->getType());
   Value *NewInner = Builder.CreateBinaryIntrinsic(MinMaxID, X, Y);
   NewInner->takeName(Inner);
   return CallInst::Create(MinMax, {NewInner, C});
@@ -1346,7 +1348,8 @@ static Instruction *factorizeMinMaxTree(IntrinsicInst *II) {
     return nullptr;
 
   Module *Mod = II->getModule();
-  Function *MinMax = Intrinsic::getDeclaration(Mod, MinMaxID, II->getType());
+  Function *MinMax =
+      Intrinsic::getOrInsertDeclaration(Mod, MinMaxID, II->getType());
   return CallInst::Create(MinMax, { MinMaxOp, ThirdOp });
 }
 
@@ -1571,7 +1574,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
           Type *Tys[3] = { CI.getArgOperand(0)->getType(),
                            CI.getArgOperand(1)->getType(),
                            CI.getArgOperand(2)->getType() };
-          CI.setCalledFunction(Intrinsic::getDeclaration(M, MemCpyID, Tys));
+          CI.setCalledFunction(
+              Intrinsic::getOrInsertDeclaration(M, MemCpyID, Tys));
           Changed = true;
         }
     }
@@ -2097,7 +2101,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
 
         Constant *LeftShiftC = ConstantExpr::getSub(WidthC, ShAmtC);
         Module *Mod = II->getModule();
-        Function *Fshl = Intrinsic::getDeclaration(Mod, Intrinsic::fshl, Ty);
+        Function *Fshl =
+            Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::fshl, Ty);
         return CallInst::Create(Fshl, { Op0, Op1, LeftShiftC });
       }
       assert(IID == Intrinsic::fshl &&
@@ -2117,7 +2122,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       // fshl i16 X, X, 8 --> bswap i16 X (reduce to more-specific form)
       if (Op0 == Op1 && BitWidth == 16 && match(ShAmtC, m_SpecificInt(8))) {
         Module *Mod = II->getModule();
-        Function *Bswap = Intrinsic::getDeclaration(Mod, Intrinsic::bswap, Ty);
+        Function *Bswap =
+            Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::bswap, Ty);
         return CallInst::Create(Bswap, { Op0 });
       }
       if (Instruction *BitOp =
@@ -2826,7 +2832,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       CallArgs.push_back(II->getArgOperand(4));
     }
 
-    Function *NewFn = Intrinsic::getDeclaration(II->getModule(), NewIntrin);
+    Function *NewFn =
+        Intrinsic::getOrInsertDeclaration(II->getModule(), NewIntrin);
     return CallInst::Create(NewFn, CallArgs);
   }
   case Intrinsic::arm_neon_vtbl1:
@@ -3676,26 +3683,12 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   //  * The intrinsic is speculatable.
   //  * The select condition is not a vector, or the intrinsic does not
   //    perform cross-lane operations.
-  switch (IID) {
-  case Intrinsic::ctlz:
-  case Intrinsic::cttz:
-  case Intrinsic::ctpop:
-  case Intrinsic::umin:
-  case Intrinsic::umax:
-  case Intrinsic::smin:
-  case Intrinsic::smax:
-  case Intrinsic::usub_sat:
-  case Intrinsic::uadd_sat:
-  case Intrinsic::ssub_sat:
-  case Intrinsic::sadd_sat:
+  if (isSafeToSpeculativelyExecuteWithVariableReplaced(&CI) &&
+      isNotCrossLaneOperation(II))
     for (Value *Op : II->args())
       if (auto *Sel = dyn_cast<SelectInst>(Op))
         if (Instruction *R = FoldOpIntoSelect(*II, Sel))
           return R;
-    [[fallthrough]];
-  default:
-    break;
-  }
 
   if (Instruction *Shuf = foldShuffledIntrinsicOperands(II, Builder))
     return Shuf;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 9934c065ebf85f3c7094cd60e4febe4951f3f6c8..6c2554ea73b7f8d01f90933633193cfc07ce7606 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -95,8 +95,8 @@ Value *InstCombinerImpl::EvaluateInDifferentType(Value *V, Type *Ty,
       default:
         llvm_unreachable("Unsupported call!");
       case Intrinsic::vscale: {
-        Function *Fn =
-            Intrinsic::getDeclaration(I->getModule(), Intrinsic::vscale, {Ty});
+        Function *Fn = Intrinsic::getOrInsertDeclaration(
+            I->getModule(), Intrinsic::vscale, {Ty});
         Res = CallInst::Create(Fn->getFunctionType(), Fn);
         break;
       }
@@ -600,7 +600,8 @@ Instruction *InstCombinerImpl::narrowFunnelShift(TruncInst &Trunc) {
   if (ShVal0 != ShVal1)
     Y = Builder.CreateTrunc(ShVal1, DestTy);
   Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
-  Function *F = Intrinsic::getDeclaration(Trunc.getModule(), IID, DestTy);
+  Function *F =
+      Intrinsic::getOrInsertDeclaration(Trunc.getModule(), IID, DestTy);
   return CallInst::Create(F, {X, Y, NarrowShAmt});
 }
 
@@ -1912,8 +1913,8 @@ Instruction *InstCombinerImpl::visitFPTrunc(FPTruncInst &FPT) {
       // Do unary FP operation on smaller type.
       // (fptrunc (fabs x)) -> (fabs (fptrunc x))
       Value *InnerTrunc = Builder.CreateFPTrunc(Src, Ty);
-      Function *Overload = Intrinsic::getDeclaration(FPT.getModule(),
-                                                     II->getIntrinsicID(), Ty);
+      Function *Overload = Intrinsic::getOrInsertDeclaration(
+          FPT.getModule(), II->getIntrinsicID(), Ty);
       SmallVector<OperandBundleDef, 1> OpBundles;
       II->getOperandBundlesAsDefs(OpBundles);
       CallInst *NewCI =
@@ -2855,8 +2856,8 @@ Instruction *InstCombinerImpl::visitBitCast(BitCastInst &CI) {
       if (IntrinsicNum != 0) {
         assert(ShufOp0->getType() == SrcTy && "Unexpected shuffle mask");
         assert(match(ShufOp1, m_Undef()) && "Unexpected shuffle op");
-        Function *BswapOrBitreverse =
-            Intrinsic::getDeclaration(CI.getModule(), IntrinsicNum, DestTy);
+        Function *BswapOrBitreverse = Intrinsic::getOrInsertDeclaration(
+            CI.getModule(), IntrinsicNum, DestTy);
         Value *ScalarX = Builder.CreateBitCast(ShufOp0, DestTy);
         return CallInst::Create(BswapOrBitreverse, {ScalarX});
       }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index d1eb84b5ca5c105ea675acd2ac9ee2ff1d87225b..7129499e0f8f9db5625f857cfacdfbdbdafd8e34 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1125,7 +1125,7 @@ static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
   // use the sadd_with_overflow intrinsic to efficiently compute both the
   // result and the overflow bit.
   Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
-  Function *F = Intrinsic::getDeclaration(
+  Function *F = Intrinsic::getOrInsertDeclaration(
       I.getModule(), Intrinsic::sadd_with_overflow, NewType);
 
   InstCombiner::BuilderTy &Builder = IC.Builder;
@@ -4790,11 +4790,11 @@ Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) {
   if (MulHadOtherUses)
     Builder.SetInsertPoint(Mul);
 
-  Function *F = Intrinsic::getDeclaration(I.getModule(),
-                                          Div->getOpcode() == Instruction::UDiv
-                                              ? Intrinsic::umul_with_overflow
-                                              : Intrinsic::smul_with_overflow,
-                                          X->getType());
+  Function *F = Intrinsic::getOrInsertDeclaration(
+      I.getModule(),
+      Div->getOpcode() == Instruction::UDiv ? Intrinsic::umul_with_overflow
+                                            : Intrinsic::smul_with_overflow,
+      X->getType());
   CallInst *Call = Builder.CreateCall(F, {X, Y}, "mul");
 
   // If the multiplication was used elsewhere, to ensure that we don't leave
@@ -6334,7 +6334,7 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
     MulA = Builder.CreateZExt(A, MulType);
   if (WidthB < MulWidth)
     MulB = Builder.CreateZExt(B, MulType);
-  Function *F = Intrinsic::getDeclaration(
+  Function *F = Intrinsic::getOrInsertDeclaration(
       I.getModule(), Intrinsic::umul_with_overflow, MulType);
   CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul");
   IC.addToWorklist(MulInstr);
@@ -7121,8 +7121,8 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
     if (auto *I = dyn_cast<Instruction>(V))
       I->copyIRFlags(&Cmp);
     Module *M = Cmp.getModule();
-    Function *F =
-        Intrinsic::getDeclaration(M, Intrinsic::vector_reverse, V->getType());
+    Function *F = Intrinsic::getOrInsertDeclaration(
+        M, Intrinsic::vector_reverse, V->getType());
     return CallInst::Create(F, V);
   };
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 3f780285efe4235709e22c1284f4252f99158397..820d3608c8dc49c580d4022bd7a2ef86fa1c6472 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1148,8 +1148,8 @@ static Instruction *foldSelectCtlzToCttz(ICmpInst *ICI, Value *TrueVal,
   if (!match(II->getOperand(0), m_c_And(m_Specific(X), m_Neg(m_Specific(X)))))
     return nullptr;
 
-  Function *F = Intrinsic::getDeclaration(II->getModule(), Intrinsic::cttz,
-                                          II->getType());
+  Function *F = Intrinsic::getOrInsertDeclaration(
+      II->getModule(), Intrinsic::cttz, II->getType());
   return CallInst::Create(F, {X, II->getArgOperand(1)});
 }
 
@@ -1288,6 +1288,10 @@ bool InstCombinerImpl::replaceInInstruction(Value *V, Value *Old, Value *New,
       !isSafeToSpeculativelyExecuteWithVariableReplaced(I))
     return false;
 
+  // Forbid potentially lane-crossing instructions.
+  if (Old->getType()->isVectorTy() && !isNotCrossLaneOperation(I))
+    return false;
+
   bool Changed = false;
   for (Use &U : I->operands()) {
     if (U == Old) {
@@ -1366,9 +1370,8 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
     // with different operands, which should not cause side-effects or trigger
     // undefined behavior). Only do this if CmpRHS is a constant, as
     // profitability is not clear for other cases.
-    // FIXME: Support vectors.
     if (OldOp == CmpLHS && match(NewOp, m_ImmConstant()) &&
-        !match(OldOp, m_Constant()) && !Cmp.getType()->isVectorTy() &&
+        !match(OldOp, m_Constant()) &&
         isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT))
       if (replaceInInstruction(TrueVal, OldOp, NewOp))
         return &Sel;
@@ -2242,8 +2245,8 @@ foldOverflowingAddSubSelect(SelectInst &SI, InstCombiner::BuilderTy &Builder) {
   else
     return nullptr;
 
-  Function *F =
-      Intrinsic::getDeclaration(SI.getModule(), NewIntrinsicID, SI.getType());
+  Function *F = Intrinsic::getOrInsertDeclaration(SI.getModule(),
+                                                  NewIntrinsicID, SI.getType());
   return CallInst::Create(F, {X, Y});
 }
 
@@ -2537,7 +2540,8 @@ static Instruction *foldSelectFunnelShift(SelectInst &Sel,
   // This is a funnel/rotate that avoids shift-by-bitwidth UB in a suboptimal way.
   // Convert to funnel shift intrinsic.
   Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
-  Function *F = Intrinsic::getDeclaration(Sel.getModule(), IID, Sel.getType());
+  Function *F =
+      Intrinsic::getOrInsertDeclaration(Sel.getModule(), IID, Sel.getType());
   ShAmt = Builder.CreateZExt(ShAmt, Sel.getType());
   return CallInst::Create(F, { SV0, SV1, ShAmt });
 }
@@ -2580,8 +2584,8 @@ static Instruction *foldSelectToCopysign(SelectInst &Sel,
   // Canonicalize the magnitude argument as the positive constant since we do
   // not care about its sign.
   Value *MagArg = ConstantFP::get(SelType, abs(*TC));
-  Function *F = Intrinsic::getDeclaration(Sel.getModule(), Intrinsic::copysign,
-                                          Sel.getType());
+  Function *F = Intrinsic::getOrInsertDeclaration(
+      Sel.getModule(), Intrinsic::copysign, Sel.getType());
   return CallInst::Create(F, { MagArg, X });
 }
 
@@ -2600,8 +2604,8 @@ Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
       if (auto *I = dyn_cast<Instruction>(V))
         I->copyIRFlags(&Sel);
       Module *M = Sel.getModule();
-      Function *F =
-          Intrinsic::getDeclaration(M, Intrinsic::vector_reverse, V->getType());
+      Function *F = Intrinsic::getOrInsertDeclaration(
+          M, Intrinsic::vector_reverse, V->getType());
       return CallInst::Create(F, V);
     };
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 3d4461dc1a87f6c2077def1767fda983dbe7291d..8ca705ae1d364d132e19279e318305d3350e966e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -898,7 +898,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
         Value *X;
         if (DemandedMask == 1 && VTy->getScalarSizeInBits() % 2 == 0 &&
             match(II->getArgOperand(0), m_Not(m_Value(X)))) {
-          Function *Ctpop = Intrinsic::getDeclaration(
+          Function *Ctpop = Intrinsic::getOrInsertDeclaration(
               II->getModule(), Intrinsic::ctpop, VTy);
           return InsertNewInstWith(CallInst::Create(Ctpop, {X}), I->getIterator());
         }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index d9b4faff4c004da406a82199a243e2f2c86a5c9b..d68ae64f08aa90a2958bc99b0ef0e21f858727f2 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2474,8 +2474,8 @@ static Instruction *foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf,
     if (IsFNeg)
       return UnaryOperator::CreateFNegFMF(NewShuf, S0);
 
-    Function *FAbs = Intrinsic::getDeclaration(Shuf.getModule(),
-                                               Intrinsic::fabs, Shuf.getType());
+    Function *FAbs = Intrinsic::getOrInsertDeclaration(
+        Shuf.getModule(), Intrinsic::fabs, Shuf.getType());
     CallInst *NewF = CallInst::Create(FAbs, {NewShuf});
     NewF->setFastMathFlags(S0->getFastMathFlags());
     return NewF;
@@ -2495,8 +2495,8 @@ static Instruction *foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf,
   if (IsFNeg) {
     NewF = UnaryOperator::CreateFNeg(NewShuf);
   } else {
-    Function *FAbs = Intrinsic::getDeclaration(Shuf.getModule(),
-                                               Intrinsic::fabs, Shuf.getType());
+    Function *FAbs = Intrinsic::getOrInsertDeclaration(
+        Shuf.getModule(), Intrinsic::fabs, Shuf.getType());
     NewF = CallInst::Create(FAbs, {NewShuf});
   }
   NewF->copyIRFlags(S0);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index c77a92e8e2d9c1f31326b55e82af719c69dc9788..bec842615bbac3c6b02566512fb990570be727fb 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -2084,8 +2084,8 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
     if (auto *BO = dyn_cast<BinaryOperator>(V))
       BO->copyIRFlags(&Inst);
     Module *M = Inst.getModule();
-    Function *F =
-        Intrinsic::getDeclaration(M, Intrinsic::vector_reverse, V->getType());
+    Function *F = Intrinsic::getOrInsertDeclaration(
+        M, Intrinsic::vector_reverse, V->getType());
     return CallInst::Create(F, V);
   };
 
@@ -3359,7 +3359,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
     if (InvokeInst *II = dyn_cast<InvokeInst>(&MI)) {
       // Replace invoke with a NOP intrinsic to maintain the original CFG
       Module *M = II->getModule();
-      Function *F = Intrinsic::getDeclaration(M, Intrinsic::donothing);
+      Function *F = Intrinsic::getOrInsertDeclaration(M, Intrinsic::donothing);
       InvokeInst::Create(F, II->getNormalDest(), II->getUnwindDest(), {}, "",
                          II->getParent());
     }
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 2ad89b5ba753a5da0f90fc6df676baa043055630..02d9fab309d83be8231775fbe832e444ca240e8f 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1109,7 +1109,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
     // alloca. We have a special @llvm.get.dynamic.area.offset intrinsic for
     // this purpose.
     if (!isa<ReturnInst>(InstBefore)) {
-      Function *DynamicAreaOffsetFunc = Intrinsic::getDeclaration(
+      Function *DynamicAreaOffsetFunc = Intrinsic::getOrInsertDeclaration(
           InstBefore->getModule(), Intrinsic::get_dynamic_area_offset,
           {IntptrTy});
 
@@ -1867,7 +1867,7 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
     const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex);
     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
     IRB.CreateCall(
-        Intrinsic::getDeclaration(M, Intrinsic::asan_check_memaccess),
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::asan_check_memaccess),
         {IRB.CreatePointerCast(Addr, PtrTy),
          ConstantInt::get(Int32Ty, AccessInfo.Packed)});
     return;
diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
index 618b6fe1aea4744b120477e01b4169b7fc5e467e..63d580d2b9d5124715497b1a74969f001a026a03 100644
--- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -194,7 +194,7 @@ static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI,
     IRB.SetInsertPoint(TrapBB);
 
     Intrinsic::ID IntrID = DebugTrapBB ? Intrinsic::ubsantrap : Intrinsic::trap;
-    auto *F = Intrinsic::getDeclaration(Fn->getParent(), IntrID);
+    auto *F = Intrinsic::getOrInsertDeclaration(Fn->getParent(), IntrID);
 
     CallInst *TrapCall;
     if (DebugTrapBB) {
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index cc7f20cffea77149b68fb7cbb08ec41cd542a4f2..5ec4973ea03d8ff28fed216a7c5534384a9164cb 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1042,14 +1042,14 @@ void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
 
   if (UseFixedShadowIntrinsic) {
     IRB.CreateCall(
-        Intrinsic::getDeclaration(
+        Intrinsic::getOrInsertDeclaration(
             M, UseShortGranules
                    ? Intrinsic::hwasan_check_memaccess_shortgranules_fixedshadow
                    : Intrinsic::hwasan_check_memaccess_fixedshadow),
         {Ptr, ConstantInt::get(Int32Ty, AccessInfo),
          ConstantInt::get(Int64Ty, Mapping.offset())});
   } else {
-    IRB.CreateCall(Intrinsic::getDeclaration(
+    IRB.CreateCall(Intrinsic::getOrInsertDeclaration(
                        M, UseShortGranules
                               ? Intrinsic::hwasan_check_memaccess_shortgranules
                               : Intrinsic::hwasan_check_memaccess),
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 1c95a4606ecc566cde79f47d593cca35961b7c0a..929c787442057a3753fecdfc1a8d535cfb742f13 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1406,9 +1406,10 @@ static inline Constant *getFuncAddrForProfData(Function *Fn) {
 
 static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
   // compiler-rt uses linker support to get data/counters/name start/end for
-  // ELF, COFF, Mach-O and XCOFF.
+  // ELF, COFF, Mach-O, XCOFF, and Wasm.
   if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
-      TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
+      TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||
+      TT.isOSBinFormatWasm())
     return false;
 
   return true;
diff --git a/llvm/lib/Transforms/Instrumentation/KCFI.cpp b/llvm/lib/Transforms/Instrumentation/KCFI.cpp
index 28dc1c02b661ac7f914d2bfc51fed8e4b8c0cdee..bbe0f4c6178192d9d8af3abd017614a1dbdb7c3b 100644
--- a/llvm/lib/Transforms/Instrumentation/KCFI.cpp
+++ b/llvm/lib/Transforms/Instrumentation/KCFI.cpp
@@ -110,7 +110,8 @@ PreservedAnalyses KCFIPass::run(Function &F, FunctionAnalysisManager &AM) {
     Instruction *ThenTerm =
         SplitBlockAndInsertIfThen(Test, Call, false, VeryUnlikelyWeights);
     Builder.SetInsertPoint(ThenTerm);
-    Builder.CreateCall(Intrinsic::getDeclaration(&M, Intrinsic::debugtrap));
+    Builder.CreateCall(
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::debugtrap));
     ++NumKCFIChecks;
   }
 
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 07d667434e0710e9ac9d6e82b22de016c04d790b..19ec97c17f31c6609b8a19989deaf9ce2bf560d5 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2853,7 +2853,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *S2Conv =
         IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
     Value *V2 = I.getOperand(2);
-    Function *Intrin = Intrinsic::getDeclaration(
+    Function *Intrin = Intrinsic::getOrInsertDeclaration(
         I.getModule(), I.getIntrinsicID(), S2Conv->getType());
     Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2});
     setShadow(&I, IRB.CreateOr(Shift, S2Conv));
@@ -3057,7 +3057,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     IRBuilder<> IRB(&I);
     Value *Op = I.getArgOperand(0);
     Type *OpType = Op->getType();
-    Function *BswapFunc = Intrinsic::getDeclaration(
+    Function *BswapFunc = Intrinsic::getOrInsertDeclaration(
         F.getParent(), Intrinsic::bswap, ArrayRef(&OpType, 1));
     setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
     setOrigin(&I, getOrigin(Op));
@@ -3287,7 +3287,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       S2_ext = IRB.CreateBitCast(S2_ext, getMMXVectorTy(64));
     }
 
-    Function *ShadowFn = Intrinsic::getDeclaration(
+    Function *ShadowFn = Intrinsic::getOrInsertDeclaration(
         F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
 
     Value *S =
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 10442fa0bb9003dcd973cebd2a8b6246ff355690..dbe908bb5e72f3fbeae9708318e12e78f9a0f0ca 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -919,7 +919,7 @@ void FunctionInstrumenter::instrument() {
     // llvm.instrprof.cover(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                      i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_cover),
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::instrprof_cover),
         {NormalizedNamePtr, CFGHash, Builder.getInt32(1), Builder.getInt32(0)});
     return;
   }
@@ -931,7 +931,7 @@ void FunctionInstrumenter::instrument() {
 
   if (IsCtxProf) {
     auto *CSIntrinsic =
-        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_callsite);
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::instrprof_callsite);
     // We want to count the instrumentable callsites, then instrument them. This
     // is because the llvm.instrprof.callsite intrinsic has an argument (like
     // the other instrprof intrinsics) capturing the total number of
@@ -972,7 +972,7 @@ void FunctionInstrumenter::instrument() {
     // llvm.instrprof.timestamp(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                          i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_timestamp),
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::instrprof_timestamp),
         {NormalizedNamePtr, CFGHash, Builder.getInt32(NumCounters),
          Builder.getInt32(I)});
     I += PGOBlockCoverage ? 8 : 1;
@@ -984,12 +984,12 @@ void FunctionInstrumenter::instrument() {
            "Cannot get the Instrumentation point");
     // llvm.instrprof.increment(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                          i32 <index>)
-    Builder.CreateCall(
-        Intrinsic::getDeclaration(&M, PGOBlockCoverage
-                                          ? Intrinsic::instrprof_cover
-                                          : Intrinsic::instrprof_increment),
-        {NormalizedNamePtr, CFGHash, Builder.getInt32(NumCounters),
-         Builder.getInt32(I++)});
+    Builder.CreateCall(Intrinsic::getOrInsertDeclaration(
+                           &M, PGOBlockCoverage
+                                   ? Intrinsic::instrprof_cover
+                                   : Intrinsic::instrprof_increment),
+                       {NormalizedNamePtr, CFGHash,
+                        Builder.getInt32(NumCounters), Builder.getInt32(I++)});
   }
 
   // Now instrument select instructions:
@@ -1038,7 +1038,8 @@ void FunctionInstrumenter::instrument() {
       SmallVector<OperandBundleDef, 1> OpBundles;
       populateEHOperandBundle(Cand, BlockColors, OpBundles);
       Builder.CreateCall(
-          Intrinsic::getDeclaration(&M, Intrinsic::instrprof_value_profile),
+          Intrinsic::getOrInsertDeclaration(&M,
+                                            Intrinsic::instrprof_value_profile),
           {NormalizedNamePtr, Builder.getInt64(FuncInfo.FunctionHash),
            ToProfile, Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)},
           OpBundles);
@@ -1726,7 +1727,7 @@ void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {
       ConstantExpr::getPointerBitCastOrAddrSpaceCast(
           FuncNameVar, PointerType::get(M->getContext(), 0));
   Builder.CreateCall(
-      Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step),
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::instrprof_increment_step),
       {NormalizedFuncNameVarPtr, Builder.getInt64(FuncHash),
        Builder.getInt32(TotalNumCtrs), Builder.getInt32(*CurCtrIdx), Step});
   ++(*CurCtrIdx);
@@ -1915,6 +1916,7 @@ static bool InstrumentAllFunctions(
   std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
   collectComdatMembers(M, ComdatMembers);
 
+  bool AnythingInstrumented = false;
   for (auto &F : M) {
     if (skipPGOGen(F))
       continue;
@@ -1924,8 +1926,9 @@ static bool InstrumentAllFunctions(
     FunctionInstrumenter FI(M, F, TLI, ComdatMembers, BPI, BFI,
                             InstrumentationType);
     FI.instrument();
+    AnythingInstrumented = true;
   }
-  return true;
+  return AnythingInstrumented;
 }
 
 PreservedAnalyses
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index db4bf709c9cc9c46db78d0843a686c439e223d38..719806fdf37f58a364168dfb2a2e7f9dda549f11 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -999,7 +999,7 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
   if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
     // Check stack depth.  If it's the deepest so far, record it.
     Module *M = F.getParent();
-    Function *GetFrameAddr = Intrinsic::getDeclaration(
+    Function *GetFrameAddr = Intrinsic::getOrInsertDeclaration(
         M, Intrinsic::frameaddress,
         IRB.getPtrTy(M->getDataLayout().getAllocaAddrSpace()));
     auto FrameAddrPtr =
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 68cf4e553013140fb1e201cd899e0e7855a4bd0a..388addfab181a45e262495b859d15fcaa1a4fc82 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -571,9 +571,10 @@ bool ThreadSanitizer::sanitizeFunction(Function &F,
   // Instrument function entry/exit points if there were instrumented accesses.
   if ((Res || HasCalls) && ClInstrumentFuncEntryExit) {
     InstrumentationIRBuilder IRB(F.getEntryBlock().getFirstNonPHI());
-    Value *ReturnAddress = IRB.CreateCall(
-        Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress),
-        IRB.getInt32(0));
+    Value *ReturnAddress =
+        IRB.CreateCall(Intrinsic::getOrInsertDeclaration(
+                           F.getParent(), Intrinsic::returnaddress),
+                       IRB.getInt32(0));
     IRB.CreateCall(TsanFuncEntry, ReturnAddress);
 
     EscapeEnumerator EE(F, "tsan_cleanup", ClHandleCxxExceptions);
diff --git a/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h b/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h
index c11691c613ac78b0271fe014a91130551fe657d3..0dedd0207571bf9deaed8bc5a0ba579745ebe4be 100644
--- a/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h
+++ b/llvm/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h
@@ -139,7 +139,7 @@ private:
     if (Decl)
       return Decl;
 
-    return Decl = Intrinsic::getDeclaration(TheModule, IntID);
+    return Decl = Intrinsic::getOrInsertDeclaration(TheModule, IntID);
   }
 };
 
diff --git a/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp b/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
index 5d9a7bca7efeca23cb58fdba5b6b07350a86cb6f..7f9129697995a06b9e4bd42a175c3b89709a906c 100644
--- a/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
+++ b/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
@@ -52,8 +52,7 @@ static void runImpl(Function &F, const TargetLibraryInfo &TLI) {
   for (Instruction &I : instructions(F)) {
     if (!I.hasMetadata(LLVMContext::MD_annotation))
       continue;
-    auto Iter = DebugLoc2Annotated.insert({I.getDebugLoc().getAsMDNode(), {}});
-    Iter.first->second.push_back(&I);
+    DebugLoc2Annotated[I.getDebugLoc().getAsMDNode()].push_back(&I);
 
     for (const MDOperand &Op :
          I.getMetadata(LLVMContext::MD_annotation)->operands()) {
@@ -62,8 +61,7 @@ static void runImpl(Function &F, const TargetLibraryInfo &TLI) {
               ? cast<MDString>(Op.get())->getString()
               : cast<MDString>(cast<MDTuple>(Op.get())->getOperand(0).get())
                     ->getString();
-      auto Iter = Mapping.insert({AnnotationStr, 0});
-      Iter.first->second++;
+      Mapping[AnnotationStr]++;
     }
   }
 
diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index 60fd2a286119b3fa588b194d3ac71a7700f65101..9317e0643079ea4615254c72a128230d647a3ff6 100644
--- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -380,7 +380,8 @@ bool InferAddressSpacesImpl::rewriteIntrinsicOperands(IntrinsicInst *II,
   case Intrinsic::masked_load: {
     Type *DestTy = II->getType();
     Type *SrcTy = NewV->getType();
-    Function *NewDecl = Intrinsic::getDeclaration(M, IID, {DestTy, SrcTy});
+    Function *NewDecl =
+        Intrinsic::getOrInsertDeclaration(M, IID, {DestTy, SrcTy});
     II->setArgOperand(0, NewV);
     II->setCalledFunction(NewDecl);
     return true;
@@ -391,7 +392,8 @@ bool InferAddressSpacesImpl::rewriteIntrinsicOperands(IntrinsicInst *II,
   case Intrinsic::masked_gather: {
     Type *RetTy = II->getType();
     Type *NewPtrTy = NewV->getType();
-    Function *NewDecl = Intrinsic::getDeclaration(M, IID, {RetTy, NewPtrTy});
+    Function *NewDecl =
+        Intrinsic::getOrInsertDeclaration(M, IID, {RetTy, NewPtrTy});
     II->setArgOperand(0, NewV);
     II->setCalledFunction(NewDecl);
     return true;
@@ -400,16 +402,16 @@ bool InferAddressSpacesImpl::rewriteIntrinsicOperands(IntrinsicInst *II,
   case Intrinsic::masked_scatter: {
     Type *ValueTy = II->getOperand(0)->getType();
     Type *NewPtrTy = NewV->getType();
-    Function *NewDecl =
-        Intrinsic::getDeclaration(M, II->getIntrinsicID(), {ValueTy, NewPtrTy});
+    Function *NewDecl = Intrinsic::getOrInsertDeclaration(
+        M, II->getIntrinsicID(), {ValueTy, NewPtrTy});
     II->setArgOperand(1, NewV);
     II->setCalledFunction(NewDecl);
     return true;
   }
   case Intrinsic::prefetch:
   case Intrinsic::is_constant: {
-    Function *NewDecl =
-        Intrinsic::getDeclaration(M, II->getIntrinsicID(), {NewV->getType()});
+    Function *NewDecl = Intrinsic::getOrInsertDeclaration(
+        M, II->getIntrinsicID(), {NewV->getType()});
     II->setArgOperand(0, NewV);
     II->setCalledFunction(NewDecl);
     return true;
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index d85166e518f1dbf9fd569efd700cad0e86ca4ae1..4043c0e9a7ddc4ec3751aedb08432b4f7b77d72d 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -405,7 +405,7 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
     IRBuilder<> Builder(P.InsertPt);
     Module *M = BB->getParent()->getParent();
     Type *I32 = Type::getInt32Ty(BB->getContext());
-    Function *PrefetchFunc = Intrinsic::getDeclaration(
+    Function *PrefetchFunc = Intrinsic::getOrInsertDeclaration(
         M, Intrinsic::prefetch, PrefPtrValue->getType());
     Builder.CreateCall(
         PrefetchFunc,
diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index d5e91d3c1decf8f8b4e66662a9193a13b0e1bc80..30369ed7c245cf666a44a6f7896e0df395e0fb56 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -978,8 +978,8 @@ static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
     assert(match(Br->getCondition(), m_Zero()) &&
            "Expected branch condition to be false");
     IRBuilder<> Builder(Br);
-    Function *F = Intrinsic::getDeclaration(M, Intrinsic::umul_with_overflow,
-                                            FI.OuterTripCount->getType());
+    Function *F = Intrinsic::getOrInsertDeclaration(
+        M, Intrinsic::umul_with_overflow, FI.OuterTripCount->getType());
     Value *Call = Builder.CreateCall(F, {FI.OuterTripCount, FI.InnerTripCount},
                                      "flatten.mul");
     FI.NewTripCount = Builder.CreateExtractValue(Call, 0, "flatten.tripcount");
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 740e1e39b9ee7704b2137c41f9f1bc1164aa9251..56006d9ae6924ad3c009ae021bf77c5d8e30028f 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -2122,7 +2122,7 @@ static CallInst *createPopcntIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
   Type *Tys[] = {Val->getType()};
 
   Module *M = IRBuilder.GetInsertBlock()->getParent()->getParent();
-  Function *Func = Intrinsic::getDeclaration(M, Intrinsic::ctpop, Tys);
+  Function *Func = Intrinsic::getOrInsertDeclaration(M, Intrinsic::ctpop, Tys);
   CallInst *CI = IRBuilder.CreateCall(Func, Ops);
   CI->setDebugLoc(DL);
 
@@ -2136,7 +2136,7 @@ static CallInst *createFFSIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
   Type *Tys[] = {Val->getType()};
 
   Module *M = IRBuilder.GetInsertBlock()->getParent()->getParent();
-  Function *Func = Intrinsic::getDeclaration(M, IID, Tys);
+  Function *Func = Intrinsic::getOrInsertDeclaration(M, IID, Tys);
   CallInst *CI = IRBuilder.CreateCall(Func, Ops);
   CI->setDebugLoc(DL);
 
diff --git a/llvm/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp
index a59ecdda1746f90c5e87aa15e07911815369ea9c..ce35349376c483ffb742bacf8d3d7a3f49bae4e5 100644
--- a/llvm/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp
@@ -44,7 +44,7 @@ static bool lowerGuardIntrinsic(Function &F) {
   if (ToLower.empty())
     return false;
 
-  auto *DeoptIntrinsic = Intrinsic::getDeclaration(
+  auto *DeoptIntrinsic = Intrinsic::getOrInsertDeclaration(
       F.getParent(), Intrinsic::experimental_deoptimize, {F.getReturnType()});
   DeoptIntrinsic->setCallingConv(GuardDecl->getCallingConv());
 
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index 0d98e844cf91ea0f233aaadd673b748c5c1a09c7..f2ea9f8faf84cd708f9483f784de50797e7b7883 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -1290,7 +1290,7 @@ public:
       if (AllowContraction) {
         // Use fmuladd for floating point operations and let the backend decide
         // if that's profitable.
-        Function *FMulAdd = Intrinsic::getDeclaration(
+        Function *FMulAdd = Intrinsic::getOrInsertDeclaration(
             Func.getParent(), Intrinsic::fmuladd, A->getType());
         return Builder.CreateCall(FMulAdd, {A, B, Sum});
       }
@@ -2486,8 +2486,7 @@ public:
       if (!ExprsInSubprogram.count(V))
         return;
 
-      auto I = Shared.insert({V, {}});
-      I.first->second.insert(Leaf);
+      Shared[V].insert(Leaf);
 
       for (Value *Op : cast<Instruction>(V)->operand_values())
         collectSharedInfo(Leaf, Op, ExprsInSubprogram, Shared);
@@ -2538,14 +2537,12 @@ public:
           auto *I = cast<Instruction>(KV.first);
           DILocation *Context = I->getDebugLoc();
           while (Context) {
-            auto I =
-                Subprog2Exprs.insert({getSubprogram(Context->getScope()), {}});
-            I.first->second.push_back(KV.first);
+            Subprog2Exprs[getSubprogram(Context->getScope())].push_back(
+                KV.first);
             Context = DebugLoc(Context).getInlinedAt();
           }
         } else {
-          auto I = Subprog2Exprs.insert({nullptr, {}});
-          I.first->second.push_back(KV.first);
+          Subprog2Exprs[nullptr].push_back(KV.first);
         }
       }
       for (auto &KV : Subprog2Exprs) {
diff --git a/llvm/lib/Transforms/Scalar/MakeGuardsExplicit.cpp b/llvm/lib/Transforms/Scalar/MakeGuardsExplicit.cpp
index aea17aa82a88a4b34a307e379276ad2fc77874ea..b9f88ba4e0780e00b3fa367f6aa1e19576418fc5 100644
--- a/llvm/lib/Transforms/Scalar/MakeGuardsExplicit.cpp
+++ b/llvm/lib/Transforms/Scalar/MakeGuardsExplicit.cpp
@@ -69,7 +69,7 @@ static bool explicifyGuards(Function &F) {
   if (GuardIntrinsics.empty())
     return false;
 
-  auto *DeoptIntrinsic = Intrinsic::getDeclaration(
+  auto *DeoptIntrinsic = Intrinsic::getOrInsertDeclaration(
       F.getParent(), Intrinsic::experimental_deoptimize, {F.getReturnType()});
   DeoptIntrinsic->setCallingConv(GuardDecl->getCallingConv());
 
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index b568811dcdbcac36b9208d2f9beee0c07cca8db1..557a75e8946dc3e4044b4f443bfb1330699ed33e 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -1855,8 +1855,8 @@ bool MemCpyOptPass::processMemMove(MemMoveInst *M) {
   // If not, then we know we can transform this.
   Type *ArgTys[3] = {M->getRawDest()->getType(), M->getRawSource()->getType(),
                      M->getLength()->getType()};
-  M->setCalledFunction(
-      Intrinsic::getDeclaration(M->getModule(), Intrinsic::memcpy, ArgTys));
+  M->setCalledFunction(Intrinsic::getOrInsertDeclaration(
+      M->getModule(), Intrinsic::memcpy, ArgTys));
 
   // For MemorySSA nothing really changes (except that memcpy may imply stricter
   // aliasing guarantees).
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index e3c12c971b9ab0530c9d39b068aeb8d490c48532..daf8fa28a71e598a997d74cdcdd65fb95382ac15 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1525,8 +1525,8 @@ static void CreateGCRelocates(ArrayRef<Value *> LiveVariables,
     if (auto *VT = dyn_cast<VectorType>(Ty))
       NewTy = FixedVectorType::get(NewTy,
                                    cast<FixedVectorType>(VT)->getNumElements());
-    return Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate,
-                                     {NewTy});
+    return Intrinsic::getOrInsertDeclaration(
+        M, Intrinsic::experimental_gc_relocate, {NewTy});
   };
 
   // Lazily populated map from input types to the canonicalized form mentioned
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index 72728c0f839e5d497f0a082d9d701f0633b241c7..b1e4c7e52d99a02e9cc9e466db155e79977ca3a6 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -733,7 +733,8 @@ bool ScalarizerVisitor::splitCall(CallInst &CI) {
   ValueVector Res(VS->NumFragments);
   ValueVector ScalarCallOps(NumArgs);
 
-  Function *NewIntrin = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+  Function *NewIntrin =
+      Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
   IRBuilder<> Builder(&CI);
 
   // Perform actual scalarization, taking care to preserve any scalar operands.
@@ -756,7 +757,7 @@ bool ScalarizerVisitor::splitCall(CallInst &CI) {
     }
 
     if (IsRemainder)
-      NewIntrin = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+      NewIntrin = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
 
     Res[I] = Builder.CreateCall(NewIntrin, ScalarCallOps,
                                 CI.getName() + ".i" + Twine(I));
diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index 3cf68e07da5be2c21ab23e210124ed508a4dd460..e1dd20478fd55fff25158354dd3cdb2991290337 100644
--- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
+++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -225,7 +225,8 @@ struct AssumeBuilderState {
       return nullptr;
     if (!DebugCounter::shouldExecute(BuildAssumeCounter))
       return nullptr;
-    Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
+    Function *FnAssume =
+        Intrinsic::getOrInsertDeclaration(M, Intrinsic::assume);
     LLVMContext &C = M->getContext();
     SmallVector<OperandBundleDef, 8> OpBundle;
     for (auto &MapElem : AssumedKnowledgeMap) {
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index fc03643e3542cce4a01f7f9fcc80b152f3ee63dc..c6ba85bd9e57d458af92f0db3e628d926f9f03a9 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -425,8 +425,8 @@ PruningFunctionCloner::cloneInstruction(BasicBlock::const_iterator II) {
 
       // Create intrinsic call.
       LLVMContext &Ctx = NewFunc->getContext();
-      Function *IFn =
-          Intrinsic::getDeclaration(NewFunc->getParent(), CIID, TParams);
+      Function *IFn = Intrinsic::getOrInsertDeclaration(NewFunc->getParent(),
+                                                        CIID, TParams);
       SmallVector<Value *, 4> Args;
       unsigned NumOperands = OldInst.getNumOperands();
       if (isa<CallInst>(OldInst))
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index f58448dd9562d56759f192a80d39d1d0dbeb5a0e..15b26a38cc28ef2be3360f010f15052be4a04966 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -917,6 +917,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::NoFPClass:
       case Attribute::CoroDestroyOnlyWhenComplete:
       case Attribute::CoroElideSafe:
+      case Attribute::NoDivergenceSource:
         continue;
       // Those attributes should be safe to propagate to the extracted function.
       case Attribute::AlwaysInline:
@@ -1124,7 +1125,8 @@ static void insertLifetimeMarkersSurroundingCall(
                                             TheCall->getFunction()) &&
              "Input memory not defined in original function");
 
-      Function *Func = Intrinsic::getDeclaration(M, MarkerFunc, Mem->getType());
+      Function *Func =
+          Intrinsic::getOrInsertDeclaration(M, MarkerFunc, Mem->getType());
       auto Marker = CallInst::Create(Func, {NegativeOne, Mem});
       if (InsertBefore)
         Marker->insertBefore(TheCall);
diff --git a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
index d12c540f9a4d04d93294c5eb2e28164549891829..47bb31905d1ac8da1b9b9526466b81e0495cf659 100644
--- a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
+++ b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
@@ -63,7 +63,7 @@ static void insertCall(Function &CurFn, StringRef Func,
         Func, FunctionType::get(Type::getVoidTy(C), ArgTypes, false));
 
     Instruction *RetAddr = CallInst::Create(
-        Intrinsic::getDeclaration(&M, Intrinsic::returnaddress),
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::returnaddress),
         ArrayRef<Value *>(ConstantInt::get(Type::getInt32Ty(C), 0)), "",
         InsertionPt);
     RetAddr->setDebugLoc(DL);
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 671b0d0822a5d91929bda8f5e46540caf742f49e..110fd6de5c69687a1157c176e08a52cab21d193b 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -2090,7 +2090,7 @@ inlineRetainOrClaimRVCalls(CallBase &CB, objcarc::ARCInstKind RVCallKind,
         if (IsUnsafeClaimRV) {
           Builder.SetInsertPoint(II);
           Function *IFn =
-              Intrinsic::getDeclaration(Mod, Intrinsic::objc_release);
+              Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::objc_release);
           Builder.CreateCall(IFn, RetOpnd, "");
         }
         II->eraseFromParent();
@@ -2125,7 +2125,8 @@ inlineRetainOrClaimRVCalls(CallBase &CB, objcarc::ARCInstKind RVCallKind,
       // matching autoreleaseRV or an annotated call in the callee. Emit a call
       // to objc_retain.
       Builder.SetInsertPoint(RI);
-      Function *IFn = Intrinsic::getDeclaration(Mod, Intrinsic::objc_retain);
+      Function *IFn =
+          Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::objc_retain);
       Builder.CreateCall(IFn, RetOpnd, "");
     }
   }
@@ -3021,7 +3022,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
       });
     } else {
       SmallVector<ReturnInst *, 8> NormalReturns;
-      Function *NewDeoptIntrinsic = Intrinsic::getDeclaration(
+      Function *NewDeoptIntrinsic = Intrinsic::getOrInsertDeclaration(
           Caller->getParent(), Intrinsic::experimental_deoptimize,
           {Caller->getReturnType()});
 
diff --git a/llvm/lib/Transforms/Utils/IntegerDivision.cpp b/llvm/lib/Transforms/Utils/IntegerDivision.cpp
index 11956816a6ec3fe2b3e93b39bddea12bd96bd769..e95a7a9ae525acda95b90017bbd405f67c80e649 100644
--- a/llvm/lib/Transforms/Utils/IntegerDivision.cpp
+++ b/llvm/lib/Transforms/Utils/IntegerDivision.cpp
@@ -157,8 +157,8 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
 
   BasicBlock *IBB = Builder.GetInsertBlock();
   Function *F = IBB->getParent();
-  Function *CTLZ = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
-                                             DivTy);
+  Function *CTLZ =
+      Intrinsic::getOrInsertDeclaration(F->getParent(), Intrinsic::ctlz, DivTy);
 
   // Our CFG is going to look like:
   // +---------------------+
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 5528691cf017150cc4ad9cc5e50752ab25149dd8..d58667600909fd688195a2c66d9a90a95ef93c3d 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -4151,7 +4151,8 @@ bool llvm::recognizeBSwapOrBitReverseIdiom(
   else
     return false;
 
-  Function *F = Intrinsic::getDeclaration(I->getModule(), Intrin, DemandedTy);
+  Function *F =
+      Intrinsic::getOrInsertDeclaration(I->getModule(), Intrin, DemandedTy);
   Value *Provider = Res->Provider;
 
   // We may need to truncate the provider.
diff --git a/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp b/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp
index 55f9400d93d79be170f1aa93f5eec5d14347cc41..cd79600657032e5497311bf4957d6f6fb7957f18 100644
--- a/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp
+++ b/llvm/lib/Transforms/Utils/LowerGlobalDtors.cpp
@@ -215,8 +215,8 @@ static bool runImpl(Module &M) {
       // If `__cxa_atexit` hits out-of-memory, trap, so that we don't misbehave.
       // This should be very rare, because if the process is running out of
       // memory before main has even started, something is wrong.
-      CallInst::Create(Intrinsic::getDeclaration(&M, Intrinsic::trap), "",
-                       FailBB);
+      CallInst::Create(Intrinsic::getOrInsertDeclaration(&M, Intrinsic::trap),
+                       "", FailBB);
       new UnreachableInst(C, FailBB);
 
       ReturnInst::Create(C, RetBB);
diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
index 1cb1a7b396badc579da5edd65be96a970a27ee8f..77abf160dc70f9e349162c62d2c9d83e7393fa48 100644
--- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
+++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
@@ -268,7 +268,7 @@ bool isLifetimeIntrinsic(Value *V) {
 
 Value *readRegister(IRBuilder<> &IRB, StringRef Name) {
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
-  Function *ReadRegister = Intrinsic::getDeclaration(
+  Function *ReadRegister = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::read_register, IRB.getIntPtrTy(M->getDataLayout()));
   MDNode *MD =
       MDNode::get(M->getContext(), {MDString::get(M->getContext(), Name)});
@@ -287,7 +287,7 @@ Value *getPC(const Triple &TargetTriple, IRBuilder<> &IRB) {
 Value *getFP(IRBuilder<> &IRB) {
   Function *F = IRB.GetInsertBlock()->getParent();
   Module *M = F->getParent();
-  auto *GetStackPointerFn = Intrinsic::getDeclaration(
+  auto *GetStackPointerFn = Intrinsic::getOrInsertDeclaration(
       M, Intrinsic::frameaddress,
       IRB.getPtrTy(M->getDataLayout().getAllocaAddrSpace()));
   return IRB.CreatePtrToInt(
@@ -301,7 +301,7 @@ Value *getAndroidSlotPtr(IRBuilder<> &IRB, int Slot) {
   // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
   // in Bionic's libc/private/bionic_tls.h.
   Function *ThreadPointerFunc =
-      Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::thread_pointer);
   return IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
                                 IRB.CreateCall(ThreadPointerFunc), 8 * Slot);
 }
diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index 186e17e166ba3d147afcbc5a5b943d8339945664..2415118cad6fb363f9fdb77de0fa748a7764f295 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -559,7 +559,7 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
     if (isa<PredicateWithEdge>(ValInfo)) {
       IRBuilder<> B(getBranchTerminator(ValInfo));
       auto NumDecls = F.getParent()->getNumNamedValues();
-      Function *IF = Intrinsic::getDeclaration(
+      Function *IF = Intrinsic::getOrInsertDeclaration(
           F.getParent(), Intrinsic::ssa_copy, Op->getType());
       if (NumDecls != F.getParent()->getNumNamedValues())
         PI.CreatedDeclarations.insert(IF);
@@ -575,7 +575,7 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
       // directly before it, assume(i1 true) is not a useful fact.
       IRBuilder<> B(PAssume->AssumeInst->getNextNode());
       auto NumDecls = F.getParent()->getNumNamedValues();
-      Function *IF = Intrinsic::getDeclaration(
+      Function *IF = Intrinsic::getOrInsertDeclaration(
           F.getParent(), Intrinsic::ssa_copy, Op->getType());
       if (NumDecls != F.getParent()->getNumNamedValues())
         PI.CreatedDeclarations.insert(IF);
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index a54e2f776572e792682a3e5925c12cc64969f2b6..db69c839781895176c0f581dd7917f3d290213ce 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -493,7 +493,7 @@ private:
 /// Given a LoadInst LI this adds assume(LI != null) after it.
 static void addAssumeNonNull(AssumptionCache *AC, Instruction *LI) {
   Function *AssumeIntrinsic =
-      Intrinsic::getDeclaration(LI->getModule(), Intrinsic::assume);
+      Intrinsic::getOrInsertDeclaration(LI->getModule(), Intrinsic::assume);
   ICmpInst *LoadNotNull = new ICmpInst(ICmpInst::ICMP_NE, LI,
                                        Constant::getNullValue(LI->getType()));
   LoadNotNull->insertAfter(LI);
diff --git a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
index 6e84965370b24826c07347efbd620de5acf913ac..2700b4307308cb6b6272fd8a6f34e46b47b4c1b8 100644
--- a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
+++ b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
@@ -151,7 +151,7 @@ static void convertToRelLookupTable(GlobalVariable &LookupTable) {
   // GEP might not be immediately followed by a LOAD, like it can be hoisted
   // outside the loop or another instruction might be inserted them in between.
   Builder.SetInsertPoint(Load);
-  Function *LoadRelIntrinsic = llvm::Intrinsic::getDeclaration(
+  Function *LoadRelIntrinsic = llvm::Intrinsic::getOrInsertDeclaration(
       &M, Intrinsic::load_relative, {Index->getType()});
 
   // Create a call to load.relative intrinsic that computes the target address
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 1ff3cd78aa98773328b7d9ac4469c751e93ec3fb..bc619e5098e09ff90b4b8999eb7757824afc44fe 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -49,6 +49,7 @@ PoisonFlags::PoisonFlags(const Instruction *I) {
   Exact = false;
   Disjoint = false;
   NNeg = false;
+  SameSign = false;
   GEPNW = GEPNoWrapFlags::none();
   if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(I)) {
     NUW = OBO->hasNoUnsignedWrap();
@@ -66,6 +67,8 @@ PoisonFlags::PoisonFlags(const Instruction *I) {
   }
   if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
     GEPNW = GEP->getNoWrapFlags();
+  if (auto *ICmp = dyn_cast<ICmpInst>(I))
+    SameSign = ICmp->hasSameSign();
 }
 
 void PoisonFlags::apply(Instruction *I) {
@@ -85,6 +88,8 @@ void PoisonFlags::apply(Instruction *I) {
   }
   if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
     GEP->setNoWrapFlags(GEPNW);
+  if (auto *ICmp = dyn_cast<ICmpInst>(I))
+    ICmp->setSameSign(SameSign);
 }
 
 /// ReuseOrCreateCast - Arrange for there to be a cast of V to Ty at IP,
@@ -2134,8 +2139,8 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
       MulV = TruncTripCount;
       OfMul = ConstantInt::getFalse(MulV->getContext());
     } else {
-      auto *MulF = Intrinsic::getDeclaration(Loc->getModule(),
-                                             Intrinsic::umul_with_overflow, Ty);
+      auto *MulF = Intrinsic::getOrInsertDeclaration(
+          Loc->getModule(), Intrinsic::umul_with_overflow, Ty);
       CallInst *Mul =
           Builder.CreateCall(MulF, {AbsStep, TruncTripCount}, "mul");
       MulV = Builder.CreateExtractValue(Mul, 0, "mul.result");
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index e06ebb691d511cc4b5cfbac9c670723570dcc71f..db2acb9eed0938eaacf9673a3420d386b05a18b1 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1960,7 +1960,7 @@ static Value *optimizeDoubleFP(CallInst *CI, IRBuilderBase &B,
   if (IsIntrinsic) {
     Module *M = CI->getModule();
     Intrinsic::ID IID = CalleeFn->getIntrinsicID();
-    Function *Fn = Intrinsic::getDeclaration(M, IID, B.getFloatTy());
+    Function *Fn = Intrinsic::getOrInsertDeclaration(M, IID, B.getFloatTy());
     R = isBinary ? B.CreateCall(Fn, V) : B.CreateCall(Fn, V[0]);
   } else {
     AttributeList CalleeAttrs = CalleeFn->getAttributes();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index cf8791a354549b96867d9b6d7271275fc7ece08f..54841914ff696ce1af479b8786d9a5753b5cd134 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -430,10 +430,10 @@ static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
 ///   3) Returns upper bound estimate if known, and if \p CanUseConstantMax.
 ///   4) Returns std::nullopt if all of the above failed.
 static std::optional<unsigned>
-getSmallBestKnownTC(ScalarEvolution &SE, Loop *L,
+getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L,
                     bool CanUseConstantMax = true) {
   // Check if exact trip count is known.
-  if (unsigned ExpectedTC = SE.getSmallConstantTripCount(L))
+  if (unsigned ExpectedTC = PSE.getSE()->getSmallConstantTripCount(L))
     return ExpectedTC;
 
   // Check if there is an expected trip count available from profile data.
@@ -445,7 +445,7 @@ getSmallBestKnownTC(ScalarEvolution &SE, Loop *L,
     return std::nullopt;
 
   // Check if upper bound estimate is known.
-  if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
+  if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
     return ExpectedTC;
 
   return std::nullopt;
@@ -573,9 +573,6 @@ public:
 protected:
   friend class LoopVectorizationPlanner;
 
-  /// A small list of PHINodes.
-  using PhiVector = SmallVector<PHINode *, 4>;
-
   struct VPIntrinsicAndKind {
     unsigned Intr;
     bool IsFP;
@@ -1900,12 +1897,15 @@ class GeneratedRTChecks {
     return SE.getTripCountFromExitCount(BackedgeTakenCount, IdxTy, OrigLoop);
   }
 
+  PredicatedScalarEvolution &PSE;
+
 public:
-  GeneratedRTChecks(ScalarEvolution &SE, DominatorTree *DT, LoopInfo *LI,
-                    TargetTransformInfo *TTI, const DataLayout &DL,
-                    bool AddBranchWeights)
-      : DT(DT), LI(LI), TTI(TTI), SCEVExp(SE, DL, "scev.check"),
-        MemCheckExp(SE, DL, "scev.check"), AddBranchWeights(AddBranchWeights) {}
+  GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
+                    LoopInfo *LI, TargetTransformInfo *TTI,
+                    const DataLayout &DL, bool AddBranchWeights)
+      : DT(DT), LI(LI), TTI(TTI), SCEVExp(*PSE.getSE(), DL, "scev.check"),
+        MemCheckExp(*PSE.getSE(), DL, "scev.check"),
+        AddBranchWeights(AddBranchWeights), PSE(PSE) {}
 
 
   /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
@@ -2090,7 +2090,7 @@ public:
 
           // Get the best known TC estimate.
           if (auto EstimatedTC = getSmallBestKnownTC(
-                  *SE, OuterLoop, /* CanUseConstantMax = */ false))
+                  PSE, OuterLoop, /* CanUseConstantMax = */ false))
             BestTripCount = *EstimatedTC;
 
           BestTripCount = std::max(BestTripCount, 1U);
@@ -2413,8 +2413,7 @@ static bool isIndvarOverflowCheckKnownFalse(
   // We know the runtime overflow check is known false iff the (max) trip-count
   // is known and (max) trip-count + (VF * UF) does not overflow in the type of
   // the vector loop induction variable.
-  if (unsigned TC =
-          Cost->PSE.getSE()->getSmallConstantMaxTripCount(Cost->TheLoop)) {
+  if (unsigned TC = Cost->PSE.getSmallConstantMaxTripCount()) {
     uint64_t MaxVF = VF.getKnownMinValue();
     if (VF.isScalable()) {
       std::optional<unsigned> MaxVScale =
@@ -4480,8 +4479,10 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   }
 
   unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
-  unsigned MaxTC = PSE.getSE()->getSmallConstantMaxTripCount(TheLoop);
+  unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
   LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
+  if (TC != MaxTC)
+    LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
   if (TC == 1) {
     reportVectorizationFailure("Single iteration (non) loop",
         "loop trip count is one, irrelevant for vectorization",
@@ -4787,7 +4788,7 @@ bool LoopVectorizationPlanner::isMoreProfitable(
   InstructionCost CostA = A.Cost;
   InstructionCost CostB = B.Cost;
 
-  unsigned MaxTripCount = PSE.getSE()->getSmallConstantMaxTripCount(OrigLoop);
+  unsigned MaxTripCount = PSE.getSmallConstantMaxTripCount();
 
   // Improve estimate for the vector width if it is scalable.
   unsigned EstimatedWidthA = A.Width.getKnownMinValue();
@@ -5422,7 +5423,7 @@ LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
   if (!Legal->isSafeForAnyVectorWidth())
     return 1;
 
-  auto BestKnownTC = getSmallBestKnownTC(*PSE.getSE(), TheLoop);
+  auto BestKnownTC = getSmallBestKnownTC(PSE, TheLoop);
   const bool HasReductions = !Legal->getReductionVars().empty();
 
   // If we did not calculate the cost for VF (because the user selected the VF)
@@ -7132,12 +7133,32 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
     // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
     // converted into select instructions. We require N - 1 selects per phi
     // node, where N is the number of incoming values.
-    if (VF.isVector() && Phi->getParent() != TheLoop->getHeader())
+    if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
+      Type *ResultTy = Phi->getType();
+
+      // All instructions in an Any-of reduction chain are narrowed to bool.
+      // Check if that is the case for this phi node.
+      auto *HeaderUser = cast_if_present<PHINode>(
+          find_singleton<User>(Phi->users(), [this](User *U, bool) -> User * {
+            auto *Phi = dyn_cast<PHINode>(U);
+            if (Phi && Phi->getParent() == TheLoop->getHeader())
+              return Phi;
+            return nullptr;
+          }));
+      if (HeaderUser) {
+        auto &ReductionVars = Legal->getReductionVars();
+        auto Iter = ReductionVars.find(HeaderUser);
+        if (Iter != ReductionVars.end() &&
+            RecurrenceDescriptor::isAnyOfRecurrenceKind(
+                Iter->second.getRecurrenceKind()))
+          ResultTy = Type::getInt1Ty(Phi->getContext());
+      }
       return (Phi->getNumIncomingValues() - 1) *
              TTI.getCmpSelInstrCost(
-                 Instruction::Select, ToVectorTy(Phi->getType(), VF),
+                 Instruction::Select, ToVectorTy(ResultTy, VF),
                  ToVectorTy(Type::getInt1Ty(Phi->getContext()), VF),
                  CmpInst::BAD_ICMP_PREDICATE, CostKind);
+    }
 
     return TTI.getCFInstrCost(Instruction::PHI, CostKind);
   }
@@ -10878,7 +10899,7 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
 static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
                                        VectorizationFactor &VF,
                                        std::optional<unsigned> VScale, Loop *L,
-                                       ScalarEvolution &SE,
+                                       PredicatedScalarEvolution &SE,
                                        ScalarEpilogueLowering SEL,
                                        bool AlwaysGenerateRuntimeChecks);
 
@@ -10986,7 +11007,7 @@ static bool processLoopInVPlanNativePath(
   {
     bool AddBranchWeights =
         hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
-    GeneratedRTChecks Checks(*PSE.getSE(), DT, LI, TTI,
+    GeneratedRTChecks Checks(PSE, DT, LI, TTI,
                              F->getDataLayout(), AddBranchWeights);
     if (VF.Width.isVector() || UF > 1)
       // NOTE: only creating checks for innermost loop vectorization, for now.
@@ -11002,7 +11023,7 @@ static bool processLoopInVPlanNativePath(
                                           PreferPredicatedWidening);
         if (!ForceVectorization &&
             !areRuntimeChecksProfitable(Checks, VF, getVScaleForTuning(L, *TTI),
-                                        L, *PSE.getSE(), SEL,
+                                        L, PSE, SEL,
                                         UserDefinedVFOrUF)) {
           ORE->emit([&]() {
             return OptimizationRemarkAnalysisAliasing(
@@ -11081,7 +11102,7 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
 static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
                                        VectorizationFactor &VF,
                                        std::optional<unsigned> VScale, Loop *L,
-                                       ScalarEvolution &SE,
+                                       PredicatedScalarEvolution &PSE,
                                        ScalarEpilogueLowering SEL,
                                        bool AlwaysGenerateRuntimeChecks) {
   InstructionCost CheckCost = Checks.getCost();
@@ -11184,7 +11205,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
 
   // Skip vectorization if the expected trip count is less than the minimum
   // required trip count.
-  if (auto ExpectedTC = getSmallBestKnownTC(SE, L)) {
+  if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
     if (ElementCount::isKnownLT(ElementCount::getFixed(*ExpectedTC),
                                 VF.MinProfitableTripCount)) {
       LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
@@ -11294,7 +11315,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
 
   // Check the loop for a trip count threshold: vectorize loops with a tiny trip
   // count by optimizing for size, to minimize overheads.
-  auto ExpectedTC = getSmallBestKnownTC(*SE, L);
+  auto ExpectedTC = getSmallBestKnownTC(PSE, L);
   if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) {
     LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
                       << "This loop is worth vectorizing only if no scalar "
@@ -11397,7 +11418,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
 
   bool AddBranchWeights =
       hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
-  GeneratedRTChecks Checks(*PSE.getSE(), DT, LI, TTI,
+  GeneratedRTChecks Checks(PSE, DT, LI, TTI,
                            F->getDataLayout(), AddBranchWeights);
   if (LVP.hasPlanWithVF(VF.Width)
       && !(VF.Width.isScalar() && Hints.isFixedVectorizationDisabled())) {
@@ -11416,7 +11437,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         Hints.getForce() == LoopVectorizeHints::FK_Enabled;
     if (!ForceVectorization &&
         !areRuntimeChecksProfitable(Checks, VF, getVScaleForTuning(L, *TTI), L,
-                                    *PSE.getSE(), SEL, UserDefinedVFOrIC)) {
+                                    PSE, SEL, UserDefinedVFOrIC)) {
       ORE->emit([&]() {
         return OptimizationRemarkAnalysisAliasing(
                    DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a721ed531cf8dc45ade033a03a2dde0d9bf24cea..7ee5942969100e2847ea3874853b09330da7e12f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2922,7 +2922,7 @@ private:
 
   /// This is the recursive part of buildTree.
   void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth,
-                     const EdgeInfo &EI);
+                     const EdgeInfo &EI, unsigned InterleaveFactor = 0);
 
   /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
   /// be vectorized to use the original vector (or aggregate "bitcast" to a
@@ -3226,7 +3226,15 @@ private:
     Instruction *MainOp = nullptr;
     Instruction *AltOp = nullptr;
 
+    /// Interleaving factor for interleaved loads Vectorize nodes.
+    unsigned InterleaveFactor = 0;
+
   public:
+    /// Returns interleave factor for interleave nodes.
+    unsigned getInterleaveFactor() const { return InterleaveFactor; }
+    /// Sets interleaving factor for the interleaving nodes.
+    void setInterleave(unsigned Factor) { InterleaveFactor = Factor; }
+
     /// Set this bundle's \p OpIdx'th operand to \p OpVL.
     void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
       if (Operands.size() < OpIdx + 1)
@@ -3390,7 +3398,12 @@ private:
       dbgs() << "State: ";
       switch (State) {
       case Vectorize:
-        dbgs() << "Vectorize\n";
+        if (InterleaveFactor > 0) {
+          dbgs() << "Vectorize with interleave factor " << InterleaveFactor
+                 << "\n";
+        } else {
+          dbgs() << "Vectorize\n";
+        }
         break;
       case ScatterVectorize:
         dbgs() << "ScatterVectorize\n";
@@ -3460,11 +3473,15 @@ private:
                           const InstructionsState &S,
                           const EdgeInfo &UserTreeIdx,
                           ArrayRef<int> ReuseShuffleIndices = {},
-                          ArrayRef<unsigned> ReorderIndices = {}) {
+                          ArrayRef<unsigned> ReorderIndices = {},
+                          unsigned InterleaveFactor = 0) {
     TreeEntry::EntryState EntryState =
         Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
-    return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
-                        ReuseShuffleIndices, ReorderIndices);
+    TreeEntry *E = newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
+                                ReuseShuffleIndices, ReorderIndices);
+    if (E && InterleaveFactor > 0)
+      E->setInterleave(InterleaveFactor);
+    return E;
   }
 
   TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
@@ -6849,7 +6866,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
     return Results;
   };
   auto ProcessGatheredLoads =
-      [&](ArrayRef<SmallVector<std::pair<LoadInst *, int>>> GatheredLoads,
+      [&, &TTI = *TTI](
+          ArrayRef<SmallVector<std::pair<LoadInst *, int>>> GatheredLoads,
           bool Final = false) {
         SmallVector<LoadInst *> NonVectorized;
         for (ArrayRef<std::pair<LoadInst *, int>> LoadsDists : GatheredLoads) {
@@ -6932,11 +6950,16 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
             // distance between scalar loads in these nodes.
             unsigned MaxVF = Slice.size();
             unsigned UserMaxVF = 0;
+            unsigned InterleaveFactor = 0;
             if (MaxVF == 2) {
               UserMaxVF = MaxVF;
             } else {
+              // Found distance between segments of the interleaved loads.
+              std::optional<unsigned> InterleavedLoadsDistance = 0;
+              unsigned Order = 0;
               std::optional<unsigned> CommonVF = 0;
               DenseMap<const TreeEntry *, unsigned> EntryToPosition;
+              SmallPtrSet<const TreeEntry *, 8> DeinterleavedNodes;
               for (auto [Idx, V] : enumerate(Slice)) {
                 for (const TreeEntry *E : ValueToGatherNodes.at(V)) {
                   UserMaxVF = std::max<unsigned>(UserMaxVF, E->Scalars.size());
@@ -6951,12 +6974,59 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
                     if (*CommonVF != E->Scalars.size())
                       CommonVF.reset();
                   }
+                  // Check if the load is the part of the interleaved load.
+                  if (Pos != Idx && InterleavedLoadsDistance) {
+                    if (!DeinterleavedNodes.contains(E) &&
+                        any_of(E->Scalars, [&, Slice = Slice](Value *V) {
+                          if (isa<Constant>(V))
+                            return false;
+                          if (getTreeEntry(V))
+                            return true;
+                          const auto &Nodes = ValueToGatherNodes.at(V);
+                          return (Nodes.size() != 1 || !Nodes.contains(E)) &&
+                                 !is_contained(Slice, V);
+                        })) {
+                      InterleavedLoadsDistance.reset();
+                      continue;
+                    }
+                    DeinterleavedNodes.insert(E);
+                    if (*InterleavedLoadsDistance == 0) {
+                      InterleavedLoadsDistance = Idx - Pos;
+                      continue;
+                    }
+                    if ((Idx - Pos) % *InterleavedLoadsDistance != 0 ||
+                        (Idx - Pos) / *InterleavedLoadsDistance < Order)
+                      InterleavedLoadsDistance.reset();
+                    Order = (Idx - Pos) / InterleavedLoadsDistance.value_or(1);
+                  }
+                }
+              }
+              DeinterleavedNodes.clear();
+              // Check if the large load represents interleaved load operation.
+              if (InterleavedLoadsDistance.value_or(0) > 1 &&
+                  CommonVF.value_or(0) != 0) {
+                InterleaveFactor = bit_ceil(*InterleavedLoadsDistance);
+                unsigned VF = *CommonVF;
+                OrdersType Order;
+                SmallVector<Value *> PointerOps;
+                // Segmented load detected - vectorize at maximum vector factor.
+                if (TTI.isLegalInterleavedAccessType(
+                        getWidenedType(Slice.front()->getType(), VF),
+                        InterleaveFactor,
+                        cast<LoadInst>(Slice.front())->getAlign(),
+                        cast<LoadInst>(Slice.front())
+                            ->getPointerAddressSpace()) &&
+                    canVectorizeLoads(Slice, Slice.front(), Order,
+                                      PointerOps) == LoadsState::Vectorize) {
+                  UserMaxVF = InterleaveFactor * VF;
+                } else {
+                  InterleaveFactor = 0;
                 }
               }
               // Cannot represent the loads as consecutive vectorizable nodes -
               // just exit.
               unsigned ConsecutiveNodesSize = 0;
-              if (!LoadEntriesToVectorize.empty() &&
+              if (!LoadEntriesToVectorize.empty() && InterleaveFactor == 0 &&
                   any_of(zip(LoadEntriesToVectorize, LoadSetsToVectorize),
                          [&, Slice = Slice](const auto &P) {
                            const auto *It = find_if(Slice, [&](Value *V) {
@@ -6976,7 +7046,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
                 continue;
               // Try to build long masked gather loads.
               UserMaxVF = bit_ceil(UserMaxVF);
-              if (any_of(seq<unsigned>(Slice.size() / UserMaxVF),
+              if (InterleaveFactor == 0 &&
+                  any_of(seq<unsigned>(Slice.size() / UserMaxVF),
                          [&, Slice = Slice](unsigned Idx) {
                            OrdersType Order;
                            SmallVector<Value *> PointerOps;
@@ -7008,9 +7079,15 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
                            }))
                   continue;
                 unsigned Sz = VectorizableTree.size();
-                buildTree_rec(SubSlice, 0, EdgeInfo());
+                buildTree_rec(SubSlice, 0, EdgeInfo(), InterleaveFactor);
                 if (Sz == VectorizableTree.size()) {
                   IsVectorized = false;
+                  // Try non-interleaved vectorization with smaller vector
+                  // factor.
+                  if (InterleaveFactor > 0) {
+                    VF = 2 * (MaxVF / InterleaveFactor);
+                    InterleaveFactor = 0;
+                  }
                   continue;
                 }
               }
@@ -7374,6 +7451,11 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
       }
       return TreeEntry::ScatterVectorize;
     case LoadsState::StridedVectorize:
+      if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
+        // Delay slow vectorized nodes for better vectorization attempts.
+        LoadEntriesToVectorize.insert(VectorizableTree.size());
+        return TreeEntry::NeedToGather;
+      }
       return TreeEntry::StridedVectorize;
     case LoadsState::Gather:
 #ifndef NDEBUG
@@ -7707,7 +7789,8 @@ public:
 } // namespace
 
 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
-                            const EdgeInfo &UserTreeIdx) {
+                            const EdgeInfo &UserTreeIdx,
+                            unsigned InterleaveFactor) {
   assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
 
   SmallVector<int> ReuseShuffleIndices;
@@ -8185,7 +8268,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       switch (State) {
       case TreeEntry::Vectorize:
         TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                          ReuseShuffleIndices, CurrentOrder);
+                          ReuseShuffleIndices, CurrentOrder, InterleaveFactor);
         if (CurrentOrder.empty())
           LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
         else
@@ -9897,6 +9980,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
             Idx = EMask[Idx];
         }
         CommonVF = E->Scalars.size();
+      } else if (std::optional<unsigned> Factor = E->getInterleaveFactor();
+                 Factor && E->Scalars.size() != Mask.size() &&
+                 ShuffleVectorInst::isDeInterleaveMaskOfFactor(CommonMask,
+                                                               *Factor)) {
+        // Deinterleaved nodes are free.
+        std::iota(CommonMask.begin(), CommonMask.end(), 0);
       }
       ExtraCost += GetNodeMinBWAffectedCost(*E, CommonVF);
       V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
@@ -10970,23 +11059,38 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
     auto *LI0 = cast<LoadInst>(VL0);
     auto GetVectorCost = [&](InstructionCost CommonCost) {
       InstructionCost VecLdCost;
-      if (E->State == TreeEntry::Vectorize) {
-        VecLdCost = TTI->getMemoryOpCost(
-            Instruction::Load, VecTy, LI0->getAlign(),
-            LI0->getPointerAddressSpace(), CostKind, TTI::OperandValueInfo());
-      } else if (E->State == TreeEntry::StridedVectorize) {
+      switch (E->State) {
+      case TreeEntry::Vectorize:
+        if (unsigned Factor = E->getInterleaveFactor()) {
+          VecLdCost = TTI->getInterleavedMemoryOpCost(
+              Instruction::Load, VecTy, Factor, std::nullopt, LI0->getAlign(),
+              LI0->getPointerAddressSpace(), CostKind);
+
+        } else {
+          VecLdCost = TTI->getMemoryOpCost(
+              Instruction::Load, VecTy, LI0->getAlign(),
+              LI0->getPointerAddressSpace(), CostKind, TTI::OperandValueInfo());
+        }
+        break;
+      case TreeEntry::StridedVectorize: {
         Align CommonAlignment =
             computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
         VecLdCost = TTI->getStridedMemoryOpCost(
             Instruction::Load, VecTy, LI0->getPointerOperand(),
             /*VariableMask=*/false, CommonAlignment, CostKind);
-      } else {
-        assert(E->State == TreeEntry::ScatterVectorize && "Unknown EntryState");
+        break;
+      }
+      case TreeEntry::ScatterVectorize: {
         Align CommonAlignment =
             computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
         VecLdCost = TTI->getGatherScatterOpCost(
             Instruction::Load, VecTy, LI0->getPointerOperand(),
             /*VariableMask=*/false, CommonAlignment, CostKind);
+        break;
+      }
+      case TreeEntry::CombinedVectorize:
+      case TreeEntry::NeedToGather:
+        llvm_unreachable("Unexpected vectorization state.");
       }
       return VecLdCost + CommonCost;
     };
@@ -11399,6 +11503,11 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
       }))
     return false;
 
+  if (VectorizableTree.back()->isGather() &&
+      VectorizableTree.back()->isAltShuffle() &&
+      VectorizableTree.back()->getVectorFactor() > 2)
+    return false;
+
   assert(VectorizableTree.empty()
              ? ExternalUses.empty()
              : true && "We shouldn't have any external users");
@@ -11746,12 +11855,15 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
   for (ExternalUser &EU : ExternalUses) {
     // Uses by ephemeral values are free (because the ephemeral value will be
     // removed prior to code generation, and so the extraction will be
-    // removed as well) as well as uses in unreachable blocks or in landing pads
-    // (rarely executed).
-    if (EphValues.count(EU.User) ||
-        (EU.User &&
-         (!DT->isReachableFromEntry(cast<Instruction>(EU.User)->getParent()) ||
-          cast<Instruction>(EU.User)->getParent()->isLandingPad())))
+    // removed as well).
+    if (EphValues.count(EU.User))
+      continue;
+
+    // Used in unreachable blocks or in landing pads (rarely executed).
+    if (BasicBlock *UserParent =
+            EU.User ? cast<Instruction>(EU.User)->getParent() : nullptr;
+        UserParent &&
+        (!DT->isReachableFromEntry(UserParent) || UserParent->isLandingPad()))
       continue;
 
     // We only add extract cost once for the same scalar.
@@ -15081,7 +15193,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
                          false /*HasGlobalPred*/);
         CF = VFDatabase(*CI).getVectorizedFunction(Shape);
       } else {
-        CF = Intrinsic::getDeclaration(F->getParent(), ID, TysForDecl);
+        CF = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, TysForDecl);
       }
 
       SmallVector<OperandBundleDef, 1> OpBundles;
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
index 0cd2240e7ff1b32e98e6a2c1bbed7b10652e13f7..9bbeca4fc1549464b0893e5c6320dbabc46a0b1d 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
@@ -59,21 +59,17 @@ bool PredIterator::operator==(const PredIterator &Other) const {
 }
 
 #ifndef NDEBUG
-void DGNode::print(raw_ostream &OS, bool PrintDeps) const { I->dumpOS(OS); }
-void DGNode::dump() const {
-  print(dbgs());
-  dbgs() << "\n";
+void DGNode::print(raw_ostream &OS, bool PrintDeps) const {
+  OS << *I << " USuccs:" << UnscheduledSuccs << "\n";
 }
+void DGNode::dump() const { print(dbgs()); }
 void MemDGNode::print(raw_ostream &OS, bool PrintDeps) const {
-  I->dumpOS(OS);
+  DGNode::print(OS, false);
   if (PrintDeps) {
     // Print memory preds.
     static constexpr const unsigned Indent = 4;
-    for (auto *Pred : MemPreds) {
-      OS.indent(Indent) << "<-";
-      Pred->print(OS, false);
-      OS << "\n";
-    }
+    for (auto *Pred : MemPreds)
+      OS.indent(Indent) << "<-" << *Pred->getInstruction() << "\n";
   }
 }
 #endif // NDEBUG
@@ -215,17 +211,63 @@ void DependencyGraph::scanAndAddDeps(MemDGNode &DstN,
   }
 }
 
-Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
-  if (Instrs.empty())
-    return {};
+void DependencyGraph::setDefUseUnscheduledSuccs(
+    const Interval<Instruction> &NewInterval) {
+  // +---+
+  // |   |  Def
+  // |   |   |
+  // |   |   v
+  // |   |  Use
+  // +---+
+  // Set the intra-interval counters in NewInterval.
+  for (Instruction &I : NewInterval) {
+    for (Value *Op : I.operands()) {
+      auto *OpI = dyn_cast<Instruction>(Op);
+      if (OpI == nullptr)
+        continue;
+      if (!NewInterval.contains(OpI))
+        continue;
+      auto *OpN = getNode(OpI);
+      if (OpN == nullptr)
+        continue;
+      ++OpN->UnscheduledSuccs;
+    }
+  }
 
-  Interval<Instruction> InstrInterval(Instrs);
+  // Now handle the cross-interval edges.
+  bool NewIsAbove = DAGInterval.empty() || NewInterval.comesBefore(DAGInterval);
+  const auto &TopInterval = NewIsAbove ? NewInterval : DAGInterval;
+  const auto &BotInterval = NewIsAbove ? DAGInterval : NewInterval;
+  // +---+
+  // |Top|
+  // |   |  Def
+  // +---+   |
+  // |   |   v
+  // |Bot|  Use
+  // |   |
+  // +---+
+  // Walk over all instructions in "BotInterval" and update the counter
+  // of operands that are in "TopInterval".
+  for (Instruction &BotI : BotInterval) {
+    for (Value *Op : BotI.operands()) {
+      auto *OpI = dyn_cast<Instruction>(Op);
+      if (OpI == nullptr)
+        continue;
+      if (!TopInterval.contains(OpI))
+        continue;
+      auto *OpN = getNode(OpI);
+      if (OpN == nullptr)
+        continue;
+      ++OpN->UnscheduledSuccs;
+    }
+  }
+}
 
-  DGNode *LastN = getOrCreateNode(InstrInterval.top());
-  // Create DGNodes for all instrs in Interval to avoid future Instruction to
-  // DGNode lookups.
+void DependencyGraph::createNewNodes(const Interval<Instruction> &NewInterval) {
+  // Create Nodes only for the new sections of the DAG.
+  DGNode *LastN = getOrCreateNode(NewInterval.top());
   MemDGNode *LastMemN = dyn_cast<MemDGNode>(LastN);
-  for (Instruction &I : drop_begin(InstrInterval)) {
+  for (Instruction &I : drop_begin(NewInterval)) {
     auto *N = getOrCreateNode(&I);
     // Build the Mem node chain.
     if (auto *MemN = dyn_cast<MemDGNode>(N)) {
@@ -235,16 +277,144 @@ Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
       LastMemN = MemN;
     }
   }
+  // Link new MemDGNode chain with the old one, if any.
+  if (!DAGInterval.empty()) {
+    bool NewIsAbove = NewInterval.comesBefore(DAGInterval);
+    const auto &TopInterval = NewIsAbove ? NewInterval : DAGInterval;
+    const auto &BotInterval = NewIsAbove ? DAGInterval : NewInterval;
+    MemDGNode *LinkTopN =
+        MemDGNodeIntervalBuilder::getBotMemDGNode(TopInterval, *this);
+    MemDGNode *LinkBotN =
+        MemDGNodeIntervalBuilder::getTopMemDGNode(BotInterval, *this);
+    assert(LinkTopN->comesBefore(LinkBotN) && "Wrong order!");
+    if (LinkTopN != nullptr && LinkBotN != nullptr) {
+      LinkTopN->setNextNode(LinkBotN);
+      LinkBotN->setPrevNode(LinkTopN);
+    }
+#ifndef NDEBUG
+    // TODO: Remove this once we've done enough testing.
+    // Check that the chain is well formed.
+    auto UnionIntvl = DAGInterval.getUnionInterval(NewInterval);
+    MemDGNode *ChainTopN =
+        MemDGNodeIntervalBuilder::getTopMemDGNode(UnionIntvl, *this);
+    MemDGNode *ChainBotN =
+        MemDGNodeIntervalBuilder::getBotMemDGNode(UnionIntvl, *this);
+    if (ChainTopN != nullptr && ChainBotN != nullptr) {
+      for (auto *N = ChainTopN->getNextNode(), *LastN = ChainTopN; N != nullptr;
+           LastN = N, N = N->getNextNode()) {
+        assert(N == LastN->getNextNode() && "Bad chain!");
+        assert(N->getPrevNode() == LastN && "Bad chain!");
+      }
+    }
+#endif // NDEBUG
+  }
+
+  setDefUseUnscheduledSuccs(NewInterval);
+}
+
+Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
+  if (Instrs.empty())
+    return {};
+
+  Interval<Instruction> InstrsInterval(Instrs);
+  Interval<Instruction> Union = DAGInterval.getUnionInterval(InstrsInterval);
+  auto NewInterval = Union.getSingleDiff(DAGInterval);
+  if (NewInterval.empty())
+    return {};
+
+  createNewNodes(NewInterval);
+
   // Create the dependencies.
-  auto DstRange = MemDGNodeIntervalBuilder::make(InstrInterval, *this);
-  if (!DstRange.empty()) {
-    for (MemDGNode &DstN : drop_begin(DstRange)) {
-      auto SrcRange = Interval<MemDGNode>(DstRange.top(), DstN.getPrevNode());
+  //
+  // 1. This is a new DAG, DAGInterval is empty. Fully scan the whole interval.
+  // +---+       -             -
+  // |   | SrcN  |             |
+  // |   |  |    | SrcRange    |
+  // |New|  v    |             | DstRange
+  // |   | DstN  -             |
+  // |   |                     |
+  // +---+                     -
+  // We are scanning for deps with destination in NewInterval and sources in
+  // NewInterval until DstN, for each DstN.
+  auto FullScan = [this](const Interval<Instruction> Intvl) {
+    auto DstRange = MemDGNodeIntervalBuilder::make(Intvl, *this);
+    if (!DstRange.empty()) {
+      for (MemDGNode &DstN : drop_begin(DstRange)) {
+        auto SrcRange = Interval<MemDGNode>(DstRange.top(), DstN.getPrevNode());
+        scanAndAddDeps(DstN, SrcRange);
+      }
+    }
+  };
+  if (DAGInterval.empty()) {
+    assert(NewInterval == InstrsInterval && "Expected empty DAGInterval!");
+    FullScan(NewInterval);
+  }
+  // 2. The new section is below the old section.
+  // +---+       -
+  // |   |       |
+  // |Old| SrcN  |
+  // |   |  |    |
+  // +---+  |    | SrcRange
+  // +---+  |    |             -
+  // |   |  |    |             |
+  // |New|  v    |             | DstRange
+  // |   | DstN  -             |
+  // |   |                     |
+  // +---+                     -
+  // We are scanning for deps with destination in NewInterval because the deps
+  // in DAGInterval have already been computed. We consider sources in the whole
+  // range including both NewInterval and DAGInterval until DstN, for each DstN.
+  else if (DAGInterval.bottom()->comesBefore(NewInterval.top())) {
+    auto DstRange = MemDGNodeIntervalBuilder::make(NewInterval, *this);
+    auto SrcRangeFull = MemDGNodeIntervalBuilder::make(
+        DAGInterval.getUnionInterval(NewInterval), *this);
+    for (MemDGNode &DstN : DstRange) {
+      auto SrcRange =
+          Interval<MemDGNode>(SrcRangeFull.top(), DstN.getPrevNode());
       scanAndAddDeps(DstN, SrcRange);
     }
   }
+  // 3. The new section is above the old section.
+  else if (NewInterval.bottom()->comesBefore(DAGInterval.top())) {
+    // +---+       -             -
+    // |   | SrcN  |             |
+    // |New|  |    | SrcRange    | DstRange
+    // |   |  v    |             |
+    // |   | DstN  -             |
+    // |   |                     |
+    // +---+                     -
+    // +---+
+    // |Old|
+    // |   |
+    // +---+
+    // When scanning for deps with destination in NewInterval we need to fully
+    // scan the interval. This is the same as the scanning for a new DAG.
+    FullScan(NewInterval);
+
+    // +---+       -
+    // |   |       |
+    // |New| SrcN  | SrcRange
+    // |   |  |    |
+    // |   |  |    |
+    // |   |  |    |
+    // +---+  |    -
+    // +---+  |                  -
+    // |Old|  v                  | DstRange
+    // |   | DstN                |
+    // +---+                     -
+    // When scanning for deps with destination in DAGInterval we need to
+    // consider sources from the NewInterval only, because all intra-DAGInterval
+    // dependencies have already been created.
+    auto DstRangeOld = MemDGNodeIntervalBuilder::make(DAGInterval, *this);
+    auto SrcRange = MemDGNodeIntervalBuilder::make(NewInterval, *this);
+    for (MemDGNode &DstN : DstRangeOld)
+      scanAndAddDeps(DstN, SrcRange);
+  } else {
+    llvm_unreachable("We don't expect extending in both directions!");
+  }
 
-  return InstrInterval;
+  DAGInterval = Union;
+  return NewInterval;
 }
 
 #ifndef NDEBUG
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
index 00a7dc3fcec93e893ef0064039f61f7579b2e5c0..20df9e344b61cda8dc7d5068ed9552dea81282c2 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
@@ -19,6 +19,10 @@
 using namespace llvm;
 namespace llvm::sandboxir {
 
+cl::opt<unsigned> SeedBundleSizeLimit(
+    "sbvec-seed-bundle-size-limit", cl::init(32), cl::Hidden,
+    cl::desc("Limit the size of the seed bundle to cap compilation time."));
+
 MutableArrayRef<Instruction *> SeedBundle::getSlice(unsigned StartIdx,
                                                     unsigned MaxVecRegBits,
                                                     bool ForcePowerOf2) {
@@ -61,4 +65,68 @@ MutableArrayRef<Instruction *> SeedBundle::getSlice(unsigned StartIdx,
     return {};
 }
 
+template <typename LoadOrStoreT>
+SeedContainer::KeyT SeedContainer::getKey(LoadOrStoreT *LSI) const {
+  assert((isa<LoadInst>(LSI) || isa<StoreInst>(LSI)) &&
+         "Expected Load or Store!");
+  Value *Ptr = Utils::getMemInstructionBase(LSI);
+  Instruction::Opcode Op = LSI->getOpcode();
+  Type *Ty = Utils::getExpectedType(LSI);
+  if (auto *VTy = dyn_cast<VectorType>(Ty))
+    Ty = VTy->getElementType();
+  return {Ptr, Ty, Op};
+}
+
+// Explicit instantiations
+template SeedContainer::KeyT
+SeedContainer::getKey<LoadInst>(LoadInst *LSI) const;
+template SeedContainer::KeyT
+SeedContainer::getKey<StoreInst>(StoreInst *LSI) const;
+
+bool SeedContainer::erase(Instruction *I) {
+  assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Expected Load or Store!");
+  auto It = SeedLookupMap.find(I);
+  if (It == SeedLookupMap.end())
+    return false;
+  SeedBundle *Bndl = It->second;
+  Bndl->setUsed(I);
+  return true;
+}
+
+template <typename LoadOrStoreT> void SeedContainer::insert(LoadOrStoreT *LSI) {
+  // Find the bundle containing seeds for this symbol and type-of-access.
+  auto &BundleVec = Bundles[getKey(LSI)];
+  // Fill this vector of bundles front to back so that only the last bundle in
+  // the vector may have available space. This avoids iteration to find one with
+  // space.
+  if (BundleVec.empty() || BundleVec.back()->size() == SeedBundleSizeLimit)
+    BundleVec.emplace_back(std::make_unique<MemSeedBundle<LoadOrStoreT>>(LSI));
+  else
+    BundleVec.back()->insert(LSI, SE);
+
+  SeedLookupMap[LSI] = BundleVec.back().get();
+}
+
+// Explicit instantiations
+template void SeedContainer::insert<LoadInst>(LoadInst *);
+template void SeedContainer::insert<StoreInst>(StoreInst *);
+
+#ifndef NDEBUG
+void SeedContainer::dump() const {
+  for (const auto &Pair : Bundles) {
+    auto [I, Ty, Opc] = Pair.first;
+    const auto &SeedsVec = Pair.second;
+    std::string RefType = dyn_cast<LoadInst>(I)    ? "Load"
+                          : dyn_cast<StoreInst>(I) ? "Store"
+                                                   : "Other";
+    dbgs() << "[Inst=" << *I << " Ty=" << Ty << " " << RefType << "]\n";
+    for (const auto &SeedPtr : SeedsVec) {
+      SeedPtr->dump(dbgs());
+      dbgs() << "\n";
+    }
+  }
+  dbgs() << "\n";
+}
+#endif // NDEBUG
+
 } // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5638b0b9cd52f95cf173a1c622565f7d8ca1d492..b30f661071ea9f17689c63300f3df0cdfe1286db 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3113,6 +3113,10 @@ public:
   /// Generate the reduction in the loop
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of VPReductionRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 326d9225bf2a22aa04323ccb26d2fbea602a789e..497a86c4beb7115e6027a11fc02c98e1e1a348a9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1213,7 +1213,7 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
   // Use vector version of the intrinsic.
   Module *M = State.Builder.GetInsertBlock()->getModule();
   Function *VectorF =
-      Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
+      Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
   assert(VectorF && "Can't retrieve vector intrinsic.");
 
   auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
@@ -1449,7 +1449,7 @@ void EPIVPWidenCallRecipe::execute(VPTransformState &State) {
     // Use vector version of the intrinsic.
     if (State.VF.isVector())
       TysForDecl[0] = VectorType::get(CI.getType()->getScalarType(), State.VF);
-    VectorF = Intrinsic::getDeclaration(M, VPVectorIntrinsicID, TysForDecl);
+    VectorF = Intrinsic::getOrInsertDeclaration(M, VPVectorIntrinsicID, TysForDecl);
     assert(VectorF && "Can't retrieve vector intrinsic.");
   } else {
     // Retrieve declare SIMD function.
@@ -2708,6 +2708,40 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
   State.set(this, NewRed, /*IsScalar*/ true);
 }
 
+InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
+                                               VPCostContext &Ctx) const {
+  RecurKind RdxKind = RdxDesc.getRecurrenceKind();
+  Type *ElementTy = Ctx.Types.inferScalarType(this);
+  auto *VectorTy = cast<VectorType>(ToVectorTy(ElementTy, VF));
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  unsigned Opcode = RdxDesc.getOpcode();
+
+  // TODO: Support any-of and in-loop reductions.
+  assert(
+      (!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) ||
+       ForceTargetInstructionCost.getNumOccurrences() > 0) &&
+      "Any-of reduction not implemented in VPlan-based cost model currently.");
+  assert(
+      (!cast<VPReductionPHIRecipe>(getOperand(0))->isInLoop() ||
+       ForceTargetInstructionCost.getNumOccurrences() > 0) &&
+      "In-loop reduction not implemented in VPlan-based cost model currently.");
+
+  assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() &&
+         "Inferred type and recurrence type mismatch.");
+
+  // Cost = Reduction cost + BinOp cost
+  InstructionCost Cost =
+      Ctx.TTI.getArithmeticInstrCost(Opcode, ElementTy, CostKind);
+  if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind)) {
+    Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
+    return Cost + Ctx.TTI.getMinMaxReductionCost(
+                      Id, VectorTy, RdxDesc.getFastMathFlags(), CostKind);
+  }
+
+  return Cost + Ctx.TTI.getArithmeticReductionCost(
+                    Opcode, VectorTy, RdxDesc.getFastMathFlags(), CostKind);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent,
                               VPSlotTracker &SlotTracker) const {
@@ -3333,7 +3367,7 @@ void EPIVPWidenLoadRecipe::execute(VPTransformState &State) {
     Mask = Builder.getTrueVector(EC);
   } else if (isReverse()) {
     VectorType *MaskTy = cast<VectorType>(Mask->getType());
-    Function *VPIntr = Intrinsic::getDeclaration(
+    Function *VPIntr = Intrinsic::getOrInsertDeclaration(
         VectorPH->getModule(), Intrinsic::experimental_vp_reverse, {MaskTy});
     Mask = Builder.CreateCall(
         VPIntr, {Mask, Builder.getTrueVector(MaskTy->getElementCount()),
@@ -3366,7 +3400,7 @@ void EPIVPWidenLoadRecipe::execute(VPTransformState &State) {
     State.addMetadata(NewLI, LI);
   } else {
     auto *VecPtr = State.get(getAddr(), /*IsScalar*/ true);
-    Function *VPIntr = Intrinsic::getDeclaration(
+    Function *VPIntr = Intrinsic::getOrInsertDeclaration(
         VectorPH->getModule(), Intrinsic::vp_load, {DataTy, VecPtr->getType()});
 
     NewLI = Builder.CreateCall(VPIntr, {VecPtr, Mask, EVLPart}, "vp.op.load");
@@ -3374,11 +3408,11 @@ void EPIVPWidenLoadRecipe::execute(VPTransformState &State) {
     State.addMetadata(NewLI, LI);
     if (isReverse()) {
       VectorType *LoadedValTy = cast<VectorType>(NewLI->getType());
-      Function *VPIntr = Intrinsic::getDeclaration(
+      Function *VPIntr = Intrinsic::getOrInsertDeclaration(
           VectorPH->getModule(), Intrinsic::experimental_vp_reverse,
           {LoadedValTy});
       // The mask must be reversed too, otherwise no reversal will happen.
-      Function *VPIntrMask = Intrinsic::getDeclaration(
+      Function *VPIntrMask = Intrinsic::getOrInsertDeclaration(
           VectorPH->getModule(), Intrinsic::experimental_vp_reverse,
           {Mask->getType()});
       Mask = Builder.CreateCall(
@@ -3427,7 +3461,7 @@ void EPIVPWidenStoreRecipe::execute(VPTransformState &State) {
     Mask = Builder.getTrueVector(EC);
   } else if (isReverse()) {
     VectorType *MaskTy = cast<VectorType>(Mask->getType());
-    Function *VPIntr = Intrinsic::getDeclaration(
+    Function *VPIntr = Intrinsic::getOrInsertDeclaration(
         VectorPH->getModule(), Intrinsic::experimental_vp_reverse, {MaskTy});
     Mask = Builder.CreateCall(
         VPIntr, {Mask, Builder.getTrueVector(MaskTy->getElementCount()),
@@ -3470,12 +3504,12 @@ void EPIVPWidenStoreRecipe::execute(VPTransformState &State) {
       // If we store to reverse consecutive memory locations, then we need
       // to reverse the order of elements in the stored value.
       VectorType *StoredValTy = cast<VectorType>(StoredVal->getType());
-      Function *VPIntr = Intrinsic::getDeclaration(
+      Function *VPIntr = Intrinsic::getOrInsertDeclaration(
           VectorPH->getModule(), Intrinsic::experimental_vp_reverse,
           {StoredValTy});
 
       // The mask must be reversed too, otherwise no reversal will happen.
-      Function *VPIntrMask = Intrinsic::getDeclaration(
+      Function *VPIntrMask = Intrinsic::getOrInsertDeclaration(
           VectorPH->getModule(), Intrinsic::experimental_vp_reverse,
           {Mask->getType()});
       Value *ReversedMask = Builder.CreateCall(
@@ -3491,7 +3525,7 @@ void EPIVPWidenStoreRecipe::execute(VPTransformState &State) {
     auto *VecPtr = State.get(getAddr(), /*IsScalar*/ true);
     VectorType *StoredValTy = cast<VectorType>(StoredVal->getType());
     Function *VPIntr =
-        Intrinsic::getDeclaration(VectorPH->getModule(), Intrinsic::vp_store,
+        Intrinsic::getOrInsertDeclaration(VectorPH->getModule(), Intrinsic::vp_store,
                                   {StoredValTy, VecPtr->getType()});
     NewSI = Builder.CreateCall(VPIntr, {StoredVal, VecPtr, Mask, EVLPart});
   }
diff --git a/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll b/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
index 8085592111ff5ae480002676ccb6917cb766efd8..f84f35d4e8ca20623817c1853e6f5c3499e1a608 100644
--- a/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
+++ b/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
@@ -4,15 +4,14 @@
 ; The UDiv in the latch may never be executed. The backedge-taken-count
 ; expressions must account for the fact that evaluating the UDiv
 ; unconditionally may trigger UB.
-; FIXME: umin_seq should be used instead of umin  for BTCs.
 define i64 @multi_exit_exit_count_with_udiv_by_value_in_latch(ptr %dst, i64 %N) {
 ; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_value_in_latch'
 ; CHECK-NEXT:  Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_value_in_latch
-; CHECK-NEXT:  Loop %loop.header: <multiple exits> backedge-taken count is ((42 /u %N) umin (0 smax %N))
+; CHECK-NEXT:  Loop %loop.header: <multiple exits> backedge-taken count is ((0 smax %N) umin_seq (42 /u %N))
 ; CHECK-NEXT:    exit count for loop.header: (0 smax %N)
 ; CHECK-NEXT:    exit count for loop.latch: (42 /u %N)
 ; CHECK-NEXT:  Loop %loop.header: constant max backedge-taken count is i64 42
-; CHECK-NEXT:  Loop %loop.header: symbolic max backedge-taken count is ((42 /u %N) umin (0 smax %N))
+; CHECK-NEXT:  Loop %loop.header: symbolic max backedge-taken count is ((0 smax %N) umin_seq (42 /u %N))
 ; CHECK-NEXT:    symbolic max exit count for loop.header: (0 smax %N)
 ; CHECK-NEXT:    symbolic max exit count for loop.latch: (42 /u %N)
 ; CHECK-NEXT:  Loop %loop.header: Trip multiple is 1
@@ -41,7 +40,6 @@ exit:
 ; The UDiv in the latch may never be executed. The backedge-taken-count
 ; expressions must account for the fact that evaluating the UDiv
 ; unconditionally may trigger UB.
-; FIXME: umin_seq should be used instead of umin  for BTCs.
 define i64 @multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds(ptr %dst, i64 %N, i64 %M) {
 ; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds'
 ; CHECK-NEXT:  Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/nodivergencesource.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/nodivergencesource.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9c893ac3ba76a0c435c1ee012e90414f9969e689
--- /dev/null
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/nodivergencesource.ll
@@ -0,0 +1,16 @@
+; RUN: opt -mtriple amdgcn-- -passes='print<uniformity>' -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: DIVERGENT: %divergentval
+; CHECK-NOT: DIVERGENT: %uniformval
+; CHECK: %uniformval
+define void @test() {
+  %divergentval = call i32 @normalfunc()
+  %uniformval = call i32 @nodivergencesourcefunc()
+  ret void
+}
+
+declare i32 @normalfunc() #0
+declare i32 @nodivergencesourcefunc() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind nodivergencesource }
diff --git a/llvm/test/Assembler/flags.ll b/llvm/test/Assembler/flags.ll
index 84209500d27a5dbf1fc5e708d1620672c28e19d6..acc8874aef443862e836bca8dd5a6f19ef44a870 100644
--- a/llvm/test/Assembler/flags.ll
+++ b/llvm/test/Assembler/flags.ll
@@ -312,6 +312,18 @@ define <2 x i32> @test_trunc_both_reversed_vector(<2 x i64> %a) {
   ret <2 x i32> %res
 }
 
+define i1 @test_icmp_samesign(i32 %a, i32 %b) {
+  ; CHECK: %res = icmp samesign ult i32 %a, %b
+  %res = icmp samesign ult i32 %a, %b
+  ret i1 %res
+}
+
+define <2 x i1> @test_icmp_samesign2(<2 x i32> %a, <2 x i32> %b) {
+  ; CHECK: %res = icmp samesign ult <2 x i32> %a, %b
+  %res = icmp samesign ult <2 x i32> %a, %b
+  ret <2 x i1> %res
+}
+
 define ptr @gep_nuw(ptr %p, i64 %idx) {
 ; CHECK: %gep = getelementptr nuw i8, ptr %p, i64 %idx
   %gep = getelementptr nuw i8, ptr %p, i64 %idx
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index a66eda19ff57357147f74bee0f1c99540a6916f5..737f49aa86a7ba224d227c9de1c7025b9655b5bc 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -537,6 +537,11 @@ define void @f91(ptr dead_on_unwind %p) {
   ret void
 }
 
+; CHECK: define void @f94() [[NODIVERGENCESOURCE:#[0-9]+]]
+define void @f94() nodivergencesource {
+  ret void;
+}
+
 ; CHECK: define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a)
 define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a) {
   ret i32 0
@@ -615,4 +620,5 @@ define void @initializes(ptr initializes((-4, 0), (4, 8)) %a) {
 ; CHECK: attributes [[FNRETTHUNKEXTERN]] = { fn_ret_thunk_extern }
 ; CHECK: attributes [[SKIPPROFILE]] = { skipprofile }
 ; CHECK: attributes [[OPTDEBUG]] = { optdebug }
+; CHECK: attributes [[NODIVERGENCESOURCE]] = { nodivergencesource }
 ; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }
diff --git a/llvm/test/Bitcode/flags.ll b/llvm/test/Bitcode/flags.ll
index fd56694ccceb2ca47fdf731fbfe6b84ed3b8463f..99988c9ba3d3d47c450b320cae760b9e7e1a6197 100644
--- a/llvm/test/Bitcode/flags.ll
+++ b/llvm/test/Bitcode/flags.ll
@@ -30,6 +30,8 @@ second:                                           ; preds = %first
   %tsv = trunc nsw <2 x i32> %aa to <2 x i16>
   %tusv = trunc nuw nsw <2 x i32> %aa to <2 x i16>
   %tv = trunc <2 x i32> %aa to <2 x i16>
+  %ii = icmp samesign ult i32 %a, %z
+  %iv = icmp samesign ult <2 x i32> %aa, %aa
   unreachable
 
 first:                                                    ; preds = %entry
@@ -53,5 +55,7 @@ first:                                                    ; preds = %entry
   %ttsv = trunc nsw <2 x i32> %aa to <2 x i16>
   %ttusv = trunc nuw nsw <2 x i32> %aa to <2 x i16>
   %ttv = trunc <2 x i32> %aa to <2 x i16>
+  %icm = icmp samesign ult i32 %a, %zz
+  %icv = icmp samesign ult <2 x i32> %aa, %aa
   br label %second
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir
index cec8d9ea5ba9ded132fd9e679bb7b1335a238e76..a29a46bd43e496482a722ef5726606f88d78a488 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-regbankselect.mir
@@ -216,11 +216,11 @@ name:            phiPropagation
 legalized:       true
 tracksRegLiveness:   true
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: gpr64sp, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 4, class: gpr, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 1, class: gpr64sp, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 3, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 4, class: gpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: gpr32 }
   - { id: 1, class: gpr64sp }
@@ -359,8 +359,8 @@ body: |
 name:            ignoreTargetSpecificInst
 legalized:       true
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr64, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: gpr64 }
   - { id: 1, class: gpr64 }
@@ -397,9 +397,9 @@ name:            bitcast_s32_gpr
 legalized:       true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# FAST-NEXT:   - { id: 1, class: fpr, preferred-register: '' }
-# GREEDY-NEXT: - { id: 1, class: gpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# FAST-NEXT:   - { id: 1, class: fpr, preferred-register: '', flags: [  ] }
+# GREEDY-NEXT: - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -423,9 +423,9 @@ name:            bitcast_s32_fpr
 legalized:       true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: fpr, preferred-register: '' }
-# FAST-NEXT:   - { id: 1, class: gpr, preferred-register: '' }
-# GREEDY-NEXT: - { id: 1, class: fpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: fpr, preferred-register: '', flags: [  ] }
+# FAST-NEXT:   - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
+# GREEDY-NEXT: - { id: 1, class: fpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -448,9 +448,9 @@ name:            bitcast_s32_gpr_fpr
 legalized:       true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# FAST-NEXT:  - { id: 1, class: fpr, preferred-register: '' }
-# GREEDY-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# FAST-NEXT:  - { id: 1, class: fpr, preferred-register: '', flags: [  ] }
+# GREEDY-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -632,8 +632,8 @@ registers:
   - { id: 0, class: fpr128}
   - { id: 1, class: _}
 # CHECK: registers:
-# CHECK:  - { id: 0, class: fpr128, preferred-register: '' }
-# CHECK:  - { id: 1, class: fpr, preferred-register: '' }
+# CHECK:  - { id: 0, class: fpr128, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 1, class: fpr, preferred-register: '', flags: [  ] }
 # CHECK: %1:fpr(s128) = COPY %0
 body:             |
   bb.1:
@@ -659,8 +659,8 @@ registers:
   - { id: 0, class: _}
   - { id: 1, class: _}
 # CHECK: registers:
-# CHECK:  - { id: 0, class: gpr, preferred-register: '' }
-# CHECK:  - { id: 1, class: gpr, preferred-register: '' }
+# CHECK:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
 # CHECK: %0:gpr(s32) = COPY $w0
 # CHECK-NEXT: %1:gpr(s16) = G_TRUNC %0(s32)
 body:             |
@@ -723,11 +723,11 @@ name:            floatingPointLoad
 legalized:       true
 
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 4, class: fpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 4, class: fpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -765,11 +765,11 @@ name:            floatingPointStore
 legalized:       true
 
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 4, class: fpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 4, class: fpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -803,10 +803,10 @@ name:            fp16Ext32
 alignment:       4
 legalized:       true
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -838,10 +838,10 @@ name:            fp16Ext64
 alignment:       4
 legalized:       true
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 3, class: fpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -873,9 +873,9 @@ name:            fp32Ext64
 alignment:       4
 legalized:       true
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: fpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: fpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -900,7 +900,7 @@ body:             |
 # Make sure we map FP16 ABI on FPR register bank.
 # CHECK-LABEL: name: passFp16
 # CHECK: registers:
-# CHECK:  - { id: 0, class: fpr, preferred-register: '' }
+# CHECK:  - { id: 0, class: fpr, preferred-register: '', flags: [  ] }
 # CHECK:  %0:fpr(s16) = COPY $h0
 # CHECK-NEXT: $h0 = COPY %0(s16)
 name:            passFp16
@@ -922,9 +922,9 @@ body:             |
 # In that example, the copy comes from an ABI lowering of a fp type.
 # CHECK-LABEL: name: passFp16ViaAllocas
 # CHECK: registers:
-# CHECK:  - { id: 0, class: fpr, preferred-register: '' }
-# CHECK:  - { id: 1, class: gpr, preferred-register: '' }
-# CHECK:  - { id: 2, class: fpr, preferred-register: '' }
+# CHECK:  - { id: 0, class: fpr, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 2, class: fpr, preferred-register: '', flags: [  ] }
 #
 # CHECK:  %0:fpr(s16) = COPY $h0
 # CHECK-NEXT: %1:gpr(p0) = G_FRAME_INDEX %stack.0.p.addr
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
index b1bfe46f4b454cc99ac92f1717fae5034bf838d2..ca8f5de381e13c2a650395273e7267a3a3d67acc 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
@@ -35,7 +35,7 @@ define void @test_simple_arg(i32 %in) {
 ; CHECK-LABEL: name: test_indirect_call
 ; CHECK: registers:
 ; Make sure the register feeding the indirect call is properly constrained.
-; CHECK: - { id: [[FUNC:[0-9]+]], class: gpr64, preferred-register: '' }
+; CHECK: - { id: [[FUNC:[0-9]+]], class: gpr64, preferred-register: '', flags: [  ] }
 ; CHECK: %[[FUNC]]:gpr64(p0) = COPY $x0
 ; CHECK: BLR %[[FUNC]](p0), csr_aarch64_aapcs, implicit-def $lr, implicit $sp
 ; CHECK: RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
index f31b64ece895727f7fa2c50993bf122b7cd31372..27d09b236256752f617a43565f5601e7a8fee43a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
@@ -61,11 +61,12 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove1
     ; CHECK: liveins: $x0, $x1, $x2
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
-    ; CHECK: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store (s8) into %ir.0, align 4), (load (s8) from %ir.1, align 4)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-NEXT: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store (s8) into %ir.0, align 4), (load (s8) from %ir.1, align 4)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = COPY $x2
@@ -83,23 +84,24 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove2_const
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
-    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
-    ; CHECK: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
-    ; CHECK: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
-    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; CHECK: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p0) :: (store (s128) into %ir.0 + 16, align 4)
-    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; CHECK: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 32, align 4)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p0) :: (store (s128) into %ir.0 + 16, align 4)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 32, align 4)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 48
@@ -117,11 +119,42 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove3_const_toolarge
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 96
-    ; CHECK: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 1 :: (store (s8) into %ir.0, align 4), (load (s8) from %ir.1, align 4)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
+    ; CHECK-NEXT: [[LOAD3:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD2]](p0) :: (load (s128) from %ir.1 + 48, align 4)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C3]](s64)
+    ; CHECK-NEXT: [[LOAD4:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD3]](p0) :: (load (s128) from %ir.1 + 64, align 4)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 80
+    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C4]](s64)
+    ; CHECK-NEXT: [[LOAD5:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD4]](p0) :: (load (s128) from %ir.1 + 80, align 4)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD5]](p0) :: (store (s128) into %ir.0 + 16, align 4)
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD6]](p0) :: (store (s128) into %ir.0 + 32, align 4)
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD3]](s128), [[PTR_ADD7]](p0) :: (store (s128) into %ir.0 + 48, align 4)
+    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C8]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD4]](s128), [[PTR_ADD8]](p0) :: (store (s128) into %ir.0 + 64, align 4)
+    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 80
+    ; CHECK-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C9]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD5]](s128), [[PTR_ADD9]](p0) :: (store (s128) into %ir.0 + 80, align 4)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 96
@@ -139,29 +172,30 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove4_const_unaligned
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
-    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
-    ; CHECK: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
-    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
-    ; CHECK: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from %ir.1 + 48)
-    ; CHECK: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
-    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; CHECK: G_STORE [[LOAD1]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 16, align 4)
-    ; CHECK: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; CHECK: G_STORE [[LOAD2]](s128), [[PTR_ADD4]](p0) :: (store (s128) into %ir.0 + 32, align 4)
-    ; CHECK: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; CHECK: G_STORE [[LOAD3]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %ir.0 + 48)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
+    ; CHECK-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from %ir.1 + 48)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 16, align 4)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD4]](p0) :: (store (s128) into %ir.0 + 32, align 4)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD3]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %ir.0 + 48)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 52
@@ -179,23 +213,24 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove_addrspace
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p2) = COPY $x1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p2) :: (load (s128) from %ir.1, align 4, addrspace 2)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p2) :: (load (s128) from %ir.1 + 16, align 4, addrspace 2)
-    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C1]](s64)
-    ; CHECK: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p2) :: (load (s128) from %ir.1 + 32, align 4, addrspace 2)
-    ; CHECK: G_STORE [[LOAD]](s128), [[COPY]](p1) :: (store (s128) into %ir.0, align 4, addrspace 1)
-    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; CHECK: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p1) :: (store (s128) into %ir.0 + 16, align 4, addrspace 1)
-    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; CHECK: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p1) :: (store (s128) into %ir.0 + 32, align 4, addrspace 1)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p2) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p2) :: (load (s128) from %ir.1, align 4, addrspace 2)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p2) :: (load (s128) from %ir.1 + 16, align 4, addrspace 2)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p2) :: (load (s128) from %ir.1 + 32, align 4, addrspace 2)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p1) :: (store (s128) into %ir.0, align 4, addrspace 1)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p1) :: (store (s128) into %ir.0 + 16, align 4, addrspace 1)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p1) :: (store (s128) into %ir.0 + 32, align 4, addrspace 1)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p1) = COPY $x0
     %1:_(p2) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 48
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-stack-protector-trap-unreachable.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-stack-protector-trap-unreachable.ll
new file mode 100644
index 0000000000000000000000000000000000000000..00ad8fec9563892136be03008dbc74fc4e398fe1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-stack-protector-trap-unreachable.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mtriple=aarch64 -global-isel -stop-after=irtranslator -verify-machineinstrs \
+; RUN:     -trap-unreachable=false -o - %s | FileCheck -check-prefix=NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=aarch64 -global-isel -stop-after=irtranslator -verify-machineinstrs \
+; RUN:     -trap-unreachable -no-trap-after-noreturn -o - %s | FileCheck -check-prefix=NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=aarch64 -global-isel -stop-after=irtranslator -verify-machineinstrs \
+; RUN:     -trap-unreachable -no-trap-after-noreturn=false -o - %s | FileCheck -check-prefix=TRAP_UNREACHABLE %s
+
+;; Make sure we emit trap instructions after stack protector checks iff NoTrapAfterNoReturn is false.
+
+define void @test() nounwind ssp {
+  ; NO_TRAP_UNREACHABLE-LABEL: name: test
+  ; NO_TRAP_UNREACHABLE: bb.1.entry:
+  ; NO_TRAP_UNREACHABLE-NEXT:   successors: %bb.2(0x7ffff800), %bb.3(0x00000800)
+  ; NO_TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.StackGuardSlot
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[LOAD_STACK_GUARD:%[0-9]+]]:gpr64sp(p0) = LOAD_STACK_GUARD :: (dereferenceable invariant load (p0) from @__stack_chk_guard)
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[LOAD_STACK_GUARD1:%[0-9]+]]:gpr64sp(p0) = LOAD_STACK_GUARD :: (dereferenceable invariant load (p0) from @__stack_chk_guard)
+  ; NO_TRAP_UNREACHABLE-NEXT:   G_STORE [[LOAD_STACK_GUARD1]](p0), [[FRAME_INDEX]](p0) :: (volatile store (p0) into %stack.0.StackGuardSlot)
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.buf
+  ; NO_TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; NO_TRAP_UNREACHABLE-NEXT:   $x0 = COPY [[FRAME_INDEX1]](p0)
+  ; NO_TRAP_UNREACHABLE-NEXT:   BL @callee, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit-def $w0
+  ; NO_TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.StackGuardSlot
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX2]](p0) :: (volatile load (s64) from %stack.0.StackGuardSlot)
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[LOAD_STACK_GUARD2:%[0-9]+]]:gpr64sp(s64) = LOAD_STACK_GUARD :: (dereferenceable invariant load (p0) from @__stack_chk_guard)
+  ; NO_TRAP_UNREACHABLE-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LOAD_STACK_GUARD2]](s64), [[LOAD]]
+  ; NO_TRAP_UNREACHABLE-NEXT:   G_BRCOND [[ICMP]](s1), %bb.3
+  ; NO_TRAP_UNREACHABLE-NEXT:   G_BR %bb.2
+  ; NO_TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; NO_TRAP_UNREACHABLE-NEXT: bb.3.entry:
+  ; NO_TRAP_UNREACHABLE-NEXT:   successors:
+  ; NO_TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; NO_TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; NO_TRAP_UNREACHABLE-NEXT:   BL &__stack_chk_fail, csr_aarch64_aapcs, implicit-def $lr, implicit $sp
+  ; NO_TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; NO_TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; NO_TRAP_UNREACHABLE-NEXT: bb.2.entry:
+  ; NO_TRAP_UNREACHABLE-NEXT:   RET_ReallyLR
+  ;
+  ; TRAP_UNREACHABLE-LABEL: name: test
+  ; TRAP_UNREACHABLE: bb.1.entry:
+  ; TRAP_UNREACHABLE-NEXT:   successors: %bb.2(0x7ffff800), %bb.3(0x00000800)
+  ; TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; TRAP_UNREACHABLE-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.StackGuardSlot
+  ; TRAP_UNREACHABLE-NEXT:   [[LOAD_STACK_GUARD:%[0-9]+]]:gpr64sp(p0) = LOAD_STACK_GUARD :: (dereferenceable invariant load (p0) from @__stack_chk_guard)
+  ; TRAP_UNREACHABLE-NEXT:   [[LOAD_STACK_GUARD1:%[0-9]+]]:gpr64sp(p0) = LOAD_STACK_GUARD :: (dereferenceable invariant load (p0) from @__stack_chk_guard)
+  ; TRAP_UNREACHABLE-NEXT:   G_STORE [[LOAD_STACK_GUARD1]](p0), [[FRAME_INDEX]](p0) :: (volatile store (p0) into %stack.0.StackGuardSlot)
+  ; TRAP_UNREACHABLE-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.buf
+  ; TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; TRAP_UNREACHABLE-NEXT:   $x0 = COPY [[FRAME_INDEX1]](p0)
+  ; TRAP_UNREACHABLE-NEXT:   BL @callee, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit-def $w0
+  ; TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; TRAP_UNREACHABLE-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; TRAP_UNREACHABLE-NEXT:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.StackGuardSlot
+  ; TRAP_UNREACHABLE-NEXT:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX2]](p0) :: (volatile load (s64) from %stack.0.StackGuardSlot)
+  ; TRAP_UNREACHABLE-NEXT:   [[LOAD_STACK_GUARD2:%[0-9]+]]:gpr64sp(s64) = LOAD_STACK_GUARD :: (dereferenceable invariant load (p0) from @__stack_chk_guard)
+  ; TRAP_UNREACHABLE-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LOAD_STACK_GUARD2]](s64), [[LOAD]]
+  ; TRAP_UNREACHABLE-NEXT:   G_BRCOND [[ICMP]](s1), %bb.3
+  ; TRAP_UNREACHABLE-NEXT:   G_BR %bb.2
+  ; TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; TRAP_UNREACHABLE-NEXT: bb.3.entry:
+  ; TRAP_UNREACHABLE-NEXT:   successors:
+  ; TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; TRAP_UNREACHABLE-NEXT:   BL &__stack_chk_fail, csr_aarch64_aapcs, implicit-def $lr, implicit $sp
+  ; TRAP_UNREACHABLE-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; TRAP_UNREACHABLE-NEXT:   G_TRAP
+  ; TRAP_UNREACHABLE-NEXT: {{  $}}
+  ; TRAP_UNREACHABLE-NEXT: bb.2.entry:
+  ; TRAP_UNREACHABLE-NEXT:   RET_ReallyLR
+entry:
+  %buf = alloca [8 x i8]
+  %result = call i32(ptr) @callee(ptr %buf) nounwind
+  ret void
+}
+
+declare i32 @callee(ptr) nounwind
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/regbankselect-dbg-value.mir b/llvm/test/CodeGen/AArch64/GlobalISel/regbankselect-dbg-value.mir
index c64e2f78ab3bfc7421e4789595d22f5de5c50a66..daf7d49797d5c8c4ea8bf8e784e642e5be7e59e4 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/regbankselect-dbg-value.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/regbankselect-dbg-value.mir
@@ -31,7 +31,7 @@
 name:            test_dbg_value
 legalized:       true
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
 body: |
   bb.0:
     liveins: $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-intrinsic-aarch64-hint.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-intrinsic-aarch64-hint.mir
index 231785355c832307c47c55d07f6f8b6aa20f5c40..c63f99218f05cabedc8e9f9e02da34ac108db10e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-intrinsic-aarch64-hint.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-intrinsic-aarch64-hint.mir
@@ -14,7 +14,7 @@ legalized:       true
 regBankSelected: true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: gpr }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-static.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-static.mir
index 2d84e0415a6ccfcaf2983b90e730880872866219..df13c02374fedd37aa362e1fc59b7bc5acdea3e4 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-static.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-static.mir
@@ -31,7 +31,7 @@ legalized:       true
 regBankSelected: true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: gpr64sp, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr64sp, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: gpr }
 
@@ -105,12 +105,12 @@ legalized:       true
 regBankSelected: true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 2, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 4, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 5, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 2, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 4, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 5, class: gpr32, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: gpr }
   - { id: 1, class: gpr }
@@ -159,12 +159,12 @@ legalized:       true
 regBankSelected: true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: fpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 2, class: fpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 4, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 5, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: fpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 2, class: fpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 4, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 5, class: gpr32, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: fpr }
   - { id: 1, class: gpr }
@@ -202,10 +202,10 @@ regBankSelected: true
 tracksRegLiveness: true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: fpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
-# CHECK-NEXT:  - { id: 2, class: fpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: fpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 2, class: fpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: fpr }
   - { id: 1, class: gpr }
@@ -240,16 +240,16 @@ regBankSelected: true
 tracksRegLiveness: true
 
 # CHECK:      registers:
-# CHECK-NEXT:  - { id: 0, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 2, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 4, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 5, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 6, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 7, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 8, class: gpr64, preferred-register: '' }
-# CHECK-NEXT:  - { id: 9, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 0, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 2, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 4, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 5, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 6, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 7, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 8, class: gpr64, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:  - { id: 9, class: gpr64, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: gpr }
   - { id: 1, class: gpr }
diff --git a/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll b/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
index 2dde251cbc02f9baf4922841bf5aa2b9d60e6c63..a39c2b5d14dddd2e3d68231fc98d322c474da660 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
@@ -1,134 +1,230 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple aarch64-none-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; Supported combines
 
 define <8 x i16> @dupsext_v8i8_v8i16(i8 %src, <8 x i8> %b) {
-; CHECK-LABEL: dupsext_v8i8_v8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v1.8b, w0
-; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupsext_v8i8_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v1.8b, w0
+; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupsext_v8i8_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    lsl w8, w0, #8
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    asr w8, w8, #8
+; CHECK-GI-NEXT:    dup v1.8h, w8
+; CHECK-GI-NEXT:    mul v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = sext i8 %src to i16
-    %ext.b = sext <8 x i8> %b to <8 x i16>
-    %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
-    %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
-    %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
-    ret <8 x i16> %out
+  %in = sext i8 %src to i16
+  %ext.b = sext <8 x i8> %b to <8 x i16>
+  %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
+  %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
+  %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
+  ret <8 x i16> %out
 }
 
 define <8 x i16> @dupzext_v8i8_v8i16(i8 %src, <8 x i8> %b) {
-; CHECK-LABEL: dupzext_v8i8_v8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v1.8b, w0
-; CHECK-NEXT:    umull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupzext_v8i8_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v1.8b, w0
+; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupzext_v8i8_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    and w8, w0, #0xff
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    dup v1.8h, w8
+; CHECK-GI-NEXT:    mul v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = zext i8 %src to i16
-    %ext.b = zext <8 x i8> %b to <8 x i16>
-    %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
-    %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
-    %out = mul nuw <8 x i16> %broadcast.splat, %ext.b
-    ret <8 x i16> %out
+  %in = zext i8 %src to i16
+  %ext.b = zext <8 x i8> %b to <8 x i16>
+  %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
+  %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
+  %out = mul nuw <8 x i16> %broadcast.splat, %ext.b
+  ret <8 x i16> %out
 }
 
 define <4 x i32> @dupsext_v4i16_v4i32(i16 %src, <4 x i16> %b) {
-; CHECK-LABEL: dupsext_v4i16_v4i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v1.4h, w0
-; CHECK-NEXT:    smull v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupsext_v4i16_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v1.4h, w0
+; CHECK-SD-NEXT:    smull v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupsext_v4i16_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxth w8, w0
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    dup v1.4s, w8
+; CHECK-GI-NEXT:    mul v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = sext i16 %src to i32
-    %ext.b = sext <4 x i16> %b to <4 x i32>
-    %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %in, i32 0
-    %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-    %out = mul nsw <4 x i32> %broadcast.splat, %ext.b
-    ret <4 x i32> %out
+  %in = sext i16 %src to i32
+  %ext.b = sext <4 x i16> %b to <4 x i32>
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %in, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %out = mul nsw <4 x i32> %broadcast.splat, %ext.b
+  ret <4 x i32> %out
 }
 
 define <4 x i32> @dupzext_v4i16_v4i32(i16 %src, <4 x i16> %b) {
-; CHECK-LABEL: dupzext_v4i16_v4i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v1.4h, w0
-; CHECK-NEXT:    umull v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupzext_v4i16_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v1.4h, w0
+; CHECK-SD-NEXT:    umull v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupzext_v4i16_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    and w8, w0, #0xffff
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    dup v1.4s, w8
+; CHECK-GI-NEXT:    mul v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = zext i16 %src to i32
-    %ext.b = zext <4 x i16> %b to <4 x i32>
-    %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %in, i32 0
-    %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-    %out = mul nuw <4 x i32> %broadcast.splat, %ext.b
-    ret <4 x i32> %out
+  %in = zext i16 %src to i32
+  %ext.b = zext <4 x i16> %b to <4 x i32>
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %in, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %out = mul nuw <4 x i32> %broadcast.splat, %ext.b
+  ret <4 x i32> %out
 }
 
 define <2 x i64> @dupsext_v2i32_v2i64(i32 %src, <2 x i32> %b) {
-; CHECK-LABEL: dupsext_v2i32_v2i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v1.2s, w0
-; CHECK-NEXT:    smull v0.2d, v1.2s, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupsext_v2i32_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v1.2s, w0
+; CHECK-SD-NEXT:    smull v0.2d, v1.2s, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupsext_v2i32_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    dup v1.2d, x8
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov x11, v0.d[1]
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    mul x9, x10, x11
+; CHECK-GI-NEXT:    mov v0.d[0], x8
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = sext i32 %src to i64
-    %ext.b = sext <2 x i32> %b to <2 x i64>
-    %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
-    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
-    %out = mul nsw <2 x i64> %broadcast.splat, %ext.b
-    ret <2 x i64> %out
+  %in = sext i32 %src to i64
+  %ext.b = sext <2 x i32> %b to <2 x i64>
+  %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
+  %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
+  %out = mul nsw <2 x i64> %broadcast.splat, %ext.b
+  ret <2 x i64> %out
 }
 
 define <2 x i64> @dupzext_v2i32_v2i64(i32 %src, <2 x i32> %b) {
-; CHECK-LABEL: dupzext_v2i32_v2i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v1.2s, w0
-; CHECK-NEXT:    umull v0.2d, v1.2s, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupzext_v2i32_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v1.2s, w0
+; CHECK-SD-NEXT:    umull v0.2d, v1.2s, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupzext_v2i32_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    dup v1.2d, x8
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov x11, v0.d[1]
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    mul x9, x10, x11
+; CHECK-GI-NEXT:    mov v0.d[0], x8
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = zext i32 %src to i64
-    %ext.b = zext <2 x i32> %b to <2 x i64>
-    %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
-    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
-    %out = mul nuw <2 x i64> %broadcast.splat, %ext.b
-    ret <2 x i64> %out
+  %in = zext i32 %src to i64
+  %ext.b = zext <2 x i32> %b to <2 x i64>
+  %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
+  %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
+  %out = mul nuw <2 x i64> %broadcast.splat, %ext.b
+  ret <2 x i64> %out
 }
 
 ; Unsupported combines
 
 define <2 x i16> @dupsext_v2i8_v2i16(i8 %src, <2 x i8> %b) {
-; CHECK-LABEL: dupsext_v2i8_v2i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sxtb w8, w0
-; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    dup v1.2s, w8
-; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-NEXT:    mul v0.2s, v1.2s, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupsext_v2i8_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxtb w8, w0
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    mul v0.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupsext_v2i8_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    lsl w8, w0, #8
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    asr w8, w8, #8
+; CHECK-GI-NEXT:    dup v1.4h, w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mul v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = sext i8 %src to i16
-    %ext.b = sext <2 x i8> %b to <2 x i16>
-    %broadcast.splatinsert = insertelement <2 x i16> undef, i16 %in, i16 0
-    %broadcast.splat = shufflevector <2 x i16> %broadcast.splatinsert, <2 x i16> undef, <2 x i32> zeroinitializer
-    %out = mul nsw <2 x i16> %broadcast.splat, %ext.b
-    ret <2 x i16> %out
+  %in = sext i8 %src to i16
+  %ext.b = sext <2 x i8> %b to <2 x i16>
+  %broadcast.splatinsert = insertelement <2 x i16> undef, i16 %in, i16 0
+  %broadcast.splat = shufflevector <2 x i16> %broadcast.splatinsert, <2 x i16> undef, <2 x i32> zeroinitializer
+  %out = mul nsw <2 x i16> %broadcast.splat, %ext.b
+  ret <2 x i16> %out
 }
 
 define <2 x i64> @dupzext_v2i16_v2i64(i16 %src, <2 x i16> %b) {
-; CHECK-LABEL: dupzext_v2i16_v2i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi d1, #0x00ffff0000ffff
-; CHECK-NEXT:    and w8, w0, #0xffff
-; CHECK-NEXT:    dup v2.2s, w8
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umull v0.2d, v2.2s, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: dupzext_v2i16_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and w8, w0, #0xffff
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v0.2d, v2.2s, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dupzext_v2i16_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.2d, #0x0000000000ffff
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    and x8, x0, #0xffff
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    dup v1.2d, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
+; CHECK-GI-NEXT:    mov x11, v0.d[1]
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    mul x9, x10, x11
+; CHECK-GI-NEXT:    mov v0.d[0], x8
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = zext i16 %src to i64
-    %ext.b = zext <2 x i16> %b to <2 x i64>
-    %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
-    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
-    %out = mul nuw <2 x i64> %broadcast.splat, %ext.b
-    ret <2 x i64> %out
+  %in = zext i16 %src to i64
+  %ext.b = zext <2 x i16> %b to <2 x i64>
+  %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
+  %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
+  %out = mul nuw <2 x i64> %broadcast.splat, %ext.b
+  ret <2 x i64> %out
 }
 
 ; dupsext_v4i8_v4i16
@@ -148,109 +244,179 @@ entry:
 ; Unsupported states
 
 define <8 x i16> @nonsplat_shuffleinsert(i8 %src, <8 x i8> %b) {
-; CHECK-LABEL: nonsplat_shuffleinsert:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v1.8b, w0
-; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: nonsplat_shuffleinsert:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v1.8b, w0
+; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: nonsplat_shuffleinsert:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    lsl w8, w0, #8
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    asr w8, w8, #8
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    ext v1.16b, v1.16b, v1.16b, #4
+; CHECK-GI-NEXT:    mul v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = sext i8 %src to i16
-    %ext.b = sext <8 x i8> %b to <8 x i16>
-    %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 1
-    %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
-    %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
-    ret <8 x i16> %out
+  %in = sext i8 %src to i16
+  %ext.b = sext <8 x i8> %b to <8 x i16>
+  %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 1
+  %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
+  %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
+  ret <8 x i16> %out
 }
 
 define <4 x i32> @nonsplat_shuffleinsert2(<4 x i16> %b, i16 %b0, i16 %b1, i16 %b2, i16 %b3) {
-; CHECK-LABEL: nonsplat_shuffleinsert2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, w0
-; CHECK-NEXT:    mov v1.h[1], w1
-; CHECK-NEXT:    mov v1.h[2], w2
-; CHECK-NEXT:    mov v1.h[3], w3
-; CHECK-NEXT:    smull v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: nonsplat_shuffleinsert2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    mov v1.h[1], w1
+; CHECK-SD-NEXT:    mov v1.h[2], w2
+; CHECK-SD-NEXT:    mov v1.h[3], w3
+; CHECK-SD-NEXT:    smull v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: nonsplat_shuffleinsert2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxth w8, w0
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    sxth w8, w1
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    sxth w8, w2
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    sxth w8, w3
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mul v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    ret
 entry:
-    %s0 = sext i16 %b0 to i32
-    %s1 = sext i16 %b1 to i32
-    %s2 = sext i16 %b2 to i32
-    %s3 = sext i16 %b3 to i32
-    %ext.b = sext <4 x i16> %b to <4 x i32>
-    %v0 = insertelement <4 x i32> undef, i32 %s0, i32 0
-    %v1 = insertelement <4 x i32> %v0, i32 %s1, i32 1
-    %v2 = insertelement <4 x i32> %v1, i32 %s2, i32 2
-    %v3 = insertelement <4 x i32> %v2, i32 %s3, i32 3
-    %out = mul nsw <4 x i32> %v3, %ext.b
-    ret <4 x i32> %out
+  %s0 = sext i16 %b0 to i32
+  %s1 = sext i16 %b1 to i32
+  %s2 = sext i16 %b2 to i32
+  %s3 = sext i16 %b3 to i32
+  %ext.b = sext <4 x i16> %b to <4 x i32>
+  %v0 = insertelement <4 x i32> undef, i32 %s0, i32 0
+  %v1 = insertelement <4 x i32> %v0, i32 %s1, i32 1
+  %v2 = insertelement <4 x i32> %v1, i32 %s2, i32 2
+  %v3 = insertelement <4 x i32> %v2, i32 %s3, i32 3
+  %out = mul nsw <4 x i32> %v3, %ext.b
+  ret <4 x i32> %out
 }
 
 define void @typei1_orig(i64 %a, ptr %p, ptr %q) {
-; CHECK-LABEL: typei1_orig:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    ldr q0, [x2]
-; CHECK-NEXT:    cset w8, gt
-; CHECK-NEXT:    dup v1.8h, w8
-; CHECK-NEXT:    cmtst v0.8h, v0.8h, v0.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, #0
-; CHECK-NEXT:    bic v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    xtn v0.8b, v0.8h
-; CHECK-NEXT:    str q0, [x1]
-; CHECK-NEXT:    ret
-    %tmp = xor <16 x i1> zeroinitializer, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
-    %tmp6 = load <8 x i16>, ptr %q, align 2
-    %tmp7 = sub <8 x i16> zeroinitializer, %tmp6
-    %tmp8 = shufflevector <8 x i16> %tmp7, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-    %tmp9 = icmp slt i64 0, %a
-    %tmp10 = zext i1 %tmp9 to i16
-    %tmp11 = insertelement <16 x i16> undef, i16 %tmp10, i64 0
-    %tmp12 = shufflevector <16 x i16> %tmp11, <16 x i16> undef, <16 x i32> zeroinitializer
-    %tmp13 = mul nuw <16 x i16> %tmp8, %tmp12
-    %tmp14 = icmp ne <16 x i16> %tmp13, zeroinitializer
-    %tmp15 = and <16 x i1> %tmp14, %tmp
-    %tmp16 = sext <16 x i1> %tmp15 to <16 x i8>
-    store <16 x i8> %tmp16, ptr %p, align 1
-    ret void
+; CHECK-SD-LABEL: typei1_orig:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmp x0, #0
+; CHECK-SD-NEXT:    ldr q0, [x2]
+; CHECK-SD-NEXT:    cset w8, gt
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    cmtst v0.8h, v0.8h, v0.8h
+; CHECK-SD-NEXT:    cmeq v1.8h, v1.8h, #0
+; CHECK-SD-NEXT:    bic v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    str q0, [x1]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: typei1_orig:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr q1, [x2]
+; CHECK-GI-NEXT:    cmp x0, #0
+; CHECK-GI-NEXT:    movi v0.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    cset w8, gt
+; CHECK-GI-NEXT:    neg v1.8h, v1.8h
+; CHECK-GI-NEXT:    dup v2.8h, w8
+; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
+; CHECK-GI-NEXT:    mul v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    cmeq v1.8h, v1.8h, #0
+; CHECK-GI-NEXT:    mvn v1.16b, v1.16b
+; CHECK-GI-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #7
+; CHECK-GI-NEXT:    sshr v0.16b, v0.16b, #7
+; CHECK-GI-NEXT:    str q0, [x1]
+; CHECK-GI-NEXT:    ret
+  %tmp = xor <16 x i1> zeroinitializer, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+  %tmp6 = load <8 x i16>, ptr %q, align 2
+  %tmp7 = sub <8 x i16> zeroinitializer, %tmp6
+  %tmp8 = shufflevector <8 x i16> %tmp7, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %tmp9 = icmp slt i64 0, %a
+  %tmp10 = zext i1 %tmp9 to i16
+  %tmp11 = insertelement <16 x i16> undef, i16 %tmp10, i64 0
+  %tmp12 = shufflevector <16 x i16> %tmp11, <16 x i16> undef, <16 x i32> zeroinitializer
+  %tmp13 = mul nuw <16 x i16> %tmp8, %tmp12
+  %tmp14 = icmp ne <16 x i16> %tmp13, zeroinitializer
+  %tmp15 = and <16 x i1> %tmp14, %tmp
+  %tmp16 = sext <16 x i1> %tmp15 to <16 x i8>
+  store <16 x i8> %tmp16, ptr %p, align 1
+  ret void
 }
 
 define <8 x i16> @typei1_v8i1_v8i16(i1 %src, <8 x i1> %b) {
-; CHECK-LABEL: typei1_v8i1_v8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.8b, #1
-; CHECK-NEXT:    and w8, w0, #0x1
-; CHECK-NEXT:    dup v2.8b, w8
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umull v0.8h, v2.8b, v0.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: typei1_v8i1_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #1
+; CHECK-SD-NEXT:    and w8, w0, #0x1
+; CHECK-SD-NEXT:    dup v2.8b, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v0.8h, v2.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: typei1_v8i1_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.8h, #1
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    and w8, w0, #0x1
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    dup v1.8h, w8
+; CHECK-GI-NEXT:    mul v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
 entry:
-    %in = zext i1 %src to i16
-    %ext.b = zext <8 x i1> %b to <8 x i16>
-    %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
-    %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
-    %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
-    ret <8 x i16> %out
+  %in = zext i1 %src to i16
+  %ext.b = zext <8 x i1> %b to <8 x i16>
+  %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
+  %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
+  %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
+  ret <8 x i16> %out
 }
 
 define <8 x i16> @missing_insert(<8 x i8> %b) {
-; CHECK-LABEL: missing_insert:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext v1.8b, v0.8b, v0.8b, #2
-; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: missing_insert:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.8b, v0.8b, v0.8b, #2
+; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: missing_insert:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ext v1.16b, v0.16b, v0.16b, #4
+; CHECK-GI-NEXT:    mul v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
 entry:
-    %ext.b = sext <8 x i8> %b to <8 x i16>
-    %broadcast.splat = shufflevector <8 x i16> %ext.b, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
-    %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
-    ret <8 x i16> %out
+  %ext.b = sext <8 x i8> %b to <8 x i16>
+  %broadcast.splat = shufflevector <8 x i16> %ext.b, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
+  %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
+  ret <8 x i16> %out
 }
 
 define <8 x i16> @shufsext_v8i8_v8i16(<8 x i8> %src, <8 x i8> %b) {
-; CHECK-LABEL: shufsext_v8i8_v8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.8b, v0.8b
-; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: shufsext_v8i8_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.8b, v0.8b
+; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: shufsext_v8i8_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI13_0
+; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ldr q0, [x8, :lo12:.LCPI13_0]
+; CHECK-GI-NEXT:    tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-GI-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    ret
 entry:
   %in = sext <8 x i8> %src to <8 x i16>
   %ext.b = sext <8 x i8> %b to <8 x i16>
@@ -260,11 +426,26 @@ entry:
 }
 
 define <2 x i64> @shufsext_v2i32_v2i64(<2 x i32> %src, <2 x i32> %b) {
-; CHECK-LABEL: shufsext_v2i32_v2i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.2s, v0.2s
-; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: shufsext_v2i32_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.2s, v0.2s
+; CHECK-SD-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: shufsext_v2i32_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    mul x9, x10, x11
+; CHECK-GI-NEXT:    mov v0.d[0], x8
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    ret
 entry:
   %in = sext <2 x i32> %src to <2 x i64>
   %ext.b = sext <2 x i32> %b to <2 x i64>
@@ -274,11 +455,21 @@ entry:
 }
 
 define <8 x i16> @shufzext_v8i8_v8i16(<8 x i8> %src, <8 x i8> %b) {
-; CHECK-LABEL: shufzext_v8i8_v8i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.8b, v0.8b
-; CHECK-NEXT:    umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: shufzext_v8i8_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.8b, v0.8b
+; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: shufzext_v8i8_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI15_0
+; CHECK-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    ldr q0, [x8, :lo12:.LCPI15_0]
+; CHECK-GI-NEXT:    tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-GI-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    ret
 entry:
   %in = zext <8 x i8> %src to <8 x i16>
   %ext.b = zext <8 x i8> %b to <8 x i16>
@@ -288,11 +479,26 @@ entry:
 }
 
 define <2 x i64> @shufzext_v2i32_v2i64(<2 x i32> %src, <2 x i32> %b) {
-; CHECK-LABEL: shufzext_v2i32_v2i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64 v0.2s, v0.2s
-; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: shufzext_v2i32_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev64 v0.2s, v0.2s
+; CHECK-SD-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: shufzext_v2i32_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    mul x9, x10, x11
+; CHECK-GI-NEXT:    mov v0.d[0], x8
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    ret
 entry:
   %in = sext <2 x i32> %src to <2 x i64>
   %ext.b = sext <2 x i32> %b to <2 x i64>
@@ -302,11 +508,20 @@ entry:
 }
 
 define <8 x i16> @shufzext_v8i8_v8i16_twoin(<8 x i8> %src1, <8 x i8> %src2, <8 x i8> %b) {
-; CHECK-LABEL: shufzext_v8i8_v8i16_twoin:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    trn1 v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umull v0.8h, v0.8b, v2.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: shufzext_v8i8_v8i16_twoin:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    trn1 v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v2.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: shufzext_v8i8_v8i16_twoin:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    trn1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    ushll v1.8h, v2.8b, #0
+; CHECK-GI-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    ret
 entry:
   %in1 = zext <8 x i8> %src1 to <8 x i16>
   %in2 = zext <8 x i8> %src2 to <8 x i16>
diff --git a/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
index f44f7863e78fc06f940f3e902bce0b8136733a34..99dfac807dcd157ae442c5bdfa51d56530b7abf0 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
@@ -1,26 +1,49 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-none-linux-gnu < %s -o -| FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define void @matrix_mul_unsigned(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i16 %val) {
-; CHECK-LABEL: matrix_mul_unsigned:
-; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    and w8, w3, #0xffff
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4h, w8
-; CHECK-NEXT:    and x8, x0, #0xfffffff8
-; CHECK-NEXT:  .LBB0_1: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x9, x2, w0, uxtw #1
-; CHECK-NEXT:    subs x8, x8, #8
-; CHECK-NEXT:    ldp d1, d2, [x9]
-; CHECK-NEXT:    add x9, x1, w0, uxtw #2
-; CHECK-NEXT:    add w0, w0, #8
-; CHECK-NEXT:    umull v1.4s, v0.4h, v1.4h
-; CHECK-NEXT:    umull v2.4s, v0.4h, v2.4h
-; CHECK-NEXT:    stp q1, q2, [x9]
-; CHECK-NEXT:    b.ne .LBB0_1
-; CHECK-NEXT:  // %bb.2: // %for.end12
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: matrix_mul_unsigned:
+; CHECK-SD:       // %bb.0: // %vector.header
+; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    dup v0.4h, w8
+; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
+; CHECK-SD-NEXT:  .LBB0_1: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-SD-NEXT:    subs x8, x8, #8
+; CHECK-SD-NEXT:    ldp d1, d2, [x9]
+; CHECK-SD-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-SD-NEXT:    add w0, w0, #8
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umull v2.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    stp q1, q2, [x9]
+; CHECK-SD-NEXT:    b.ne .LBB0_1
+; CHECK-SD-NEXT:  // %bb.2: // %for.end12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: matrix_mul_unsigned:
+; CHECK-GI:       // %bb.0: // %vector.header
+; CHECK-GI-NEXT:    and w8, w3, #0xffff
+; CHECK-GI-NEXT:    dup v0.4s, w8
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    and x8, x8, #0xfffffff8
+; CHECK-GI-NEXT:  .LBB0_1: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-GI-NEXT:    subs x8, x8, #8
+; CHECK-GI-NEXT:    ldp d1, d2, [x9]
+; CHECK-GI-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-GI-NEXT:    add w0, w0, #8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q1, q2, [x9]
+; CHECK-GI-NEXT:    b.ne .LBB0_1
+; CHECK-GI-NEXT:  // %bb.2: // %for.end12
+; CHECK-GI-NEXT:    ret
 vector.header:
   %conv4 = zext i16 %val to i32
   %wide.trip.count = zext i32 %N to i64
@@ -66,25 +89,48 @@ for.end12:                                        ; preds = %vector.body
 }
 
 define void @matrix_mul_signed(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i16 %val) {
-; CHECK-LABEL: matrix_mul_signed:
-; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    sxth w8, w3
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4h, w8
-; CHECK-NEXT:    and x8, x0, #0xfffffff8
-; CHECK-NEXT:  .LBB1_1: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x9, x2, w0, sxtw #1
-; CHECK-NEXT:    subs x8, x8, #8
-; CHECK-NEXT:    ldp d1, d2, [x9]
-; CHECK-NEXT:    add x9, x1, w0, sxtw #2
-; CHECK-NEXT:    add w0, w0, #8
-; CHECK-NEXT:    smull v1.4s, v0.4h, v1.4h
-; CHECK-NEXT:    smull v2.4s, v0.4h, v2.4h
-; CHECK-NEXT:    stp q1, q2, [x9]
-; CHECK-NEXT:    b.ne .LBB1_1
-; CHECK-NEXT:  // %bb.2: // %for.end12
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: matrix_mul_signed:
+; CHECK-SD:       // %bb.0: // %vector.header
+; CHECK-SD-NEXT:    sxth w8, w3
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    dup v0.4h, w8
+; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
+; CHECK-SD-NEXT:  .LBB1_1: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    add x9, x2, w0, sxtw #1
+; CHECK-SD-NEXT:    subs x8, x8, #8
+; CHECK-SD-NEXT:    ldp d1, d2, [x9]
+; CHECK-SD-NEXT:    add x9, x1, w0, sxtw #2
+; CHECK-SD-NEXT:    add w0, w0, #8
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    smull v2.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    stp q1, q2, [x9]
+; CHECK-SD-NEXT:    b.ne .LBB1_1
+; CHECK-SD-NEXT:  // %bb.2: // %for.end12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: matrix_mul_signed:
+; CHECK-GI:       // %bb.0: // %vector.header
+; CHECK-GI-NEXT:    sxth w9, w3
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    dup v0.4s, w9
+; CHECK-GI-NEXT:    and x8, x8, #0xfffffff8
+; CHECK-GI-NEXT:  .LBB1_1: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    add x9, x2, w0, sxtw #1
+; CHECK-GI-NEXT:    subs x8, x8, #8
+; CHECK-GI-NEXT:    ldp d1, d2, [x9]
+; CHECK-GI-NEXT:    add x9, x1, w0, sxtw #2
+; CHECK-GI-NEXT:    add w0, w0, #8
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q1, q2, [x9]
+; CHECK-GI-NEXT:    b.ne .LBB1_1
+; CHECK-GI-NEXT:  // %bb.2: // %for.end12
+; CHECK-GI-NEXT:    ret
 vector.header:
   %conv4 = sext i16 %val to i32
   %wide.trip.count = sext i32 %N to i64
@@ -131,25 +177,48 @@ for.end12:                                        ; preds = %vector.body
 
 
 define void @matrix_mul_double_shuffle(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i16 %val) {
-; CHECK-LABEL: matrix_mul_double_shuffle:
-; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    and w8, w3, #0xffff
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4h, w8
-; CHECK-NEXT:    and x8, x0, #0xfffffff8
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0 def $x0
-; CHECK-NEXT:  .LBB2_1: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrh w9, [x2], #16
-; CHECK-NEXT:    subs x8, x8, #8
-; CHECK-NEXT:    dup v1.4h, w9
-; CHECK-NEXT:    ubfiz x9, x0, #2, #32
-; CHECK-NEXT:    add w0, w0, #8
-; CHECK-NEXT:    umull v1.4s, v0.4h, v1.4h
-; CHECK-NEXT:    str q1, [x1, x9]
-; CHECK-NEXT:    b.ne .LBB2_1
-; CHECK-NEXT:  // %bb.2: // %for.end12
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: matrix_mul_double_shuffle:
+; CHECK-SD:       // %bb.0: // %vector.header
+; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    dup v0.4h, w8
+; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0 def $x0
+; CHECK-SD-NEXT:  .LBB2_1: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldrh w9, [x2], #16
+; CHECK-SD-NEXT:    subs x8, x8, #8
+; CHECK-SD-NEXT:    dup v1.4h, w9
+; CHECK-SD-NEXT:    ubfiz x9, x0, #2, #32
+; CHECK-SD-NEXT:    add w0, w0, #8
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    str q1, [x1, x9]
+; CHECK-SD-NEXT:    b.ne .LBB2_1
+; CHECK-SD-NEXT:  // %bb.2: // %for.end12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: matrix_mul_double_shuffle:
+; CHECK-GI:       // %bb.0: // %vector.header
+; CHECK-GI-NEXT:    and w9, w3, #0xffff
+; CHECK-GI-NEXT:    adrp x8, .LCPI2_0
+; CHECK-GI-NEXT:    dup v1.4s, w9
+; CHECK-GI-NEXT:    mov w9, w0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI2_0]
+; CHECK-GI-NEXT:    and x8, x9, #0xfffffff8
+; CHECK-GI-NEXT:  .LBB2_1: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldrh w9, [x2], #16
+; CHECK-GI-NEXT:    subs x8, x8, #8
+; CHECK-GI-NEXT:    mov v0.s[0], w9
+; CHECK-GI-NEXT:    mov w9, w0
+; CHECK-GI-NEXT:    add w0, w0, #8
+; CHECK-GI-NEXT:    lsl x9, x9, #2
+; CHECK-GI-NEXT:    tbl v3.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT:    mul v3.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    str q3, [x1, x9]
+; CHECK-GI-NEXT:    b.ne .LBB2_1
+; CHECK-GI-NEXT:  // %bb.2: // %for.end12
+; CHECK-GI-NEXT:    ret
 vector.header:
   %conv4 = zext i16 %val to i32
   %wide.trip.count = zext i32 %N to i64
@@ -187,52 +256,102 @@ for.end12:                                        ; preds = %vector.body
 
 
 define void @larger_smull(ptr nocapture noundef readonly %x, i16 noundef %y, ptr noalias nocapture noundef writeonly %s, i32 noundef %n) {
-; CHECK-LABEL: larger_smull:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cmp w3, #1
-; CHECK-NEXT:    b.lt .LBB3_8
-; CHECK-NEXT:  // %bb.1: // %for.body.preheader
-; CHECK-NEXT:    sxth w8, w1
-; CHECK-NEXT:    cmp w3, #15
-; CHECK-NEXT:    mov w9, w3
-; CHECK-NEXT:    b.hi .LBB3_3
-; CHECK-NEXT:  // %bb.2:
-; CHECK-NEXT:    mov x10, xzr
-; CHECK-NEXT:    b .LBB3_6
-; CHECK-NEXT:  .LBB3_3: // %vector.ph
-; CHECK-NEXT:    dup v0.8h, w8
-; CHECK-NEXT:    and x10, x9, #0xfffffff0
-; CHECK-NEXT:    add x11, x2, #32
-; CHECK-NEXT:    add x12, x0, #16
-; CHECK-NEXT:    mov x13, x10
-; CHECK-NEXT:  .LBB3_4: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldp q1, q2, [x12, #-16]
-; CHECK-NEXT:    subs x13, x13, #16
-; CHECK-NEXT:    add x12, x12, #32
-; CHECK-NEXT:    smull2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT:    smull v1.4s, v0.4h, v1.4h
-; CHECK-NEXT:    smull2 v4.4s, v0.8h, v2.8h
-; CHECK-NEXT:    smull v2.4s, v0.4h, v2.4h
-; CHECK-NEXT:    stp q1, q3, [x11, #-32]
-; CHECK-NEXT:    stp q2, q4, [x11], #64
-; CHECK-NEXT:    b.ne .LBB3_4
-; CHECK-NEXT:  // %bb.5: // %middle.block
-; CHECK-NEXT:    cmp x10, x9
-; CHECK-NEXT:    b.eq .LBB3_8
-; CHECK-NEXT:  .LBB3_6: // %for.body.preheader1
-; CHECK-NEXT:    add x11, x2, x10, lsl #2
-; CHECK-NEXT:    add x12, x0, x10, lsl #1
-; CHECK-NEXT:    sub x9, x9, x10
-; CHECK-NEXT:  .LBB3_7: // %for.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrsh w10, [x12], #2
-; CHECK-NEXT:    subs x9, x9, #1
-; CHECK-NEXT:    mul w10, w10, w8
-; CHECK-NEXT:    str w10, [x11], #4
-; CHECK-NEXT:    b.ne .LBB3_7
-; CHECK-NEXT:  .LBB3_8: // %for.cond.cleanup
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: larger_smull:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmp w3, #1
+; CHECK-SD-NEXT:    b.lt .LBB3_8
+; CHECK-SD-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-SD-NEXT:    sxth w8, w1
+; CHECK-SD-NEXT:    cmp w3, #15
+; CHECK-SD-NEXT:    mov w9, w3
+; CHECK-SD-NEXT:    b.hi .LBB3_3
+; CHECK-SD-NEXT:  // %bb.2:
+; CHECK-SD-NEXT:    mov x10, xzr
+; CHECK-SD-NEXT:    b .LBB3_6
+; CHECK-SD-NEXT:  .LBB3_3: // %vector.ph
+; CHECK-SD-NEXT:    dup v0.8h, w8
+; CHECK-SD-NEXT:    and x10, x9, #0xfffffff0
+; CHECK-SD-NEXT:    add x11, x2, #32
+; CHECK-SD-NEXT:    add x12, x0, #16
+; CHECK-SD-NEXT:    mov x13, x10
+; CHECK-SD-NEXT:  .LBB3_4: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldp q1, q2, [x12, #-16]
+; CHECK-SD-NEXT:    subs x13, x13, #16
+; CHECK-SD-NEXT:    add x12, x12, #32
+; CHECK-SD-NEXT:    smull2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    smull2 v4.4s, v0.8h, v2.8h
+; CHECK-SD-NEXT:    smull v2.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    stp q1, q3, [x11, #-32]
+; CHECK-SD-NEXT:    stp q2, q4, [x11], #64
+; CHECK-SD-NEXT:    b.ne .LBB3_4
+; CHECK-SD-NEXT:  // %bb.5: // %middle.block
+; CHECK-SD-NEXT:    cmp x10, x9
+; CHECK-SD-NEXT:    b.eq .LBB3_8
+; CHECK-SD-NEXT:  .LBB3_6: // %for.body.preheader1
+; CHECK-SD-NEXT:    add x11, x2, x10, lsl #2
+; CHECK-SD-NEXT:    add x12, x0, x10, lsl #1
+; CHECK-SD-NEXT:    sub x9, x9, x10
+; CHECK-SD-NEXT:  .LBB3_7: // %for.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldrsh w10, [x12], #2
+; CHECK-SD-NEXT:    subs x9, x9, #1
+; CHECK-SD-NEXT:    mul w10, w10, w8
+; CHECK-SD-NEXT:    str w10, [x11], #4
+; CHECK-SD-NEXT:    b.ne .LBB3_7
+; CHECK-SD-NEXT:  .LBB3_8: // %for.cond.cleanup
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: larger_smull:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmp w3, #0
+; CHECK-GI-NEXT:    b.le .LBB3_7
+; CHECK-GI-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-GI-NEXT:    sxth w8, w1
+; CHECK-GI-NEXT:    mov x9, xzr
+; CHECK-GI-NEXT:    cmp w3, #16
+; CHECK-GI-NEXT:    mov w10, w3
+; CHECK-GI-NEXT:    b.lo .LBB3_5
+; CHECK-GI-NEXT:  // %bb.2: // %vector.ph
+; CHECK-GI-NEXT:    dup v0.4s, w8
+; CHECK-GI-NEXT:    and x9, x10, #0xfffffff0
+; CHECK-GI-NEXT:    add x11, x2, #32
+; CHECK-GI-NEXT:    add x12, x0, #16
+; CHECK-GI-NEXT:    mov x13, x9
+; CHECK-GI-NEXT:  .LBB3_3: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldp q1, q2, [x12, #-16]
+; CHECK-GI-NEXT:    mov x14, x11
+; CHECK-GI-NEXT:    subs x13, x13, #16
+; CHECK-GI-NEXT:    add x12, x12, #32
+; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    sshll v4.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    mul v3.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v4.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q3, q1, [x14, #-32]!
+; CHECK-GI-NEXT:    stp q4, q2, [x11], #64
+; CHECK-GI-NEXT:    b.ne .LBB3_3
+; CHECK-GI-NEXT:  // %bb.4: // %middle.block
+; CHECK-GI-NEXT:    cmp x9, x10
+; CHECK-GI-NEXT:    b.eq .LBB3_7
+; CHECK-GI-NEXT:  .LBB3_5: // %for.body.preheader1
+; CHECK-GI-NEXT:    add x11, x2, x9, lsl #2
+; CHECK-GI-NEXT:    add x12, x0, x9, lsl #1
+; CHECK-GI-NEXT:    sub x9, x10, x9
+; CHECK-GI-NEXT:  .LBB3_6: // %for.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldrsh w10, [x12], #2
+; CHECK-GI-NEXT:    subs x9, x9, #1
+; CHECK-GI-NEXT:    mul w10, w10, w8
+; CHECK-GI-NEXT:    str w10, [x11], #4
+; CHECK-GI-NEXT:    b.ne .LBB3_6
+; CHECK-GI-NEXT:  .LBB3_7: // %for.cond.cleanup
+; CHECK-GI-NEXT:    ret
 entry:
   %conv1 = sext i16 %y to i32
   %cmp8 = icmp sgt i32 %n, 0
@@ -299,52 +418,102 @@ for.body:                                         ; preds = %for.body.preheader1
 
 
 define void @larger_umull(ptr nocapture noundef readonly %x, i16 noundef %y, ptr noalias nocapture noundef writeonly %s, i32 noundef %n) {
-; CHECK-LABEL: larger_umull:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cmp w3, #1
-; CHECK-NEXT:    b.lt .LBB4_8
-; CHECK-NEXT:  // %bb.1: // %for.body.preheader
-; CHECK-NEXT:    cmp w3, #15
-; CHECK-NEXT:    and w8, w1, #0xffff
-; CHECK-NEXT:    mov w9, w3
-; CHECK-NEXT:    b.hi .LBB4_3
-; CHECK-NEXT:  // %bb.2:
-; CHECK-NEXT:    mov x10, xzr
-; CHECK-NEXT:    b .LBB4_6
-; CHECK-NEXT:  .LBB4_3: // %vector.ph
-; CHECK-NEXT:    dup v0.8h, w8
-; CHECK-NEXT:    and x10, x9, #0xfffffff0
-; CHECK-NEXT:    add x11, x2, #32
-; CHECK-NEXT:    add x12, x0, #16
-; CHECK-NEXT:    mov x13, x10
-; CHECK-NEXT:  .LBB4_4: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldp q1, q2, [x12, #-16]
-; CHECK-NEXT:    subs x13, x13, #16
-; CHECK-NEXT:    add x12, x12, #32
-; CHECK-NEXT:    umull2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT:    umull v1.4s, v0.4h, v1.4h
-; CHECK-NEXT:    umull2 v4.4s, v0.8h, v2.8h
-; CHECK-NEXT:    umull v2.4s, v0.4h, v2.4h
-; CHECK-NEXT:    stp q1, q3, [x11, #-32]
-; CHECK-NEXT:    stp q2, q4, [x11], #64
-; CHECK-NEXT:    b.ne .LBB4_4
-; CHECK-NEXT:  // %bb.5: // %middle.block
-; CHECK-NEXT:    cmp x10, x9
-; CHECK-NEXT:    b.eq .LBB4_8
-; CHECK-NEXT:  .LBB4_6: // %for.body.preheader1
-; CHECK-NEXT:    add x11, x2, x10, lsl #2
-; CHECK-NEXT:    add x12, x0, x10, lsl #1
-; CHECK-NEXT:    sub x9, x9, x10
-; CHECK-NEXT:  .LBB4_7: // %for.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrh w10, [x12], #2
-; CHECK-NEXT:    subs x9, x9, #1
-; CHECK-NEXT:    mul w10, w10, w8
-; CHECK-NEXT:    str w10, [x11], #4
-; CHECK-NEXT:    b.ne .LBB4_7
-; CHECK-NEXT:  .LBB4_8: // %for.cond.cleanup
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: larger_umull:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmp w3, #1
+; CHECK-SD-NEXT:    b.lt .LBB4_8
+; CHECK-SD-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-SD-NEXT:    cmp w3, #15
+; CHECK-SD-NEXT:    and w8, w1, #0xffff
+; CHECK-SD-NEXT:    mov w9, w3
+; CHECK-SD-NEXT:    b.hi .LBB4_3
+; CHECK-SD-NEXT:  // %bb.2:
+; CHECK-SD-NEXT:    mov x10, xzr
+; CHECK-SD-NEXT:    b .LBB4_6
+; CHECK-SD-NEXT:  .LBB4_3: // %vector.ph
+; CHECK-SD-NEXT:    dup v0.8h, w8
+; CHECK-SD-NEXT:    and x10, x9, #0xfffffff0
+; CHECK-SD-NEXT:    add x11, x2, #32
+; CHECK-SD-NEXT:    add x12, x0, #16
+; CHECK-SD-NEXT:    mov x13, x10
+; CHECK-SD-NEXT:  .LBB4_4: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldp q1, q2, [x12, #-16]
+; CHECK-SD-NEXT:    subs x13, x13, #16
+; CHECK-SD-NEXT:    add x12, x12, #32
+; CHECK-SD-NEXT:    umull2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umull2 v4.4s, v0.8h, v2.8h
+; CHECK-SD-NEXT:    umull v2.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    stp q1, q3, [x11, #-32]
+; CHECK-SD-NEXT:    stp q2, q4, [x11], #64
+; CHECK-SD-NEXT:    b.ne .LBB4_4
+; CHECK-SD-NEXT:  // %bb.5: // %middle.block
+; CHECK-SD-NEXT:    cmp x10, x9
+; CHECK-SD-NEXT:    b.eq .LBB4_8
+; CHECK-SD-NEXT:  .LBB4_6: // %for.body.preheader1
+; CHECK-SD-NEXT:    add x11, x2, x10, lsl #2
+; CHECK-SD-NEXT:    add x12, x0, x10, lsl #1
+; CHECK-SD-NEXT:    sub x9, x9, x10
+; CHECK-SD-NEXT:  .LBB4_7: // %for.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldrh w10, [x12], #2
+; CHECK-SD-NEXT:    subs x9, x9, #1
+; CHECK-SD-NEXT:    mul w10, w10, w8
+; CHECK-SD-NEXT:    str w10, [x11], #4
+; CHECK-SD-NEXT:    b.ne .LBB4_7
+; CHECK-SD-NEXT:  .LBB4_8: // %for.cond.cleanup
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: larger_umull:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmp w3, #0
+; CHECK-GI-NEXT:    b.le .LBB4_7
+; CHECK-GI-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-GI-NEXT:    mov x9, xzr
+; CHECK-GI-NEXT:    cmp w3, #16
+; CHECK-GI-NEXT:    and w8, w1, #0xffff
+; CHECK-GI-NEXT:    mov w10, w3
+; CHECK-GI-NEXT:    b.lo .LBB4_5
+; CHECK-GI-NEXT:  // %bb.2: // %vector.ph
+; CHECK-GI-NEXT:    dup v0.4s, w8
+; CHECK-GI-NEXT:    and x9, x10, #0xfffffff0
+; CHECK-GI-NEXT:    add x11, x2, #32
+; CHECK-GI-NEXT:    add x12, x0, #16
+; CHECK-GI-NEXT:    mov x13, x9
+; CHECK-GI-NEXT:  .LBB4_3: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldp q1, q2, [x12, #-16]
+; CHECK-GI-NEXT:    mov x14, x11
+; CHECK-GI-NEXT:    subs x13, x13, #16
+; CHECK-GI-NEXT:    add x12, x12, #32
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    ushll v4.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    mul v3.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v4.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q3, q1, [x14, #-32]!
+; CHECK-GI-NEXT:    stp q4, q2, [x11], #64
+; CHECK-GI-NEXT:    b.ne .LBB4_3
+; CHECK-GI-NEXT:  // %bb.4: // %middle.block
+; CHECK-GI-NEXT:    cmp x9, x10
+; CHECK-GI-NEXT:    b.eq .LBB4_7
+; CHECK-GI-NEXT:  .LBB4_5: // %for.body.preheader1
+; CHECK-GI-NEXT:    add x11, x2, x9, lsl #2
+; CHECK-GI-NEXT:    add x12, x0, x9, lsl #1
+; CHECK-GI-NEXT:    sub x9, x10, x9
+; CHECK-GI-NEXT:  .LBB4_6: // %for.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldrh w10, [x12], #2
+; CHECK-GI-NEXT:    subs x9, x9, #1
+; CHECK-GI-NEXT:    mul w10, w10, w8
+; CHECK-GI-NEXT:    str w10, [x11], #4
+; CHECK-GI-NEXT:    b.ne .LBB4_6
+; CHECK-GI-NEXT:  .LBB4_7: // %for.cond.cleanup
+; CHECK-GI-NEXT:    ret
 entry:
   %conv1 = zext i16 %y to i32
   %cmp8 = icmp sgt i32 %n, 0
@@ -411,57 +580,116 @@ for.body:                                         ; preds = %for.body.preheader1
 
 
 define i16 @red_mla_dup_ext_u8_s8_s16(ptr noalias nocapture noundef readonly %A, i8 noundef %B, i32 noundef %n) {
-; CHECK-LABEL: red_mla_dup_ext_u8_s8_s16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cbz w2, .LBB5_3
-; CHECK-NEXT:  // %bb.1: // %for.body.preheader
-; CHECK-NEXT:    sxtb w9, w1
-; CHECK-NEXT:    cmp w2, #15
-; CHECK-NEXT:    mov w10, w2
-; CHECK-NEXT:    b.hi .LBB5_4
-; CHECK-NEXT:  // %bb.2:
-; CHECK-NEXT:    mov x11, xzr
-; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    b .LBB5_7
-; CHECK-NEXT:  .LBB5_3:
-; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    mov w0, w8
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB5_4: // %vector.ph
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    and x11, x10, #0xfffffff0
-; CHECK-NEXT:    dup v2.8h, w9
-; CHECK-NEXT:    add x8, x0, #8
-; CHECK-NEXT:    mov x12, x11
-; CHECK-NEXT:  .LBB5_5: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldp d3, d4, [x8, #-8]
-; CHECK-NEXT:    subs x12, x12, #16
-; CHECK-NEXT:    add x8, x8, #16
-; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
-; CHECK-NEXT:    mla v0.8h, v2.8h, v3.8h
-; CHECK-NEXT:    mla v1.8h, v2.8h, v4.8h
-; CHECK-NEXT:    b.ne .LBB5_5
-; CHECK-NEXT:  // %bb.6: // %middle.block
-; CHECK-NEXT:    add v0.8h, v1.8h, v0.8h
-; CHECK-NEXT:    cmp x11, x10
-; CHECK-NEXT:    addv h0, v0.8h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    b.eq .LBB5_9
-; CHECK-NEXT:  .LBB5_7: // %for.body.preheader1
-; CHECK-NEXT:    sub x10, x10, x11
-; CHECK-NEXT:    add x11, x0, x11
-; CHECK-NEXT:  .LBB5_8: // %for.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrb w12, [x11], #1
-; CHECK-NEXT:    subs x10, x10, #1
-; CHECK-NEXT:    madd w8, w12, w9, w8
-; CHECK-NEXT:    b.ne .LBB5_8
-; CHECK-NEXT:  .LBB5_9: // %for.cond.cleanup
-; CHECK-NEXT:    mov w0, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: red_mla_dup_ext_u8_s8_s16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cbz w2, .LBB5_3
+; CHECK-SD-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-SD-NEXT:    sxtb w9, w1
+; CHECK-SD-NEXT:    cmp w2, #15
+; CHECK-SD-NEXT:    mov w10, w2
+; CHECK-SD-NEXT:    b.hi .LBB5_4
+; CHECK-SD-NEXT:  // %bb.2:
+; CHECK-SD-NEXT:    mov x11, xzr
+; CHECK-SD-NEXT:    mov w8, wzr
+; CHECK-SD-NEXT:    b .LBB5_7
+; CHECK-SD-NEXT:  .LBB5_3:
+; CHECK-SD-NEXT:    mov w8, wzr
+; CHECK-SD-NEXT:    mov w0, w8
+; CHECK-SD-NEXT:    ret
+; CHECK-SD-NEXT:  .LBB5_4: // %vector.ph
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    and x11, x10, #0xfffffff0
+; CHECK-SD-NEXT:    dup v2.8h, w9
+; CHECK-SD-NEXT:    add x8, x0, #8
+; CHECK-SD-NEXT:    mov x12, x11
+; CHECK-SD-NEXT:  .LBB5_5: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldp d3, d4, [x8, #-8]
+; CHECK-SD-NEXT:    subs x12, x12, #16
+; CHECK-SD-NEXT:    add x8, x8, #16
+; CHECK-SD-NEXT:    ushll v3.8h, v3.8b, #0
+; CHECK-SD-NEXT:    ushll v4.8h, v4.8b, #0
+; CHECK-SD-NEXT:    mla v0.8h, v2.8h, v3.8h
+; CHECK-SD-NEXT:    mla v1.8h, v2.8h, v4.8h
+; CHECK-SD-NEXT:    b.ne .LBB5_5
+; CHECK-SD-NEXT:  // %bb.6: // %middle.block
+; CHECK-SD-NEXT:    add v0.8h, v1.8h, v0.8h
+; CHECK-SD-NEXT:    cmp x11, x10
+; CHECK-SD-NEXT:    addv h0, v0.8h
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    b.eq .LBB5_9
+; CHECK-SD-NEXT:  .LBB5_7: // %for.body.preheader1
+; CHECK-SD-NEXT:    sub x10, x10, x11
+; CHECK-SD-NEXT:    add x11, x0, x11
+; CHECK-SD-NEXT:  .LBB5_8: // %for.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldrb w12, [x11], #1
+; CHECK-SD-NEXT:    subs x10, x10, #1
+; CHECK-SD-NEXT:    madd w8, w12, w9, w8
+; CHECK-SD-NEXT:    b.ne .LBB5_8
+; CHECK-SD-NEXT:  .LBB5_9: // %for.cond.cleanup
+; CHECK-SD-NEXT:    mov w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: red_mla_dup_ext_u8_s8_s16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cbz w2, .LBB5_3
+; CHECK-GI-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-GI-NEXT:    cmp w2, #16
+; CHECK-GI-NEXT:    mov w8, w2
+; CHECK-GI-NEXT:    b.hs .LBB5_4
+; CHECK-GI-NEXT:  // %bb.2:
+; CHECK-GI-NEXT:    mov w10, #0 // =0x0
+; CHECK-GI-NEXT:    mov x9, xzr
+; CHECK-GI-NEXT:    fmov s0, w10
+; CHECK-GI-NEXT:    b .LBB5_8
+; CHECK-GI-NEXT:  .LBB5_3:
+; CHECK-GI-NEXT:    mov w0, wzr
+; CHECK-GI-NEXT:    ret
+; CHECK-GI-NEXT:  .LBB5_4: // %vector.ph
+; CHECK-GI-NEXT:    lsl w9, w1, #8
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    add x10, x0, #8
+; CHECK-GI-NEXT:    sxth w9, w9
+; CHECK-GI-NEXT:    asr w9, w9, #8
+; CHECK-GI-NEXT:    dup v2.8h, w9
+; CHECK-GI-NEXT:    and x9, x8, #0xfffffff0
+; CHECK-GI-NEXT:    mov x11, x9
+; CHECK-GI-NEXT:  .LBB5_5: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldp d3, d4, [x10, #-8]
+; CHECK-GI-NEXT:    subs x11, x11, #16
+; CHECK-GI-NEXT:    add x10, x10, #16
+; CHECK-GI-NEXT:    ushll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT:    ushll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT:    mla v0.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT:    mla v1.8h, v2.8h, v4.8h
+; CHECK-GI-NEXT:    b.ne .LBB5_5
+; CHECK-GI-NEXT:  // %bb.6: // %middle.block
+; CHECK-GI-NEXT:    add v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    cmp x9, x8
+; CHECK-GI-NEXT:    addv h0, v0.8h
+; CHECK-GI-NEXT:    b.ne .LBB5_8
+; CHECK-GI-NEXT:  // %bb.7:
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
+; CHECK-GI-NEXT:  .LBB5_8: // %for.body.preheader1
+; CHECK-GI-NEXT:    sxtb w10, w1
+; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    add x9, x0, x9
+; CHECK-GI-NEXT:  .LBB5_9: // %for.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldrb w11, [x9], #1
+; CHECK-GI-NEXT:    fmov w12, s0
+; CHECK-GI-NEXT:    subs x8, x8, #1
+; CHECK-GI-NEXT:    mul w11, w11, w10
+; CHECK-GI-NEXT:    add w0, w11, w12, uxth
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    b.ne .LBB5_9
+; CHECK-GI-NEXT:  // %bb.10: // %for.cond.cleanup
+; CHECK-GI-NEXT:    ret
 entry:
   %conv2 = sext i8 %B to i16
   %cmp10.not = icmp eq i32 %n, 0
@@ -529,21 +757,46 @@ for.body:                                         ; preds = %for.body.preheader1
 }
 
 define void @sink_v2z64_1(ptr %p, ptr %d, i64 %n, <2 x i32> %a) {
-; CHECK-LABEL: sink_v2z64_1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:  .LBB6_1: // %loop
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    subs x2, x2, #8
-; CHECK-NEXT:    add x8, x8, #8
-; CHECK-NEXT:    umull v1.2d, v1.2s, v0.s[1]
-; CHECK-NEXT:    shrn v1.2s, v1.2d, #15
-; CHECK-NEXT:    str d1, [x0], #32
-; CHECK-NEXT:    b.ne .LBB6_1
-; CHECK-NEXT:  // %bb.2: // %exit
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sink_v2z64_1:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, xzr
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:  .LBB6_1: // %loop
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldr d1, [x0]
+; CHECK-SD-NEXT:    subs x2, x2, #8
+; CHECK-SD-NEXT:    add x8, x8, #8
+; CHECK-SD-NEXT:    umull v1.2d, v1.2s, v0.s[1]
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #15
+; CHECK-SD-NEXT:    str d1, [x0], #32
+; CHECK-SD-NEXT:    b.ne .LBB6_1
+; CHECK-SD-NEXT:  // %bb.2: // %exit
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sink_v2z64_1:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov x8, xzr
+; CHECK-GI-NEXT:    dup v0.2d, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:  .LBB6_1: // %loop
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldr d0, [x0]
+; CHECK-GI-NEXT:    subs x2, x2, #8
+; CHECK-GI-NEXT:    add x8, x8, #8
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    fmov x11, d0
+; CHECK-GI-NEXT:    mov x12, v0.d[1]
+; CHECK-GI-NEXT:    mul x11, x11, x10
+; CHECK-GI-NEXT:    mul x12, x12, x9
+; CHECK-GI-NEXT:    mov v0.d[0], x11
+; CHECK-GI-NEXT:    mov v0.d[1], x12
+; CHECK-GI-NEXT:    shrn v0.2s, v0.2d, #15
+; CHECK-GI-NEXT:    str d0, [x0], #32
+; CHECK-GI-NEXT:    b.ne .LBB6_1
+; CHECK-GI-NEXT:  // %bb.2: // %exit
+; CHECK-GI-NEXT:    ret
 entry:
   %ext = zext <2 x i32> %a to <2 x i64>
   %broadcast.splat = shufflevector <2 x i64> %ext, <2 x i64> poison, <2 x i32> <i32 1, i32 1>
@@ -570,23 +823,56 @@ exit:
 }
 
 define void @sink_v4i64_1(ptr %p, ptr %d, i64 %n, <2 x i32> %a) {
-; CHECK-LABEL: sink_v4i64_1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:  .LBB7_1: // %loop
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr q1, [x0]
-; CHECK-NEXT:    subs x2, x2, #8
-; CHECK-NEXT:    add x8, x8, #8
-; CHECK-NEXT:    smull v2.2d, v1.2s, v0.s[1]
-; CHECK-NEXT:    smull2 v1.2d, v1.4s, v0.s[1]
-; CHECK-NEXT:    shrn v2.2s, v2.2d, #15
-; CHECK-NEXT:    shrn2 v2.4s, v1.2d, #15
-; CHECK-NEXT:    str q2, [x0], #32
-; CHECK-NEXT:    b.ne .LBB7_1
-; CHECK-NEXT:  // %bb.2: // %exit
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sink_v4i64_1:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, xzr
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:  .LBB7_1: // %loop
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldr q1, [x0]
+; CHECK-SD-NEXT:    subs x2, x2, #8
+; CHECK-SD-NEXT:    add x8, x8, #8
+; CHECK-SD-NEXT:    smull v2.2d, v1.2s, v0.s[1]
+; CHECK-SD-NEXT:    smull2 v1.2d, v1.4s, v0.s[1]
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #15
+; CHECK-SD-NEXT:    shrn2 v2.4s, v1.2d, #15
+; CHECK-SD-NEXT:    str q2, [x0], #32
+; CHECK-SD-NEXT:    b.ne .LBB7_1
+; CHECK-SD-NEXT:  // %bb.2: // %exit
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sink_v4i64_1:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    mov x8, xzr
+; CHECK-GI-NEXT:    dup v0.2d, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:  .LBB7_1: // %loop
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldr q0, [x0]
+; CHECK-GI-NEXT:    subs x2, x2, #8
+; CHECK-GI-NEXT:    add x8, x8, #8
+; CHECK-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x12, v1.d[1]
+; CHECK-GI-NEXT:    fmov x13, d0
+; CHECK-GI-NEXT:    mov x14, v0.d[1]
+; CHECK-GI-NEXT:    mul x11, x11, x10
+; CHECK-GI-NEXT:    mul x13, x13, x10
+; CHECK-GI-NEXT:    mul x12, x12, x9
+; CHECK-GI-NEXT:    mov v0.d[0], x11
+; CHECK-GI-NEXT:    mul x11, x14, x9
+; CHECK-GI-NEXT:    mov v1.d[0], x13
+; CHECK-GI-NEXT:    mov v0.d[1], x12
+; CHECK-GI-NEXT:    mov v1.d[1], x11
+; CHECK-GI-NEXT:    shrn v0.2s, v0.2d, #15
+; CHECK-GI-NEXT:    shrn2 v0.4s, v1.2d, #15
+; CHECK-GI-NEXT:    str q0, [x0], #32
+; CHECK-GI-NEXT:    b.ne .LBB7_1
+; CHECK-GI-NEXT:  // %bb.2: // %exit
+; CHECK-GI-NEXT:    ret
 entry:
   %ext = sext <2 x i32> %a to <2 x i64>
   %broadcast.splat = shufflevector <2 x i64> %ext, <2 x i64> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
@@ -613,22 +899,40 @@ exit:
 }
 
 define void @sink_v8z16_0(ptr %p, ptr %d, i64 %n, <16 x i8> %a) {
-; CHECK-LABEL: sink_v8z16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v0.8b, v0.b[0]
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:  .LBB8_1: // %loop
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    subs x2, x2, #8
-; CHECK-NEXT:    add x8, x8, #8
-; CHECK-NEXT:    umull v1.8h, v1.8b, v0.8b
-; CHECK-NEXT:    cmlt v1.8h, v1.8h, #0
-; CHECK-NEXT:    xtn v1.8b, v1.8h
-; CHECK-NEXT:    str d1, [x0], #32
-; CHECK-NEXT:    b.ne .LBB8_1
-; CHECK-NEXT:  // %bb.2: // %exit
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sink_v8z16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-SD-NEXT:    mov x8, xzr
+; CHECK-SD-NEXT:  .LBB8_1: // %loop
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldr d1, [x0]
+; CHECK-SD-NEXT:    subs x2, x2, #8
+; CHECK-SD-NEXT:    add x8, x8, #8
+; CHECK-SD-NEXT:    umull v1.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-SD-NEXT:    xtn v1.8b, v1.8h
+; CHECK-SD-NEXT:    str d1, [x0], #32
+; CHECK-SD-NEXT:    b.ne .LBB8_1
+; CHECK-SD-NEXT:  // %bb.2: // %exit
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sink_v8z16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov x8, xzr
+; CHECK-GI-NEXT:  .LBB8_1: // %loop
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldr d1, [x0]
+; CHECK-GI-NEXT:    subs x2, x2, #8
+; CHECK-GI-NEXT:    add x8, x8, #8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mul v1.8h, v1.8h, v0.h[0]
+; CHECK-GI-NEXT:    sshr v1.8h, v1.8h, #15
+; CHECK-GI-NEXT:    xtn v1.8b, v1.8h
+; CHECK-GI-NEXT:    str d1, [x0], #32
+; CHECK-GI-NEXT:    b.ne .LBB8_1
+; CHECK-GI-NEXT:  // %bb.2: // %exit
+; CHECK-GI-NEXT:    ret
 entry:
   %ext = zext <16 x i8> %a to <16 x i16>
   %broadcast.splat = shufflevector <16 x i16> %ext, <16 x i16> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
@@ -655,24 +959,45 @@ exit:
 }
 
 define void @sink_v16s16_8(ptr %p, ptr %d, i64 %n, <16 x i8> %a) {
-; CHECK-LABEL: sink_v16s16_8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    dup v0.16b, v0.b[10]
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:  .LBB9_1: // %loop
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr q1, [x0]
-; CHECK-NEXT:    subs x2, x2, #8
-; CHECK-NEXT:    add x8, x8, #8
-; CHECK-NEXT:    smull v2.8h, v1.8b, v0.8b
-; CHECK-NEXT:    smull2 v1.8h, v1.16b, v0.16b
-; CHECK-NEXT:    cmlt v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmlt v2.8h, v2.8h, #0
-; CHECK-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    str q1, [x0], #32
-; CHECK-NEXT:    b.ne .LBB9_1
-; CHECK-NEXT:  // %bb.2: // %exit
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sink_v16s16_8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    dup v0.16b, v0.b[10]
+; CHECK-SD-NEXT:    mov x8, xzr
+; CHECK-SD-NEXT:  .LBB9_1: // %loop
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldr q1, [x0]
+; CHECK-SD-NEXT:    subs x2, x2, #8
+; CHECK-SD-NEXT:    add x8, x8, #8
+; CHECK-SD-NEXT:    smull v2.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT:    smull2 v1.8h, v1.16b, v0.16b
+; CHECK-SD-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-SD-NEXT:    cmlt v2.8h, v2.8h, #0
+; CHECK-SD-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT:    str q1, [x0], #32
+; CHECK-SD-NEXT:    b.ne .LBB9_1
+; CHECK-SD-NEXT:  // %bb.2: // %exit
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sink_v16s16_8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT:    mov x8, xzr
+; CHECK-GI-NEXT:  .LBB9_1: // %loop
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldr q1, [x0]
+; CHECK-GI-NEXT:    subs x2, x2, #8
+; CHECK-GI-NEXT:    add x8, x8, #8
+; CHECK-GI-NEXT:    sshll v2.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT:    mul v2.8h, v2.8h, v0.h[2]
+; CHECK-GI-NEXT:    mul v1.8h, v1.8h, v0.h[2]
+; CHECK-GI-NEXT:    sshr v2.8h, v2.8h, #15
+; CHECK-GI-NEXT:    sshr v1.8h, v1.8h, #15
+; CHECK-GI-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
+; CHECK-GI-NEXT:    str q1, [x0], #32
+; CHECK-GI-NEXT:    b.ne .LBB9_1
+; CHECK-GI-NEXT:  // %bb.2: // %exit
+; CHECK-GI-NEXT:    ret
 entry:
   %ext = sext <16 x i8> %a to <16 x i16>
   %broadcast.splat = shufflevector <16 x i16> %ext, <16 x i16> poison, <16 x i32> <i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
@@ -699,25 +1024,47 @@ exit:
 }
 
 define void @matrix_mul_unsigned_and(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
-; CHECK-LABEL: matrix_mul_unsigned_and:
-; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    and w8, w3, #0xffff
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4h, w8
-; CHECK-NEXT:    and x8, x0, #0xfffffff8
-; CHECK-NEXT:  .LBB10_1: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x9, x2, w0, uxtw #1
-; CHECK-NEXT:    subs x8, x8, #8
-; CHECK-NEXT:    ldp d1, d2, [x9]
-; CHECK-NEXT:    add x9, x1, w0, uxtw #2
-; CHECK-NEXT:    add w0, w0, #8
-; CHECK-NEXT:    umull v1.4s, v0.4h, v1.4h
-; CHECK-NEXT:    umull v2.4s, v0.4h, v2.4h
-; CHECK-NEXT:    stp q1, q2, [x9]
-; CHECK-NEXT:    b.ne .LBB10_1
-; CHECK-NEXT:  // %bb.2: // %for.end12
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: matrix_mul_unsigned_and:
+; CHECK-SD:       // %bb.0: // %vector.header
+; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    dup v0.4h, w8
+; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
+; CHECK-SD-NEXT:  .LBB10_1: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-SD-NEXT:    subs x8, x8, #8
+; CHECK-SD-NEXT:    ldp d1, d2, [x9]
+; CHECK-SD-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-SD-NEXT:    add w0, w0, #8
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umull v2.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    stp q1, q2, [x9]
+; CHECK-SD-NEXT:    b.ne .LBB10_1
+; CHECK-SD-NEXT:  // %bb.2: // %for.end12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: matrix_mul_unsigned_and:
+; CHECK-GI:       // %bb.0: // %vector.header
+; CHECK-GI-NEXT:    and w8, w3, #0xffff
+; CHECK-GI-NEXT:    dup v0.4s, w8
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    and x8, x8, #0xfffffff8
+; CHECK-GI-NEXT:  .LBB10_1: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-GI-NEXT:    subs x8, x8, #8
+; CHECK-GI-NEXT:    ldp d1, d2, [x9]
+; CHECK-GI-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-GI-NEXT:    add w0, w0, #8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q1, q2, [x9]
+; CHECK-GI-NEXT:    b.ne .LBB10_1
+; CHECK-GI-NEXT:  // %bb.2: // %for.end12
+; CHECK-GI-NEXT:    ret
 vector.header:
   %conv4 = and i32 %val, 65535
   %wide.trip.count = zext i32 %N to i64
@@ -763,29 +1110,57 @@ for.end12:                                        ; preds = %vector.body
 }
 
 define void @matrix_mul_unsigned_and_double(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
-; CHECK-LABEL: matrix_mul_unsigned_and_double:
-; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    and w8, w3, #0xffff
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.8h, w8
-; CHECK-NEXT:    and x8, x0, #0xfffffff0
-; CHECK-NEXT:  .LBB11_1: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x9, x2, w0, uxtw #1
-; CHECK-NEXT:    subs x8, x8, #16
-; CHECK-NEXT:    ldr q1, [x9]
-; CHECK-NEXT:    ldur q2, [x9, #8]
-; CHECK-NEXT:    add x9, x1, w0, uxtw #2
-; CHECK-NEXT:    add w0, w0, #16
-; CHECK-NEXT:    umull2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT:    umull v1.4s, v0.4h, v1.4h
-; CHECK-NEXT:    umull2 v4.4s, v0.8h, v2.8h
-; CHECK-NEXT:    umull v2.4s, v0.4h, v2.4h
-; CHECK-NEXT:    stp q1, q3, [x9]
-; CHECK-NEXT:    stp q2, q4, [x9, #32]
-; CHECK-NEXT:    b.ne .LBB11_1
-; CHECK-NEXT:  // %bb.2: // %for.end12
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: matrix_mul_unsigned_and_double:
+; CHECK-SD:       // %bb.0: // %vector.header
+; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    dup v0.8h, w8
+; CHECK-SD-NEXT:    and x8, x0, #0xfffffff0
+; CHECK-SD-NEXT:  .LBB11_1: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-SD-NEXT:    subs x8, x8, #16
+; CHECK-SD-NEXT:    ldr q1, [x9]
+; CHECK-SD-NEXT:    ldur q2, [x9, #8]
+; CHECK-SD-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-SD-NEXT:    add w0, w0, #16
+; CHECK-SD-NEXT:    umull2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umull2 v4.4s, v0.8h, v2.8h
+; CHECK-SD-NEXT:    umull v2.4s, v0.4h, v2.4h
+; CHECK-SD-NEXT:    stp q1, q3, [x9]
+; CHECK-SD-NEXT:    stp q2, q4, [x9, #32]
+; CHECK-SD-NEXT:    b.ne .LBB11_1
+; CHECK-SD-NEXT:  // %bb.2: // %for.end12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: matrix_mul_unsigned_and_double:
+; CHECK-GI:       // %bb.0: // %vector.header
+; CHECK-GI-NEXT:    and w8, w3, #0xffff
+; CHECK-GI-NEXT:    dup v0.4s, w8
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    and x8, x8, #0xfffffff0
+; CHECK-GI-NEXT:  .LBB11_1: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-GI-NEXT:    subs x8, x8, #16
+; CHECK-GI-NEXT:    ldr q1, [x9]
+; CHECK-GI-NEXT:    ldur q2, [x9, #8]
+; CHECK-GI-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-GI-NEXT:    add w0, w0, #16
+; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    ushll v4.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    mul v3.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v4.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q3, q1, [x9]
+; CHECK-GI-NEXT:    stp q4, q2, [x9, #32]!
+; CHECK-GI-NEXT:    b.ne .LBB11_1
+; CHECK-GI-NEXT:  // %bb.2: // %for.end12
+; CHECK-GI-NEXT:    ret
 vector.header:
   %conv4 = and i32 %val, 65535
   %wide.trip.count = zext i32 %N to i64
@@ -831,27 +1206,49 @@ for.end12:                                        ; preds = %vector.body
 }
 
 define void @matrix_mul_signed_and(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
-; CHECK-LABEL: matrix_mul_signed_and:
-; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    and w8, w3, #0xffff
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4s, w8
-; CHECK-NEXT:    and x8, x0, #0xfffffff8
-; CHECK-NEXT:  .LBB12_1: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x9, x2, w0, uxtw #1
-; CHECK-NEXT:    subs x8, x8, #8
-; CHECK-NEXT:    ldp d1, d2, [x9]
-; CHECK-NEXT:    add x9, x1, w0, uxtw #2
-; CHECK-NEXT:    add w0, w0, #8
-; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-NEXT:    sshll v2.4s, v2.4h, #0
-; CHECK-NEXT:    mul v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    mul v2.4s, v0.4s, v2.4s
-; CHECK-NEXT:    stp q1, q2, [x9]
-; CHECK-NEXT:    b.ne .LBB12_1
-; CHECK-NEXT:  // %bb.2: // %for.end12
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: matrix_mul_signed_and:
+; CHECK-SD:       // %bb.0: // %vector.header
+; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    dup v0.4s, w8
+; CHECK-SD-NEXT:    and x8, x0, #0xfffffff8
+; CHECK-SD-NEXT:  .LBB12_1: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-SD-NEXT:    subs x8, x8, #8
+; CHECK-SD-NEXT:    ldp d1, d2, [x9]
+; CHECK-SD-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-SD-NEXT:    add w0, w0, #8
+; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    stp q1, q2, [x9]
+; CHECK-SD-NEXT:    b.ne .LBB12_1
+; CHECK-SD-NEXT:  // %bb.2: // %for.end12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: matrix_mul_signed_and:
+; CHECK-GI:       // %bb.0: // %vector.header
+; CHECK-GI-NEXT:    and w8, w3, #0xffff
+; CHECK-GI-NEXT:    dup v0.4s, w8
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    and x8, x8, #0xfffffff8
+; CHECK-GI-NEXT:  .LBB12_1: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-GI-NEXT:    subs x8, x8, #8
+; CHECK-GI-NEXT:    ldp d1, d2, [x9]
+; CHECK-GI-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-GI-NEXT:    add w0, w0, #8
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q1, q2, [x9]
+; CHECK-GI-NEXT:    b.ne .LBB12_1
+; CHECK-GI-NEXT:  // %bb.2: // %for.end12
+; CHECK-GI-NEXT:    ret
 vector.header:
   %conv4 = and i32 %val, 65535
   %wide.trip.count = zext i32 %N to i64
@@ -897,33 +1294,61 @@ for.end12:                                        ; preds = %vector.body
 }
 
 define void @matrix_mul_signed_and_double(i32 %N, ptr nocapture %C, ptr nocapture readonly %A, i32 %val) {
-; CHECK-LABEL: matrix_mul_signed_and_double:
-; CHECK:       // %bb.0: // %vector.header
-; CHECK-NEXT:    and w8, w3, #0xffff
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    dup v0.4s, w8
-; CHECK-NEXT:    and x8, x0, #0xfffffff0
-; CHECK-NEXT:  .LBB13_1: // %vector.body
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x9, x2, w0, uxtw #1
-; CHECK-NEXT:    subs x8, x8, #16
-; CHECK-NEXT:    ldr q1, [x9]
-; CHECK-NEXT:    ldur q2, [x9, #8]
-; CHECK-NEXT:    add x9, x1, w0, uxtw #2
-; CHECK-NEXT:    add w0, w0, #16
-; CHECK-NEXT:    sshll2 v3.4s, v1.8h, #0
-; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-NEXT:    sshll2 v4.4s, v2.8h, #0
-; CHECK-NEXT:    sshll v2.4s, v2.4h, #0
-; CHECK-NEXT:    mul v3.4s, v0.4s, v3.4s
-; CHECK-NEXT:    mul v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    mul v4.4s, v0.4s, v4.4s
-; CHECK-NEXT:    mul v2.4s, v0.4s, v2.4s
-; CHECK-NEXT:    stp q1, q3, [x9]
-; CHECK-NEXT:    stp q2, q4, [x9, #32]
-; CHECK-NEXT:    b.ne .LBB13_1
-; CHECK-NEXT:  // %bb.2: // %for.end12
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: matrix_mul_signed_and_double:
+; CHECK-SD:       // %bb.0: // %vector.header
+; CHECK-SD-NEXT:    and w8, w3, #0xffff
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    dup v0.4s, w8
+; CHECK-SD-NEXT:    and x8, x0, #0xfffffff0
+; CHECK-SD-NEXT:  .LBB13_1: // %vector.body
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-SD-NEXT:    subs x8, x8, #16
+; CHECK-SD-NEXT:    ldr q1, [x9]
+; CHECK-SD-NEXT:    ldur q2, [x9, #8]
+; CHECK-SD-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-SD-NEXT:    add w0, w0, #16
+; CHECK-SD-NEXT:    sshll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    sshll2 v4.4s, v2.8h, #0
+; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    mul v3.4s, v0.4s, v3.4s
+; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    mul v4.4s, v0.4s, v4.4s
+; CHECK-SD-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    stp q1, q3, [x9]
+; CHECK-SD-NEXT:    stp q2, q4, [x9, #32]
+; CHECK-SD-NEXT:    b.ne .LBB13_1
+; CHECK-SD-NEXT:  // %bb.2: // %for.end12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: matrix_mul_signed_and_double:
+; CHECK-GI:       // %bb.0: // %vector.header
+; CHECK-GI-NEXT:    and w8, w3, #0xffff
+; CHECK-GI-NEXT:    dup v0.4s, w8
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    and x8, x8, #0xfffffff0
+; CHECK-GI-NEXT:  .LBB13_1: // %vector.body
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    add x9, x2, w0, uxtw #1
+; CHECK-GI-NEXT:    subs x8, x8, #16
+; CHECK-GI-NEXT:    ldr q1, [x9]
+; CHECK-GI-NEXT:    ldur q2, [x9, #8]
+; CHECK-GI-NEXT:    add x9, x1, w0, uxtw #2
+; CHECK-GI-NEXT:    add w0, w0, #16
+; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT:    sshll v4.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    mul v3.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v4.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    stp q3, q1, [x9]
+; CHECK-GI-NEXT:    stp q4, q2, [x9, #32]!
+; CHECK-GI-NEXT:    b.ne .LBB13_1
+; CHECK-GI-NEXT:  // %bb.2: // %for.end12
+; CHECK-GI-NEXT:    ret
 vector.header:
   %conv4 = and i32 %val, 65535
   %wide.trip.count = zext i32 %N to i64
@@ -970,3 +1395,5 @@ for.end12:                                        ; preds = %vector.body
 
 declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
 
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll b/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
index cf913db187bc76489afe04a35000218e6a2a2956..c9864a357186dd5be58c676f5a8b96fd5dfa8b1d 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll
@@ -1,11 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-none-linux-gnu < %s -o -| FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define i64 @umull(i64 %x0, i64 %x1) {
-; CHECK-LABEL: umull:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    umull x0, w1, w0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umull x0, w1, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %and = and i64 %x0, 4294967295
   %and1 = and i64 %x1, 4294967295
@@ -14,10 +22,17 @@ entry:
 }
 
 define i64 @umull2(i64 %x, i32 %y) {
-; CHECK-LABEL: umull2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    umull x0, w0, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umull x0, w0, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %and = and i64 %x, 4294967295
   %conv = zext i32 %y to i64
@@ -26,10 +41,17 @@ entry:
 }
 
 define i64 @umull2_commuted(i64 %x, i32 %y) {
-; CHECK-LABEL: umull2_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    umull x0, w0, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull2_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umull x0, w0, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull2_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %and = and i64 %x, 4294967295
   %conv = zext i32 %y to i64
@@ -38,10 +60,17 @@ entry:
 }
 
 define i64 @smull(i64 %x0, i64 %x1) {
-; CHECK-LABEL: smull:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    smull x0, w1, w0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    smull x0, w1, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %sext = shl i64 %x0, 32
   %conv1 = ashr exact i64 %sext, 32
@@ -52,10 +81,18 @@ entry:
 }
 
 define i64 @smull2(i64 %x, i32 %y) {
-; CHECK-LABEL: smull2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    smull x0, w0, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    smull x0, w0, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %shl = shl i64 %x, 32
   %shr = ashr exact i64 %shl, 32
@@ -65,10 +102,18 @@ entry:
 }
 
 define i64 @smull2_commuted(i64 %x, i32 %y) {
-; CHECK-LABEL: smull2_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    smull x0, w0, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull2_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    smull x0, w0, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull2_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %shl = shl i64 %x, 32
   %shr = ashr exact i64 %shl, 32
@@ -78,13 +123,21 @@ entry:
 }
 
 define i64 @smull_ldrsb_b(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smull_ldrsb_b:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsb_b:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsb_b:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -94,13 +147,21 @@ entry:
 }
 
 define i64 @smull_ldrsb_b_commuted(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smull_ldrsb_b_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smull x0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsb_b_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smull x0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsb_b_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -110,13 +171,21 @@ entry:
 }
 
 define i64 @smull_ldrsb_h(ptr %x0, i16 %x1) {
-; CHECK-LABEL: smull_ldrsb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -126,11 +195,19 @@ entry:
 }
 
 define i64 @smull_ldrsb_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: smull_ldrsb_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    smull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsb_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    smull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsb_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -140,13 +217,21 @@ entry:
 }
 
 define i64 @smull_ldrsh_b(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smull_ldrsh_b:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsh_b:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsh_b:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -156,13 +241,21 @@ entry:
 }
 
 define i64 @smull_ldrsh_h(ptr %x0, i16 %x1) {
-; CHECK-LABEL: smull_ldrsh_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsh_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsh_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -172,13 +265,21 @@ entry:
 }
 
 define i64 @smull_ldrsh_h_commuted(ptr %x0, i16 %x1) {
-; CHECK-LABEL: smull_ldrsh_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smull x0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsh_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smull x0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsh_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -188,11 +289,19 @@ entry:
 }
 
 define i64 @smull_ldrsh_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: smull_ldrsh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    smull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    smull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -202,13 +311,21 @@ entry:
 }
 
 define i64 @smull_ldrsw_b(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smull_ldrsw_b:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_b:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_b:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -218,13 +335,21 @@ entry:
 }
 
 define i64 @smull_ldrsw_h(ptr %x0, i16 %x1) {
-; CHECK-LABEL: smull_ldrsw_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -234,11 +359,19 @@ entry:
 }
 
 define i64 @smull_ldrsw_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: smull_ldrsw_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    smull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    smull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -248,11 +381,19 @@ entry:
 }
 
 define i64 @smull_ldrsw_w_commuted(ptr %x0, i32 %x1) {
-; CHECK-LABEL: smull_ldrsw_w_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    smull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_w_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    smull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_w_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -262,14 +403,23 @@ entry:
 }
 
 define i64 @smull_sext_bb(i8 %x0, i8 %x1) {
-; CHECK-LABEL: smull_sext_bb:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    sxtb x8, w0
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_sext_bb:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxtb x8, w0
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_sext_bb:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    sxtb x8, w0
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %sext = sext i8 %x0 to i64
   %sext4 = sext i8 %x1 to i64
@@ -278,11 +428,18 @@ entry:
 }
 
 define i64 @smull_ldrsw_shift(ptr %x0, i64 %x1) {
-; CHECK-LABEL: smull_ldrsw_shift:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    smull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_shift:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    smull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_shift:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -308,13 +465,21 @@ entry:
 }
 
 define i64 @smull_ldrsw_zexth(ptr %x0, i16 %x1) {
-; CHECK-LABEL: smull_ldrsw_zexth:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_zexth:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_zexth:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -324,13 +489,21 @@ entry:
 }
 
 define i64 @smull_ldrsw_zextb(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smull_ldrsw_zextb:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_zextb:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_zextb:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -340,13 +513,21 @@ entry:
 }
 
 define i64 @smull_ldrsw_zextb_commuted(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smull_ldrsw_zextb_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    smull x0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ldrsw_zextb_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    smull x0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ldrsw_zextb_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -356,13 +537,21 @@ entry:
 }
 
 define i64 @smaddl_ldrsb_h(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -373,13 +562,21 @@ entry:
 }
 
 define i64 @smaddl_ldrsb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsb_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smaddl x0, w9, w8, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsb_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smaddl x0, w9, w8, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsb_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    madd x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -390,11 +587,19 @@ entry:
 }
 
 define i64 @smaddl_ldrsh_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    smaddl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    smaddl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -405,11 +610,19 @@ entry:
 }
 
 define i64 @smaddl_ldrsh_w_commuted(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsh_w_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    smaddl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsh_w_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    smaddl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsh_w_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    madd x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -420,13 +633,21 @@ entry:
 }
 
 define i64 @smaddl_ldrsw_b(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsw_b:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsw_b:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsw_b:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
   %sext2 = sext i8 %x1 to i64
@@ -436,13 +657,21 @@ define i64 @smaddl_ldrsw_b(ptr %x0, i8 %x1, i64 %x2) {
 }
 
 define i64 @smaddl_ldrsw_b_commuted(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsw_b_commuted:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smaddl x0, w9, w8, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsw_b_commuted:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smaddl x0, w9, w8, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsw_b_commuted:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    madd x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
   %sext2 = sext i8 %x1 to i64
@@ -452,12 +681,19 @@ define i64 @smaddl_ldrsw_b_commuted(ptr %x0, i8 %x1, i64 %x2) {
 }
 
 define i64 @smaddl_ldrsw_ldrsw(ptr %x0, ptr %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsw_ldrsw:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    ldrsw x9, [x1]
-; CHECK-NEXT:    smaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsw_ldrsw:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    ldrsw x9, [x1]
+; CHECK-SD-NEXT:    smaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsw_ldrsw:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    ldrsw x9, [x1]
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %ext64_2 = load i32, ptr %x1
@@ -469,14 +705,23 @@ entry:
 }
 
 define i64 @smaddl_sext_hh(i16 %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_sext_hh:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    sxth x8, w0
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_sext_hh:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxth x8, w0
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_sext_hh:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    sxth x8, w0
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %sext = sext i16 %x0 to i64
   %sext2 = sext i16 %x1 to i64
@@ -486,11 +731,18 @@ entry:
 }
 
 define i64 @smaddl_ldrsw_shift(ptr %x0, i64 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsw_shift:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    smaddl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsw_shift:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    smaddl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsw_shift:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -502,13 +754,21 @@ entry:
 }
 
 define i64 @smaddl_ldrsw_zextb(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: smaddl_ldrsw_zextb:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    smaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smaddl_ldrsw_zextb:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    smaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smaddl_ldrsw_zextb:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -519,13 +779,21 @@ entry:
 }
 
 define i64 @smnegl_ldrsb_h(ptr %x0, i16 %x1) {
-; CHECK-LABEL: smnegl_ldrsb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -536,13 +804,21 @@ entry:
 }
 
 define i64 @smnegl_ldrsb_h_commuted(ptr %x0, i16 %x1) {
-; CHECK-LABEL: smnegl_ldrsb_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smnegl x0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsb_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smnegl x0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsb_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    mneg x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -553,11 +829,19 @@ entry:
 }
 
 define i64 @smnegl_ldrsh_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: smnegl_ldrsh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    smnegl x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    smnegl x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -568,11 +852,19 @@ entry:
 }
 
 define i64 @smnegl_ldrsh_w_commuted(ptr %x0, i32 %x1) {
-; CHECK-LABEL: smnegl_ldrsh_w_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    smnegl x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsh_w_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    smnegl x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsh_w_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mneg x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -583,13 +875,21 @@ entry:
 }
 
 define i64 @smnegl_ldrsw_b(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smnegl_ldrsw_b:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsw_b:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsw_b:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
   %sext2 = sext i8 %x1 to i64
@@ -599,13 +899,21 @@ define i64 @smnegl_ldrsw_b(ptr %x0, i8 %x1) {
 }
 
 define i64 @smnegl_ldrsw_b_commuted(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smnegl_ldrsw_b_commuted:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smnegl x0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsw_b_commuted:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smnegl x0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsw_b_commuted:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    mneg x0, x9, x8
+; CHECK-GI-NEXT:    ret
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
   %sext2 = sext i8 %x1 to i64
@@ -615,12 +923,19 @@ define i64 @smnegl_ldrsw_b_commuted(ptr %x0, i8 %x1) {
 }
 
 define i64 @smnegl_ldrsw_ldrsw(ptr %x0, ptr %x1) {
-; CHECK-LABEL: smnegl_ldrsw_ldrsw:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    ldrsw x9, [x1]
-; CHECK-NEXT:    smnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsw_ldrsw:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    ldrsw x9, [x1]
+; CHECK-SD-NEXT:    smnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsw_ldrsw:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    ldrsw x9, [x1]
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %ext64_2 = load i32, ptr %x1
@@ -632,14 +947,23 @@ entry:
 }
 
 define i64 @smnegl_sext_hh(i16 %x0, i16 %x1) {
-; CHECK-LABEL: smnegl_sext_hh:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    sxth x8, w0
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_sext_hh:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxth x8, w0
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_sext_hh:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    sxth x8, w0
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %sext = sext i16 %x0 to i64
   %sext2 = sext i16 %x1 to i64
@@ -649,11 +973,18 @@ entry:
 }
 
 define i64 @smnegl_ldrsw_shift(ptr %x0, i64 %x1) {
-; CHECK-LABEL: smnegl_ldrsw_shift:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    smnegl x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsw_shift:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    smnegl x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsw_shift:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -665,13 +996,21 @@ entry:
 }
 
 define i64 @smnegl_ldrsw_zextb(ptr %x0, i8 %x1) {
-; CHECK-LABEL: smnegl_ldrsw_zextb:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    smnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smnegl_ldrsw_zextb:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    smnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smnegl_ldrsw_zextb:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -682,13 +1021,21 @@ entry:
 }
 
 define i64 @smsubl_ldrsb_h(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -699,13 +1046,21 @@ entry:
 }
 
 define i64 @smsubl_ldrsb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsb_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsb x8, [x0]
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smsubl x0, w9, w8, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsb_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsb x8, [x0]
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smsubl x0, w9, w8, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsb_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsb x8, [x0]
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    msub x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %sext = sext i8 %ext64 to i64
@@ -716,11 +1071,19 @@ entry:
 }
 
 define i64 @smsubl_ldrsh_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    smsubl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    smsubl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -731,11 +1094,19 @@ entry:
 }
 
 define i64 @smsubl_ldrsh_w_commuted(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsh_w_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsh x8, [x0]
-; CHECK-NEXT:    smsubl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsh_w_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsh x8, [x0]
+; CHECK-SD-NEXT:    smsubl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsh_w_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsh x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    msub x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %sext = sext i16 %ext64 to i64
@@ -746,13 +1117,21 @@ entry:
 }
 
 define i64 @smsubl_ldrsw_b(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsw_b:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsw_b:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsw_b:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
   %sext2 = sext i8 %x1 to i64
@@ -762,13 +1141,21 @@ define i64 @smsubl_ldrsw_b(ptr %x0, i8 %x1, i64 %x2) {
 }
 
 define i64 @smsubl_ldrsw_b_commuted(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsw_b_commuted:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    sxtb x9, w1
-; CHECK-NEXT:    smsubl x0, w9, w8, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsw_b_commuted:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    sxtb x9, w1
+; CHECK-SD-NEXT:    smsubl x0, w9, w8, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsw_b_commuted:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtb x9, w1
+; CHECK-GI-NEXT:    msub x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
   %sext2 = sext i8 %x1 to i64
@@ -778,12 +1165,19 @@ define i64 @smsubl_ldrsw_b_commuted(ptr %x0, i8 %x1, i64 %x2) {
 }
 
 define i64 @smsubl_ldrsw_ldrsw(ptr %x0, ptr %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsw_ldrsw:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    ldrsw x9, [x1]
-; CHECK-NEXT:    smsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsw_ldrsw:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    ldrsw x9, [x1]
+; CHECK-SD-NEXT:    smsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsw_ldrsw:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    ldrsw x9, [x1]
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %ext64_2 = load i32, ptr %x1
@@ -795,14 +1189,23 @@ entry:
 }
 
 define i64 @smsubl_sext_hh(i16 %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_sext_hh:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    sxth x8, w0
-; CHECK-NEXT:    sxth x9, w1
-; CHECK-NEXT:    smsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_sext_hh:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxth x8, w0
+; CHECK-SD-NEXT:    sxth x9, w1
+; CHECK-SD-NEXT:    smsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_sext_hh:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    sxth x8, w0
+; CHECK-GI-NEXT:    sxth x9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %sext = sext i16 %x0 to i64
   %sext2 = sext i16 %x1 to i64
@@ -812,11 +1215,18 @@ entry:
 }
 
 define i64 @smsubl_ldrsw_shift(ptr %x0, i64 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsw_shift:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    smsubl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsw_shift:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    smsubl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsw_shift:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    sxtw x9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -828,13 +1238,21 @@ entry:
 }
 
 define i64 @smsubl_ldrsw_zextb(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: smsubl_ldrsw_zextb:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrsw x8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    smsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smsubl_ldrsw_zextb:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrsw x8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    smsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smsubl_ldrsw_zextb:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrsw x8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %sext = sext i32 %ext64 to i64
@@ -860,11 +1278,19 @@ entry:
 }
 
 define i64 @smull_sext_ashr32(i32 %a, i64 %b) nounwind {
-; CHECK-LABEL: smull_sext_ashr32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    asr x8, x1, #32
-; CHECK-NEXT:    smull x0, w8, w0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_sext_ashr32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    asr x8, x1, #32
+; CHECK-SD-NEXT:    smull x0, w8, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_sext_ashr32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    asr x9, x1, #32
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %tmp1 = sext i32 %a to i64
   %c = ashr i64 %b, 32
@@ -888,12 +1314,19 @@ entry:
 }
 
 define i64 @smull_ashr32_both(i64 %a, i64 %b) nounwind {
-; CHECK-LABEL: smull_ashr32_both:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    asr x8, x0, #32
-; CHECK-NEXT:    asr x9, x1, #32
-; CHECK-NEXT:    smull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: smull_ashr32_both:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    asr x8, x0, #32
+; CHECK-SD-NEXT:    asr x9, x1, #32
+; CHECK-SD-NEXT:    smull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_ashr32_both:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    asr x8, x0, #32
+; CHECK-GI-NEXT:    asr x9, x1, #32
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %tmp1 = ashr i64 %a, 32
   %c = ashr i64 %b, 32
@@ -902,13 +1335,21 @@ entry:
 }
 
 define i64 @umull_ldrb_h(ptr %x0, i16 %x1) {
-; CHECK-LABEL: umull_ldrb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldrb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldrb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -918,13 +1359,21 @@ entry:
 }
 
 define i64 @umull_ldrb_h_commuted(ptr %x0, i16 %x1) {
-; CHECK-LABEL: umull_ldrb_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umull x0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldrb_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umull x0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldrb_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -934,11 +1383,18 @@ entry:
 }
 
 define i64 @umull_ldrh_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: umull_ldrh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    umull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldrh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    umull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldrh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrh w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -948,13 +1404,21 @@ entry:
 }
 
 define i64 @umull_ldr_b(ptr %x0, i8 %x1) {
-; CHECK-LABEL: umull_ldr_b:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    umull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldr_b:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    umull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldr_b:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -964,11 +1428,18 @@ entry:
 }
 
 define i64 @umull_ldr2_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: umull_ldr2_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldr2_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldr2_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -978,12 +1449,19 @@ entry:
 }
 
 define i64 @umull_ldr2_ldr2(ptr %x0, ptr %x1) {
-; CHECK-LABEL: umull_ldr2_ldr2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    umull x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldr2_ldr2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    ldr w9, [x1]
+; CHECK-SD-NEXT:    umull x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldr2_ldr2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -994,11 +1472,18 @@ entry:
 }
 
 define i64 @umull_ldr2_d(ptr %x0, i64 %x1) {
-; CHECK-LABEL: umull_ldr2_d:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldr2_d:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldr2_d:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1008,13 +1493,21 @@ entry:
 }
 
 define i64 @umaddl_ldrb_h(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: umaddl_ldrb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_ldrb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_ldrb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1025,13 +1518,21 @@ entry:
 }
 
 define i64 @umaddl_ldrb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: umaddl_ldrb_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umaddl x0, w9, w8, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_ldrb_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umaddl x0, w9, w8, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_ldrb_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    madd x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1042,11 +1543,18 @@ entry:
 }
 
 define i64 @umaddl_ldrh_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: umaddl_ldrh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    umaddl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_ldrh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    umaddl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_ldrh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrh w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -1057,13 +1565,21 @@ entry:
 }
 
 define i64 @umaddl_ldr_b(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: umaddl_ldr_b:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    umaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_ldr_b:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    umaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_ldr_b:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -1074,11 +1590,18 @@ entry:
 }
 
 define i64 @umaddl_ldr2_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: umaddl_ldr2_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umaddl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_ldr2_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umaddl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_ldr2_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1089,12 +1612,19 @@ entry:
 }
 
 define i64 @umaddl_ldr2_ldr2(ptr %x0, ptr %x1, i64 %x2) {
-; CHECK-LABEL: umaddl_ldr2_ldr2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    umaddl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_ldr2_ldr2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    ldr w9, [x1]
+; CHECK-SD-NEXT:    umaddl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_ldr2_ldr2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1106,11 +1636,18 @@ entry:
 }
 
 define i64 @umaddl_ldr2_d(ptr %x0, i64 %x1, i64 %x2) {
-; CHECK-LABEL: umaddl_ldr2_d:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umaddl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_ldr2_d:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umaddl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_ldr2_d:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1121,13 +1658,21 @@ entry:
 }
 
 define i64 @umnegl_ldrb_h(ptr %x0, i16 %x1) {
-; CHECK-LABEL: umnegl_ldrb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umnegl_ldrb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umnegl_ldrb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1138,13 +1683,21 @@ entry:
 }
 
 define i64 @umnegl_ldrb_h_commuted(ptr %x0, i16 %x1) {
-; CHECK-LABEL: umnegl_ldrb_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umnegl x0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umnegl_ldrb_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umnegl x0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umnegl_ldrb_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    mneg x0, x9, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1155,11 +1708,18 @@ entry:
 }
 
 define i64 @umnegl_ldrh_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: umnegl_ldrh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    umnegl x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umnegl_ldrh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    umnegl x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umnegl_ldrh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrh w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -1170,13 +1730,21 @@ entry:
 }
 
 define i64 @umnegl_ldr_b(ptr %x0, i8 %x1) {
-; CHECK-LABEL: umnegl_ldr_b:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    umnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umnegl_ldr_b:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    umnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umnegl_ldr_b:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -1187,11 +1755,18 @@ entry:
 }
 
 define i64 @umnegl_ldr2_w(ptr %x0, i32 %x1) {
-; CHECK-LABEL: umnegl_ldr2_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umnegl x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umnegl_ldr2_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umnegl x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umnegl_ldr2_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1202,12 +1777,19 @@ entry:
 }
 
 define i64 @umnegl_ldr2_ldr2(ptr %x0, ptr %x1) {
-; CHECK-LABEL: umnegl_ldr2_ldr2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    umnegl x0, w8, w9
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umnegl_ldr2_ldr2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    ldr w9, [x1]
+; CHECK-SD-NEXT:    umnegl x0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umnegl_ldr2_ldr2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1219,11 +1801,18 @@ entry:
 }
 
 define i64 @umnegl_ldr2_d(ptr %x0, i64 %x1) {
-; CHECK-LABEL: umnegl_ldr2_d:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umnegl x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umnegl_ldr2_d:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umnegl x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umnegl_ldr2_d:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1234,13 +1823,21 @@ entry:
 }
 
 define i64 @umsubl_ldrb_h(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: umsubl_ldrb_h:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umsubl_ldrb_h:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umsubl_ldrb_h:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1251,13 +1848,21 @@ entry:
 }
 
 define i64 @umsubl_ldrb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
-; CHECK-LABEL: umsubl_ldrb_h_commuted:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrb w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    umsubl x0, w9, w8, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umsubl_ldrb_h_commuted:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xffff
+; CHECK-SD-NEXT:    umsubl x0, w9, w8, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umsubl_ldrb_h_commuted:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xffff
+; CHECK-GI-NEXT:    msub x0, x9, x8, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
@@ -1268,11 +1873,18 @@ entry:
 }
 
 define i64 @umsubl_ldrh_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: umsubl_ldrh_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    umsubl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umsubl_ldrh_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    umsubl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umsubl_ldrh_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrh w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
@@ -1283,13 +1895,21 @@ entry:
 }
 
 define i64 @umsubl_ldr_b(ptr %x0, i8 %x1, i64 %x2) {
-; CHECK-LABEL: umsubl_ldr_b:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    umsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umsubl_ldr_b:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    and x9, x1, #0xff
+; CHECK-SD-NEXT:    umsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umsubl_ldr_b:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-GI-NEXT:    and x9, x1, #0xff
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
@@ -1300,11 +1920,18 @@ entry:
 }
 
 define i64 @umsubl_ldr2_w(ptr %x0, i32 %x1, i64 %x2) {
-; CHECK-LABEL: umsubl_ldr2_w:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umsubl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umsubl_ldr2_w:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umsubl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umsubl_ldr2_w:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1315,12 +1942,19 @@ entry:
 }
 
 define i64 @umsubl_ldr2_ldr2(ptr %x0, ptr %x1, i64 %x2) {
-; CHECK-LABEL: umsubl_ldr2_ldr2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    umsubl x0, w8, w9, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umsubl_ldr2_ldr2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    ldr w9, [x1]
+; CHECK-SD-NEXT:    umsubl x0, w8, w9, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umsubl_ldr2_ldr2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1332,11 +1966,18 @@ entry:
 }
 
 define i64 @umsubl_ldr2_d(ptr %x0, i64 %x1, i64 %x2) {
-; CHECK-LABEL: umsubl_ldr2_d:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    umsubl x0, w8, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umsubl_ldr2_d:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x0]
+; CHECK-SD-NEXT:    umsubl x0, w8, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umsubl_ldr2_d:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    msub x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
@@ -1347,12 +1988,20 @@ entry:
 }
 
 define i64 @umull_ldr2_w_cc1(ptr %x0, i32 %x1) {
-; CHECK-LABEL: umull_ldr2_w_cc1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr x8, [x0]
-; CHECK-NEXT:    and x8, x8, #0x7fffffff
-; CHECK-NEXT:    umull x0, w8, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_ldr2_w_cc1:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr x8, [x0]
+; CHECK-SD-NEXT:    and x8, x8, #0x7fffffff
+; CHECK-SD-NEXT:    umull x0, w8, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_ldr2_w_cc1:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr x8, [x0]
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    and x8, x8, #0x7fffffff
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 2147483647
@@ -1393,11 +2042,18 @@ entry:
 }
 
 define i64 @umull_and_lshr(i64 %x) {
-; CHECK-LABEL: umull_and_lshr:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsr x8, x0, #32
-; CHECK-NEXT:    umull x0, w0, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_and_lshr:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    lsr x8, x0, #32
+; CHECK-SD-NEXT:    umull x0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_and_lshr:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    lsr x8, x0, #32
+; CHECK-GI-NEXT:    mov w9, w0
+; CHECK-GI-NEXT:    mul x0, x9, x8
+; CHECK-GI-NEXT:    ret
     %lo = and i64 %x, u0xffffffff
     %hi = lshr i64 %x, 32
     %mul = mul i64 %lo, %hi
@@ -1405,10 +2061,17 @@ define i64 @umull_and_lshr(i64 %x) {
 }
 
 define i64 @umull_and_and(i64 %x, i64 %y) {
-; CHECK-LABEL: umull_and_and:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    umull x0, w0, w1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umull_and_and:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    umull x0, w0, w1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umull_and_and:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
     %lo = and i64 %x, u0xffffffff
     %hi = and i64 %y, u0xffffffff
     %mul = mul i64 %lo, %hi
@@ -1416,11 +2079,18 @@ define i64 @umull_and_and(i64 %x, i64 %y) {
 }
 
 define i64 @umaddl_and_lshr(i64 %x, i64 %a) {
-; CHECK-LABEL: umaddl_and_lshr:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    lsr x8, x0, #32
-; CHECK-NEXT:    umaddl x0, w0, w8, x1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_and_lshr:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    lsr x8, x0, #32
+; CHECK-SD-NEXT:    umaddl x0, w0, w8, x1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_and_lshr:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    lsr x8, x0, #32
+; CHECK-GI-NEXT:    mov w9, w0
+; CHECK-GI-NEXT:    madd x0, x9, x8, x1
+; CHECK-GI-NEXT:    ret
     %lo = and i64 %x, u0xffffffff
     %hi = lshr i64 %x, 32
     %mul = mul i64 %lo, %hi
@@ -1429,10 +2099,17 @@ define i64 @umaddl_and_lshr(i64 %x, i64 %a) {
 }
 
 define i64 @umaddl_and_and(i64 %x, i64 %y, i64 %a) {
-; CHECK-LABEL: umaddl_and_and:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    umaddl x0, w0, w1, x2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umaddl_and_and:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    umaddl x0, w0, w1, x2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umaddl_and_and:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov w8, w0
+; CHECK-GI-NEXT:    mov w9, w1
+; CHECK-GI-NEXT:    madd x0, x8, x9, x2
+; CHECK-GI-NEXT:    ret
     %lo = and i64 %x, u0xffffffff
     %hi = and i64 %y, u0xffffffff
     %mul = mul i64 %lo, %hi
@@ -1442,6 +2119,22 @@ define i64 @umaddl_and_and(i64 %x, i64 %y, i64 %a) {
 
 ; Check which can contain multiple copies that should all be removed.
 define i32 @f(i32 %0) {
+; CHECK-SD-LABEL: f:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    neg w8, w0
+; CHECK-SD-NEXT:  .LBB93_1: // %B
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    cbnz x8, .LBB93_1
+; CHECK-SD-NEXT:    b .LBB93_1
+;
+; CHECK-GI-LABEL: f:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    neg w8, w0
+; CHECK-GI-NEXT:  .LBB93_1: // %B
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    cbnz x8, .LBB93_1
+; CHECK-GI-NEXT:    b .LBB93_1
 entry:
   %1 = sext i32 %0 to i64
   br label %A
diff --git a/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll b/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll
index 410c2d9021d6d58114415c33e4bb38c19aae484d..f7e16b857f815247c598889f331363742ccbbd29 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple aarch64-unknown-linux-gnu | FileCheck %s --check-prefixes=CHECK-SD
-; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-GI
+; RUN: llc -mtriple aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; Tests for wider-than-legal extensions into mul/mla.
 
@@ -223,3 +223,5 @@ entry:
   %d = add <16 x i64> %m, %c
   ret <16 x i64> %d
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/arm64-mul.ll b/llvm/test/CodeGen/AArch64/arm64-mul.ll
index da978b49fff749c8909d4b0e112bc25af8a61bef..e4d2ce7ccdabc89cd36c7bacf7fb0eb537cf9487 100644
--- a/llvm/test/CodeGen/AArch64/arm64-mul.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-mul.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=arm64-eabi | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-eabi | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=arm64-eabi -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; rdar://9296808
 ; rdar://9349137
@@ -19,12 +20,24 @@ entry:
 }
 
 define i128 @t2(i64 %a, i64 %b) nounwind readnone ssp {
-; CHECK-LABEL: t2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mul x8, x0, x1
-; CHECK-NEXT:    smulh x1, x0, x1
-; CHECK-NEXT:    mov x0, x8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: t2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mul x8, x0, x1
+; CHECK-SD-NEXT:    smulh x1, x0, x1
+; CHECK-SD-NEXT:    mov x0, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: t2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    asr x9, x1, #63
+; CHECK-GI-NEXT:    asr x10, x0, #63
+; CHECK-GI-NEXT:    mul x8, x0, x1
+; CHECK-GI-NEXT:    mul x9, x0, x9
+; CHECK-GI-NEXT:    umulh x11, x0, x1
+; CHECK-GI-NEXT:    mov x0, x8
+; CHECK-GI-NEXT:    madd x9, x10, x1, x9
+; CHECK-GI-NEXT:    add x1, x9, x11
+; CHECK-GI-NEXT:    ret
 entry:
   %tmp1 = sext i64 %a to i128
   %tmp2 = sext i64 %b to i128
@@ -139,12 +152,20 @@ entry:
 
 ; Check the sext_inreg case.
 define i64 @t11(i64 %a) nounwind {
-; CHECK-LABEL: t11:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #29594 // =0x739a
-; CHECK-NEXT:    movk w8, #65499, lsl #16
-; CHECK-NEXT:    smnegl x0, w0, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: t11:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #29594 // =0x739a
+; CHECK-SD-NEXT:    movk w8, #65499, lsl #16
+; CHECK-SD-NEXT:    smnegl x0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: t11:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    mov x9, #-35942 // =0xffffffffffff739a
+; CHECK-GI-NEXT:    movk x9, #65499, lsl #16
+; CHECK-GI-NEXT:    mneg x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %tmp1 = trunc i64 %a to i32
   %tmp2 = sext i32 %tmp1 to i64
@@ -154,12 +175,20 @@ entry:
 }
 
 define i64 @t12(i64 %a, i64 %b) nounwind {
-; CHECK-LABEL: t12:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #35118 // =0x892e
-; CHECK-NEXT:    movk w8, #65008, lsl #16
-; CHECK-NEXT:    smaddl x0, w0, w8, x1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: t12:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #35118 // =0x892e
+; CHECK-SD-NEXT:    movk w8, #65008, lsl #16
+; CHECK-SD-NEXT:    smaddl x0, w0, w8, x1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: t12:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtw x8, w0
+; CHECK-GI-NEXT:    mov x9, #-30418 // =0xffffffffffff892e
+; CHECK-GI-NEXT:    movk x9, #65008, lsl #16
+; CHECK-GI-NEXT:    madd x0, x8, x9, x1
+; CHECK-GI-NEXT:    ret
 entry:
   %tmp1 = trunc i64 %a to i32
   %tmp2 = sext i32 %tmp1 to i64
diff --git a/llvm/test/CodeGen/AArch64/arm64-popcnt.ll b/llvm/test/CodeGen/AArch64/arm64-popcnt.ll
index f5ce73a366125b66e0685f2923b81543ee67e666..0030e9ce80abb4cceaf6bec2b14ed79cd7f15a8b 100644
--- a/llvm/test/CodeGen/AArch64/arm64-popcnt.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-popcnt.ll
@@ -15,7 +15,7 @@ define i32 @cnt32_advsimd(i32 %x) nounwind readnone {
 ; CHECK-NONEON-LABEL: cnt32_advsimd:
 ; CHECK-NONEON:       // %bb.0:
 ; CHECK-NONEON-NEXT:    lsr w9, w0, #1
-; CHECK-NONEON-NEXT:    mov w8, #16843009
+; CHECK-NONEON-NEXT:    mov w8, #16843009 // =0x1010101
 ; CHECK-NONEON-NEXT:    and w9, w9, #0x55555555
 ; CHECK-NONEON-NEXT:    sub w9, w0, w9
 ; CHECK-NONEON-NEXT:    lsr w10, w9, #2
@@ -50,7 +50,7 @@ define i32 @cnt32_advsimd_2(<2 x i32> %x) {
 ; CHECK-NONEON-LABEL: cnt32_advsimd_2:
 ; CHECK-NONEON:       // %bb.0:
 ; CHECK-NONEON-NEXT:    lsr w9, w0, #1
-; CHECK-NONEON-NEXT:    mov w8, #16843009
+; CHECK-NONEON-NEXT:    mov w8, #16843009 // =0x1010101
 ; CHECK-NONEON-NEXT:    and w9, w9, #0x55555555
 ; CHECK-NONEON-NEXT:    sub w9, w0, w9
 ; CHECK-NONEON-NEXT:    lsr w10, w9, #2
@@ -86,7 +86,7 @@ define i64 @cnt64_advsimd(i64 %x) nounwind readnone {
 ; CHECK-NONEON-LABEL: cnt64_advsimd:
 ; CHECK-NONEON:       // %bb.0:
 ; CHECK-NONEON-NEXT:    lsr x9, x0, #1
-; CHECK-NONEON-NEXT:    mov x8, #72340172838076673
+; CHECK-NONEON-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
 ; CHECK-NONEON-NEXT:    and x9, x9, #0x5555555555555555
 ; CHECK-NONEON-NEXT:    sub x9, x0, x9
 ; CHECK-NONEON-NEXT:    lsr x10, x9, #2
@@ -114,7 +114,7 @@ define i32 @cnt32(i32 %x) nounwind readnone noimplicitfloat {
 ; CHECK-LABEL: cnt32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr w9, w0, #1
-; CHECK-NEXT:    mov w8, #16843009
+; CHECK-NEXT:    mov w8, #16843009 // =0x1010101
 ; CHECK-NEXT:    and w9, w9, #0x55555555
 ; CHECK-NEXT:    sub w9, w0, w9
 ; CHECK-NEXT:    lsr w10, w9, #2
@@ -130,7 +130,7 @@ define i32 @cnt32(i32 %x) nounwind readnone noimplicitfloat {
 ; CHECK-NONEON-LABEL: cnt32:
 ; CHECK-NONEON:       // %bb.0:
 ; CHECK-NONEON-NEXT:    lsr w9, w0, #1
-; CHECK-NONEON-NEXT:    mov w8, #16843009
+; CHECK-NONEON-NEXT:    mov w8, #16843009 // =0x1010101
 ; CHECK-NONEON-NEXT:    and w9, w9, #0x55555555
 ; CHECK-NONEON-NEXT:    sub w9, w0, w9
 ; CHECK-NONEON-NEXT:    lsr w10, w9, #2
@@ -155,7 +155,7 @@ define i64 @cnt64(i64 %x) nounwind readnone noimplicitfloat {
 ; CHECK-LABEL: cnt64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr x9, x0, #1
-; CHECK-NEXT:    mov x8, #72340172838076673
+; CHECK-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
 ; CHECK-NEXT:    and x9, x9, #0x5555555555555555
 ; CHECK-NEXT:    sub x9, x0, x9
 ; CHECK-NEXT:    lsr x10, x9, #2
@@ -171,7 +171,7 @@ define i64 @cnt64(i64 %x) nounwind readnone noimplicitfloat {
 ; CHECK-NONEON-LABEL: cnt64:
 ; CHECK-NONEON:       // %bb.0:
 ; CHECK-NONEON-NEXT:    lsr x9, x0, #1
-; CHECK-NONEON-NEXT:    mov x8, #72340172838076673
+; CHECK-NONEON-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
 ; CHECK-NONEON-NEXT:    and x9, x9, #0x5555555555555555
 ; CHECK-NONEON-NEXT:    sub x9, x0, x9
 ; CHECK-NONEON-NEXT:    lsr x10, x9, #2
@@ -278,5 +278,59 @@ define i1 @ctpop32_ne_one(i32 %x) nounwind readnone {
   ret i1 %cmp
 }
 
+define i1 @ctpop32_eq_one_nonzero(i32 %x) {
+; CHECK-LABEL: ctpop32_eq_one_nonzero:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub w8, w0, #1
+; CHECK-NEXT:    tst w0, w8
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+;
+; CHECK-NONEON-LABEL: ctpop32_eq_one_nonzero:
+; CHECK-NONEON:       // %bb.0: // %entry
+; CHECK-NONEON-NEXT:    sub w8, w0, #1
+; CHECK-NONEON-NEXT:    tst w0, w8
+; CHECK-NONEON-NEXT:    cset w0, eq
+; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: ctpop32_eq_one_nonzero:
+; CHECK-CSSC:       // %bb.0: // %entry
+; CHECK-CSSC-NEXT:    sub w8, w0, #1
+; CHECK-CSSC-NEXT:    tst w0, w8
+; CHECK-CSSC-NEXT:    cset w0, eq
+; CHECK-CSSC-NEXT:    ret
+entry:
+  %popcnt = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp eq i32 %popcnt, 1
+  ret i1 %cmp
+}
+
+define i1 @ctpop32_ne_one_nonzero(i32 %x) {
+; CHECK-LABEL: ctpop32_ne_one_nonzero:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub w8, w0, #1
+; CHECK-NEXT:    tst w0, w8
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+;
+; CHECK-NONEON-LABEL: ctpop32_ne_one_nonzero:
+; CHECK-NONEON:       // %bb.0: // %entry
+; CHECK-NONEON-NEXT:    sub w8, w0, #1
+; CHECK-NONEON-NEXT:    tst w0, w8
+; CHECK-NONEON-NEXT:    cset w0, ne
+; CHECK-NONEON-NEXT:    ret
+;
+; CHECK-CSSC-LABEL: ctpop32_ne_one_nonzero:
+; CHECK-CSSC:       // %bb.0: // %entry
+; CHECK-CSSC-NEXT:    sub w8, w0, #1
+; CHECK-CSSC-NEXT:    tst w0, w8
+; CHECK-CSSC-NEXT:    cset w0, ne
+; CHECK-CSSC-NEXT:    ret
+entry:
+  %popcnt = tail call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp ne i32 %popcnt, 1
+  ret i1 %cmp
+}
+
 declare i32 @llvm.ctpop.i32(i32) nounwind readnone
 declare i64 @llvm.ctpop.i64(i64) nounwind readnone
diff --git a/llvm/test/CodeGen/AArch64/consecutive-stores-of-faddv.ll b/llvm/test/CodeGen/AArch64/consecutive-stores-of-faddv.ll
new file mode 100644
index 0000000000000000000000000000000000000000..64482e15aed81e3ea2934799cad62b4d69daa95c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/consecutive-stores-of-faddv.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -mtriple=aarch64-linux-gnu -mattr=+sve,+sme -O3 < %s -o - | FileCheck %s --check-prefixes=CHECK
+
+; Tests consecutive stores of @llvm.aarch64.sve.faddv. Within SDAG faddv is
+; lowered as a FADDV + EXTRACT_VECTOR_ELT (of lane 0). Stores of extracts can
+; be matched by DAGCombiner::mergeConsecutiveStores(), which we want to avoid in
+; some cases as it can lead to worse codegen.
+
+; TODO: A single `stp s0, s1, [x0]` may be preferred here.
+define void @consecutive_stores_pair(ptr %dest0, <vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1) {
+; CHECK-LABEL: consecutive_stores_pair:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    faddv s0, p0, z0.s
+; CHECK-NEXT:    faddv s1, p0, z1.s
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %dest1 = getelementptr inbounds i8, ptr %dest0, i64 4
+  %reduce0 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec0)
+  %reduce1 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec1)
+  store float %reduce0, ptr %dest0, align 4
+  store float %reduce1, ptr %dest1, align 4
+  ret void
+}
+
+define void @consecutive_stores_quadruple(ptr %dest0, <vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1, <vscale x 4 x float> %vec2, <vscale x 4 x float> %vec3) {
+; CHECK-LABEL: consecutive_stores_quadruple:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    faddv s0, p0, z0.s
+; CHECK-NEXT:    faddv s1, p0, z1.s
+; CHECK-NEXT:    faddv s2, p0, z2.s
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    faddv s3, p0, z3.s
+; CHECK-NEXT:    mov v2.s[1], v3.s[0]
+; CHECK-NEXT:    stp d0, d2, [x0]
+; CHECK-NEXT:    ret
+  %dest1 = getelementptr inbounds i8, ptr %dest0, i64 4
+  %dest2 = getelementptr inbounds i8, ptr %dest1, i64 4
+  %dest3 = getelementptr inbounds i8, ptr %dest2, i64 4
+  %reduce0 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec0)
+  %reduce1 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec1)
+  %reduce2 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec2)
+  %reduce3 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec3)
+  store float %reduce0, ptr %dest0, align 4
+  store float %reduce1, ptr %dest1, align 4
+  store float %reduce2, ptr %dest2, align 4
+  store float %reduce3, ptr %dest3, align 4
+  ret void
+}
+
+define void @consecutive_stores_pair_streaming_function(ptr %dest0, <vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1) "aarch64_pstate_sm_enabled"  {
+; CHECK-LABEL: consecutive_stores_pair_streaming_function:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    faddv s0, p0, z0.s
+; CHECK-NEXT:    faddv s1, p0, z1.s
+; CHECK-NEXT:    stp s0, s1, [x0]
+; CHECK-NEXT:    ret
+  %dest1 = getelementptr inbounds i8, ptr %dest0, i64 4
+  %reduce0 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec0)
+  %reduce1 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec1)
+  store float %reduce0, ptr %dest0, align 4
+  store float %reduce1, ptr %dest1, align 4
+  ret void
+}
+
+define void @consecutive_stores_quadruple_streaming_function(ptr %dest0, <vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1, <vscale x 4 x float> %vec2, <vscale x 4 x float> %vec3) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: consecutive_stores_quadruple_streaming_function:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    faddv s0, p0, z0.s
+; CHECK-NEXT:    faddv s1, p0, z1.s
+; CHECK-NEXT:    faddv s2, p0, z2.s
+; CHECK-NEXT:    stp s0, s1, [x0]
+; CHECK-NEXT:    faddv s3, p0, z3.s
+; CHECK-NEXT:    stp s2, s3, [x0, #8]
+; CHECK-NEXT:    ret
+  %dest1 = getelementptr inbounds i8, ptr %dest0, i64 4
+  %dest2 = getelementptr inbounds i8, ptr %dest1, i64 4
+  %dest3 = getelementptr inbounds i8, ptr %dest2, i64 4
+  %reduce0 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec0)
+  %reduce1 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec1)
+  %reduce2 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec2)
+  %reduce3 = call float @llvm.aarch64.sve.faddv.nxv4f32(<vscale x 4 x i1> splat(i1 true), <vscale x 4 x float> %vec3)
+  store float %reduce0, ptr %dest0, align 4
+  store float %reduce1, ptr %dest1, align 4
+  store float %reduce2, ptr %dest2, align 4
+  store float %reduce3, ptr %dest3, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/memmove-inline.ll b/llvm/test/CodeGen/AArch64/memmove-inline.ll
new file mode 100644
index 0000000000000000000000000000000000000000..641c48dd0f1c54ae6e15c4ee5e80b32bc0d47ad9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/memmove-inline.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ALIGNED
+; RUN: llc -mtriple=aarch64 -mattr=+strict-align < %s | FileCheck %s --check-prefixes=CHECK,CHECK-UNALIGNED
+
+; Small (16 bytes here) unaligned memmove() should be a function call if
+; strict-alignment is turned on.
+define void @t16(ptr %out, ptr %in) {
+; CHECK-ALIGNED-LABEL: t16:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldr q0, [x1]
+; CHECK-ALIGNED-NEXT:    str q0, [x0]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t16:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-UNALIGNED-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-UNALIGNED-NEXT:    .cfi_offset w30, -16
+; CHECK-UNALIGNED-NEXT:    mov w2, #16 // =0x10
+; CHECK-UNALIGNED-NEXT:    bl memmove
+; CHECK-UNALIGNED-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 16, i1 false)
+  ret void
+}
+
+; Small (16 bytes here) aligned memmove() should be inlined even if
+; strict-alignment is turned on.
+define void @t16_aligned(ptr align 8 %out, ptr align 8 %in) {
+; CHECK-ALIGNED-LABEL: t16_aligned:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldr q0, [x1]
+; CHECK-ALIGNED-NEXT:    str q0, [x0]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t16_aligned:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    ldp x9, x8, [x1]
+; CHECK-UNALIGNED-NEXT:    stp x9, x8, [x0]
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr align 8 %out, ptr align 8 %in, i64 16, i1 false)
+  ret void
+}
+
+; Tiny (4 bytes here) unaligned memmove() should be inlined with byte sized
+; loads and stores if strict-alignment is turned on.
+define void @t4(ptr %out, ptr %in) {
+; CHECK-ALIGNED-LABEL: t4:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldr w8, [x1]
+; CHECK-ALIGNED-NEXT:    str w8, [x0]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t4:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    ldrb w8, [x1, #3]
+; CHECK-UNALIGNED-NEXT:    ldrb w9, [x1, #2]
+; CHECK-UNALIGNED-NEXT:    ldrb w10, [x1]
+; CHECK-UNALIGNED-NEXT:    ldrb w11, [x1, #1]
+; CHECK-UNALIGNED-NEXT:    strb w8, [x0, #3]
+; CHECK-UNALIGNED-NEXT:    strb w9, [x0, #2]
+; CHECK-UNALIGNED-NEXT:    strb w11, [x0, #1]
+; CHECK-UNALIGNED-NEXT:    strb w10, [x0]
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 4, i1 false)
+  ret void
+}
+
+define void @t256(ptr %out, ptr %in) {
+; CHECK-ALIGNED-LABEL: t256:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldp q0, q1, [x1]
+; CHECK-ALIGNED-NEXT:    ldp q2, q3, [x1, #32]
+; CHECK-ALIGNED-NEXT:    ldp q4, q5, [x1, #64]
+; CHECK-ALIGNED-NEXT:    ldp q6, q7, [x1, #96]
+; CHECK-ALIGNED-NEXT:    ldp q16, q17, [x1, #224]
+; CHECK-ALIGNED-NEXT:    ldp q18, q19, [x1, #128]
+; CHECK-ALIGNED-NEXT:    ldp q20, q21, [x1, #160]
+; CHECK-ALIGNED-NEXT:    ldp q22, q23, [x1, #192]
+; CHECK-ALIGNED-NEXT:    stp q0, q1, [x0]
+; CHECK-ALIGNED-NEXT:    stp q2, q3, [x0, #32]
+; CHECK-ALIGNED-NEXT:    stp q4, q5, [x0, #64]
+; CHECK-ALIGNED-NEXT:    stp q6, q7, [x0, #96]
+; CHECK-ALIGNED-NEXT:    stp q18, q19, [x0, #128]
+; CHECK-ALIGNED-NEXT:    stp q20, q21, [x0, #160]
+; CHECK-ALIGNED-NEXT:    stp q22, q23, [x0, #192]
+; CHECK-ALIGNED-NEXT:    stp q16, q17, [x0, #224]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t256:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-UNALIGNED-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-UNALIGNED-NEXT:    .cfi_offset w30, -16
+; CHECK-UNALIGNED-NEXT:    mov w2, #256 // =0x100
+; CHECK-UNALIGNED-NEXT:    bl memmove
+; CHECK-UNALIGNED-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 256, i1 false)
+  ret void
+}
+
+define void @t257(ptr %out, ptr %in) {
+; CHECK-LABEL: t257:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov w2, #257 // =0x101
+; CHECK-NEXT:    bl memmove
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 257, i1 false)
+  ret void
+}
+
+declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1)
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index a446aae21ee8fb0f0859561314cb21f87fabe332..c345c1e50bbbb7f34fa4d6bc5ab46a28b583536e 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -1,5 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+dotprod    < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod    < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:       warning: Instruction selection used fallback path for test_udot_v5i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v5i8_nomla
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v5i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v5i8_double
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v25i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v25i8_nomla
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v25i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v25i8_double
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v33i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_udot_v33i8_nomla
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v33i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v33i8_double
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla
 
 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
 declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
@@ -13,17 +30,63 @@ declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>)
 declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)
 
 define i32 @test_udot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v4i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ldr s1, [x1]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    umull v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ldr s1, [x1]
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    umull v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    uxtb w8, w8
+; CHECK-GI-NEXT:    uxtb w9, w9
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b3, v0.b[2]
+; CHECK-GI-NEXT:    mov b5, v2.b[2]
+; CHECK-GI-NEXT:    mov b4, v0.b[3]
+; CHECK-GI-NEXT:    mov b0, v2.b[1]
+; CHECK-GI-NEXT:    mov b6, v2.b[3]
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fmov w10, s1
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fmov w13, s5
+; CHECK-GI-NEXT:    fmov w8, s4
+; CHECK-GI-NEXT:    fmov w12, s0
+; CHECK-GI-NEXT:    uxtb w10, w10
+; CHECK-GI-NEXT:    uxtb w11, w11
+; CHECK-GI-NEXT:    uxtb w13, w13
+; CHECK-GI-NEXT:    uxtb w8, w8
+; CHECK-GI-NEXT:    uxtb w12, w12
+; CHECK-GI-NEXT:    mov v1.h[1], w10
+; CHECK-GI-NEXT:    fmov w10, s6
+; CHECK-GI-NEXT:    fmov s0, w11
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    mov v2.h[1], w12
+; CHECK-GI-NEXT:    uxtb w10, w10
+; CHECK-GI-NEXT:    mov v0.h[1], w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov v3.h[1], w10
+; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <4 x i8>, ptr %a
   %1 = zext <4 x i8> %0 to <4 x i32>
@@ -36,14 +99,31 @@ entry:
 }
 
 define i32 @test_udot_v4i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v4i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v4i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v4i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov v2.b[0], v0.b[0]
+; CHECK-GI-NEXT:    mov b3, v0.b[2]
+; CHECK-GI-NEXT:    mov b0, v0.b[3]
+; CHECK-GI-NEXT:    mov v2.b[1], v1.b[0]
+; CHECK-GI-NEXT:    mov v2.b[2], v3.b[0]
+; CHECK-GI-NEXT:    mov v2.b[3], v0.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v2.8b, #0
+; CHECK-GI-NEXT:    uaddlv s0, v0.4h
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    and w0, w8, #0xffff
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <4 x i8>, ptr %a1
   %1 = zext <4 x i8> %0 to <4 x i32>
@@ -51,17 +131,63 @@ entry:
   ret i32 %2
 }
 define i32 @test_sdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v4i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ldr s1, [x1]
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    smull v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s0, [x0]
+; CHECK-SD-NEXT:    ldr s1, [x1]
+; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    smull v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    ldr w9, [x1]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w9, w9
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b3, v0.b[2]
+; CHECK-GI-NEXT:    mov b5, v2.b[2]
+; CHECK-GI-NEXT:    mov b4, v0.b[3]
+; CHECK-GI-NEXT:    mov b0, v2.b[1]
+; CHECK-GI-NEXT:    mov b6, v2.b[3]
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fmov w10, s1
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fmov w13, s5
+; CHECK-GI-NEXT:    fmov w8, s4
+; CHECK-GI-NEXT:    fmov w12, s0
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    sxtb w11, w11
+; CHECK-GI-NEXT:    sxtb w13, w13
+; CHECK-GI-NEXT:    sxtb w8, w8
+; CHECK-GI-NEXT:    sxtb w12, w12
+; CHECK-GI-NEXT:    mov v1.h[1], w10
+; CHECK-GI-NEXT:    fmov w10, s6
+; CHECK-GI-NEXT:    fmov s0, w11
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    mov v2.h[1], w12
+; CHECK-GI-NEXT:    sxtb w10, w10
+; CHECK-GI-NEXT:    mov v0.h[1], w8
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov v3.h[1], w10
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <4 x i8>, ptr %a
   %1 = sext <4 x i8> %0 to <4 x i32>
@@ -74,25 +200,48 @@ entry:
 }
 
 define i32 @test_sdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
-; CHECK-LABEL: test_sdot_v4i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v3.4s, v3.4h, #0
-; CHECK-NEXT:    ushll v2.4s, v2.4h, #0
-; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    shl v2.4s, v2.4s, #24
-; CHECK-NEXT:    shl v3.4s, v3.4s, #24
-; CHECK-NEXT:    shl v1.4s, v1.4s, #24
-; CHECK-NEXT:    shl v0.4s, v0.4s, #24
-; CHECK-NEXT:    sshr v2.4s, v2.4s, #24
-; CHECK-NEXT:    sshr v3.4s, v3.4s, #24
-; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
-; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
-; CHECK-NEXT:    mul v2.4s, v2.4s, v3.4s
-; CHECK-NEXT:    mla v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v2.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v4i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    shl v2.4s, v2.4s, #24
+; CHECK-SD-NEXT:    shl v3.4s, v3.4s, #24
+; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-SD-NEXT:    sshr v2.4s, v2.4s, #24
+; CHECK-SD-NEXT:    sshr v3.4s, v3.4s, #24
+; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-SD-NEXT:    mul v2.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT:    mla v2.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v2.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v4i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-GI-NEXT:    shl v2.4s, v2.4s, #24
+; CHECK-GI-NEXT:    shl v3.4s, v3.4s, #24
+; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-GI-NEXT:    sshr v2.4s, v2.4s, #24
+; CHECK-GI-NEXT:    sshr v3.4s, v3.4s, #24
+; CHECK-GI-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mul v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <4 x i8> %a to <4 x i32>
   %bz = sext <4 x i8> %b to <4 x i32>
@@ -107,17 +256,31 @@ entry:
 }
 
 define i32 @test_sdot_v4i8_double_nomla(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
-; CHECK-LABEL: test_sdot_v4i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    ushll v1.4s, v2.4h, #0
-; CHECK-NEXT:    shl v0.4s, v0.4s, #24
-; CHECK-NEXT:    shl v1.4s, v1.4s, #24
-; CHECK-NEXT:    sshr v0.4s, v0.4s, #24
-; CHECK-NEXT:    ssra v0.4s, v1.4s, #24
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v4i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ushll v1.4s, v2.4h, #0
+; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-SD-NEXT:    ssra v0.4s, v1.4s, #24
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v4i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    shl v1.4h, v2.4h, #8
+; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-GI-NEXT:    saddlv s1, v1.4h
+; CHECK-GI-NEXT:    saddlv s0, v0.4h
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    sxth w8, w8
+; CHECK-GI-NEXT:    add w0, w8, w9, sxth
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <4 x i8> %a to <4 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %az)
@@ -272,15 +435,25 @@ entry:
 }
 
 define i32 @test_udot_v8i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v8i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.8b, #1
-; CHECK-NEXT:    ldr d2, [x0]
-; CHECK-NEXT:    udot v0.2s, v2.8b, v1.8b
-; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v8i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.8b, #1
+; CHECK-SD-NEXT:    ldr d2, [x0]
+; CHECK-SD-NEXT:    udot v0.2s, v2.8b, v1.8b
+; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v8i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.8b, #1
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr d2, [x0]
+; CHECK-GI-NEXT:    udot v1.2s, v2.8b, v0.8b
+; CHECK-GI-NEXT:    addp v0.2s, v1.2s, v1.2s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %a1
   %1 = zext <8 x i8> %0 to <8 x i32>
@@ -309,15 +482,25 @@ entry:
 }
 
 define i32 @test_sdot_v8i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_sdot_v8i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.8b, #1
-; CHECK-NEXT:    ldr d2, [x0]
-; CHECK-NEXT:    sdot v0.2s, v2.8b, v1.8b
-; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v8i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.8b, #1
+; CHECK-SD-NEXT:    ldr d2, [x0]
+; CHECK-SD-NEXT:    sdot v0.2s, v2.8b, v1.8b
+; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v8i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.8b, #1
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr d2, [x0]
+; CHECK-GI-NEXT:    sdot v1.2s, v2.8b, v0.8b
+; CHECK-GI-NEXT:    addp v0.2s, v1.2s, v1.2s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <8 x i8>, ptr %a1
   %1 = sext <8 x i8> %0 to <8 x i32>
@@ -406,14 +589,27 @@ entry:
 
 
 define i32 @test_udot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
-; CHECK-LABEL: test_udot_v8i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v4.2d, #0000000000000000
-; CHECK-NEXT:    udot v4.2s, v2.8b, v3.8b
-; CHECK-NEXT:    udot v4.2s, v0.8b, v1.8b
-; CHECK-NEXT:    addp v0.2s, v4.2s, v4.2s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v8i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT:    udot v4.2s, v2.8b, v3.8b
+; CHECK-SD-NEXT:    udot v4.2s, v0.8b, v1.8b
+; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v8i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-GI-NEXT:    udot v5.2s, v0.8b, v1.8b
+; CHECK-GI-NEXT:    udot v4.2s, v2.8b, v3.8b
+; CHECK-GI-NEXT:    addp v0.2s, v5.2s, v5.2s
+; CHECK-GI-NEXT:    addp v1.2s, v4.2s, v4.2s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = zext <8 x i8> %a to <8 x i32>
   %bz = zext <8 x i8> %b to <8 x i32>
@@ -428,15 +624,29 @@ entry:
 }
 
 define i32 @test_udot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
-; CHECK-LABEL: test_udot_v8i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    movi v3.8b, #1
-; CHECK-NEXT:    udot v1.2s, v2.8b, v3.8b
-; CHECK-NEXT:    udot v1.2s, v0.8b, v3.8b
-; CHECK-NEXT:    addp v0.2s, v1.2s, v1.2s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v8i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v3.8b, #1
+; CHECK-SD-NEXT:    udot v1.2s, v2.8b, v3.8b
+; CHECK-SD-NEXT:    udot v1.2s, v0.8b, v3.8b
+; CHECK-SD-NEXT:    addp v0.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v8i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.8b, #1
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    udot v4.2s, v0.8b, v1.8b
+; CHECK-GI-NEXT:    udot v3.2s, v2.8b, v1.8b
+; CHECK-GI-NEXT:    addp v0.2s, v4.2s, v4.2s
+; CHECK-GI-NEXT:    addp v1.2s, v3.2s, v3.2s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = zext <8 x i8> %a to <8 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
@@ -447,14 +657,27 @@ entry:
 }
 
 define i32 @test_udot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
-; CHECK-LABEL: test_udot_v16i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v4.2d, #0000000000000000
-; CHECK-NEXT:    udot v4.4s, v2.16b, v3.16b
-; CHECK-NEXT:    udot v4.4s, v0.16b, v1.16b
-; CHECK-NEXT:    addv s0, v4.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v16i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT:    udot v4.4s, v2.16b, v3.16b
+; CHECK-SD-NEXT:    udot v4.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v4.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v16i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-GI-NEXT:    udot v5.4s, v0.16b, v1.16b
+; CHECK-GI-NEXT:    udot v4.4s, v2.16b, v3.16b
+; CHECK-GI-NEXT:    addv s0, v5.4s
+; CHECK-GI-NEXT:    addv s1, v4.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = zext <16 x i8> %a to <16 x i32>
   %bz = zext <16 x i8> %b to <16 x i32>
@@ -469,15 +692,29 @@ entry:
 }
 
 define i32 @test_udot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
-; CHECK-LABEL: test_udot_v16i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.16b, #1
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    udot v3.4s, v2.16b, v1.16b
-; CHECK-NEXT:    udot v3.4s, v0.16b, v1.16b
-; CHECK-NEXT:    addv s0, v3.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v16i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.16b, #1
+; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT:    udot v3.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    udot v3.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v3.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v16i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.16b, #1
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    udot v4.4s, v0.16b, v1.16b
+; CHECK-GI-NEXT:    udot v3.4s, v2.16b, v1.16b
+; CHECK-GI-NEXT:    addv s0, v4.4s
+; CHECK-GI-NEXT:    addv s1, v3.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = zext <16 x i8> %a to <16 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
@@ -488,14 +725,27 @@ entry:
 }
 
 define i32 @test_sdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
-; CHECK-LABEL: test_sdot_v8i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v4.2d, #0000000000000000
-; CHECK-NEXT:    sdot v4.2s, v2.8b, v3.8b
-; CHECK-NEXT:    sdot v4.2s, v0.8b, v1.8b
-; CHECK-NEXT:    addp v0.2s, v4.2s, v4.2s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v8i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT:    sdot v4.2s, v2.8b, v3.8b
+; CHECK-SD-NEXT:    sdot v4.2s, v0.8b, v1.8b
+; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v8i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-GI-NEXT:    sdot v5.2s, v0.8b, v1.8b
+; CHECK-GI-NEXT:    sdot v4.2s, v2.8b, v3.8b
+; CHECK-GI-NEXT:    addp v0.2s, v5.2s, v5.2s
+; CHECK-GI-NEXT:    addp v1.2s, v4.2s, v4.2s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <8 x i8> %a to <8 x i32>
   %bz = sext <8 x i8> %b to <8 x i32>
@@ -510,15 +760,29 @@ entry:
 }
 
 define i32 @test_sdot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
-; CHECK-LABEL: test_sdot_v8i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    movi v3.8b, #1
-; CHECK-NEXT:    sdot v1.2s, v2.8b, v3.8b
-; CHECK-NEXT:    sdot v1.2s, v0.8b, v3.8b
-; CHECK-NEXT:    addp v0.2s, v1.2s, v1.2s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v8i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v3.8b, #1
+; CHECK-SD-NEXT:    sdot v1.2s, v2.8b, v3.8b
+; CHECK-SD-NEXT:    sdot v1.2s, v0.8b, v3.8b
+; CHECK-SD-NEXT:    addp v0.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v8i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.8b, #1
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    sdot v4.2s, v0.8b, v1.8b
+; CHECK-GI-NEXT:    sdot v3.2s, v2.8b, v1.8b
+; CHECK-GI-NEXT:    addp v0.2s, v4.2s, v4.2s
+; CHECK-GI-NEXT:    addp v1.2s, v3.2s, v3.2s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <8 x i8> %a to <8 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
@@ -529,14 +793,27 @@ entry:
 }
 
 define i32 @test_sdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
-; CHECK-LABEL: test_sdot_v16i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v4.2d, #0000000000000000
-; CHECK-NEXT:    sdot v4.4s, v2.16b, v3.16b
-; CHECK-NEXT:    sdot v4.4s, v0.16b, v1.16b
-; CHECK-NEXT:    addv s0, v4.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v16i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT:    sdot v4.4s, v2.16b, v3.16b
+; CHECK-SD-NEXT:    sdot v4.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v4.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v16i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-GI-NEXT:    sdot v5.4s, v0.16b, v1.16b
+; CHECK-GI-NEXT:    sdot v4.4s, v2.16b, v3.16b
+; CHECK-GI-NEXT:    addv s0, v5.4s
+; CHECK-GI-NEXT:    addv s1, v4.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <16 x i8> %a to <16 x i32>
   %bz = sext <16 x i8> %b to <16 x i32>
@@ -551,15 +828,29 @@ entry:
 }
 
 define i32 @test_sdot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
-; CHECK-LABEL: test_sdot_v16i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.16b, #1
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    sdot v3.4s, v2.16b, v1.16b
-; CHECK-NEXT:    sdot v3.4s, v0.16b, v1.16b
-; CHECK-NEXT:    addv s0, v3.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v16i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.16b, #1
+; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT:    sdot v3.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    sdot v3.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v3.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v16i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.16b, #1
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    sdot v4.4s, v0.16b, v1.16b
+; CHECK-GI-NEXT:    sdot v3.4s, v2.16b, v1.16b
+; CHECK-GI-NEXT:    addv s0, v4.4s
+; CHECK-GI-NEXT:    addv s1, v3.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <16 x i8> %a to <16 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
@@ -570,23 +861,39 @@ entry:
 }
 
 define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v24i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ldr q2, [x0]
-; CHECK-NEXT:    ldr q3, [x1]
-; CHECK-NEXT:    ldr d4, [x0, #16]
-; CHECK-NEXT:    ldr d5, [x1, #16]
-; CHECK-NEXT:    udot v1.2s, v5.8b, v4.8b
-; CHECK-NEXT:    udot v0.4s, v3.16b, v2.16b
-; CHECK-NEXT:    addp v1.2s, v1.2s, v1.2s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v24i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldr q2, [x0]
+; CHECK-SD-NEXT:    ldr q3, [x1]
+; CHECK-SD-NEXT:    ldr d4, [x0, #16]
+; CHECK-SD-NEXT:    ldr d5, [x1, #16]
+; CHECK-SD-NEXT:    udot v1.2s, v5.8b, v4.8b
+; CHECK-SD-NEXT:    udot v0.4s, v3.16b, v2.16b
+; CHECK-SD-NEXT:    addp v1.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s1
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    add w8, w9, w8
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v24i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr q2, [x0]
+; CHECK-GI-NEXT:    ldr d3, [x0, #16]
+; CHECK-GI-NEXT:    ldr q4, [x1]
+; CHECK-GI-NEXT:    ldr d5, [x1, #16]
+; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b
+; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <24 x i8>, ptr %a
   %1 = zext <24 x i8> %0 to <24 x i32>
@@ -599,22 +906,38 @@ entry:
 }
 
 define i32 @test_udot_v24i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v24i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.8b, #1
-; CHECK-NEXT:    ldr q4, [x0]
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-NEXT:    movi v3.16b, #1
-; CHECK-NEXT:    ldr d5, [x0, #16]
-; CHECK-NEXT:    udot v2.2s, v5.8b, v1.8b
-; CHECK-NEXT:    udot v0.4s, v4.16b, v3.16b
-; CHECK-NEXT:    addp v1.2s, v2.2s, v2.2s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    add w0, w9, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v24i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.8b, #1
+; CHECK-SD-NEXT:    ldr q4, [x0]
+; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v3.16b, #1
+; CHECK-SD-NEXT:    ldr d5, [x0, #16]
+; CHECK-SD-NEXT:    udot v2.2s, v5.8b, v1.8b
+; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT:    addp v1.2s, v2.2s, v2.2s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s1
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    add w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v24i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.8b, #1
+; CHECK-GI-NEXT:    movi v1.8b, #1
+; CHECK-GI-NEXT:    ldr q4, [x0]
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr d5, [x0, #16]
+; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v0.16b
+; CHECK-GI-NEXT:    udot v3.4s, v4.16b, v1.16b
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <24 x i8>, ptr %a1
   %1 = zext <24 x i8> %0 to <24 x i32>
@@ -622,23 +945,39 @@ entry:
   ret i32 %2
 }
 define i32 @test_sdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v24i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ldr q2, [x0]
-; CHECK-NEXT:    ldr q3, [x1]
-; CHECK-NEXT:    ldr d4, [x0, #16]
-; CHECK-NEXT:    ldr d5, [x1, #16]
-; CHECK-NEXT:    sdot v1.2s, v5.8b, v4.8b
-; CHECK-NEXT:    sdot v0.4s, v3.16b, v2.16b
-; CHECK-NEXT:    addp v1.2s, v1.2s, v1.2s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v24i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldr q2, [x0]
+; CHECK-SD-NEXT:    ldr q3, [x1]
+; CHECK-SD-NEXT:    ldr d4, [x0, #16]
+; CHECK-SD-NEXT:    ldr d5, [x1, #16]
+; CHECK-SD-NEXT:    sdot v1.2s, v5.8b, v4.8b
+; CHECK-SD-NEXT:    sdot v0.4s, v3.16b, v2.16b
+; CHECK-SD-NEXT:    addp v1.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s1
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    add w8, w9, w8
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v24i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr q2, [x0]
+; CHECK-GI-NEXT:    ldr d3, [x0, #16]
+; CHECK-GI-NEXT:    ldr q4, [x1]
+; CHECK-GI-NEXT:    ldr d5, [x1, #16]
+; CHECK-GI-NEXT:    sdot v1.4s, v4.16b, v2.16b
+; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <24 x i8>, ptr %a
   %1 = sext <24 x i8> %0 to <24 x i32>
@@ -651,209 +990,420 @@ entry:
 }
 
 define i32 @test_sdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
-; CHECK-LABEL: test_sdot_v24i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    ldr b1, [sp, #144]
-; CHECK-NEXT:    add x10, sp, #152
-; CHECK-NEXT:    add x9, sp, #160
-; CHECK-NEXT:    add x8, sp, #168
-; CHECK-NEXT:    ldr b2, [sp, #272]
-; CHECK-NEXT:    ld1 { v1.b }[1], [x10]
-; CHECK-NEXT:    add x11, sp, #280
-; CHECK-NEXT:    ldr b3, [sp, #80]
-; CHECK-NEXT:    mov v0.b[1], w1
-; CHECK-NEXT:    ldr b4, [sp, #528]
-; CHECK-NEXT:    add x10, sp, #88
-; CHECK-NEXT:    ld1 { v2.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #536
-; CHECK-NEXT:    ldr b5, [sp, #336]
-; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-NEXT:    ld1 { v3.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #344
-; CHECK-NEXT:    ld1 { v4.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #176
-; CHECK-NEXT:    ldr b6, [sp, #656]
-; CHECK-NEXT:    mov v0.b[2], w2
-; CHECK-NEXT:    ld1 { v5.b }[1], [x10]
-; CHECK-NEXT:    ldr b7, [sp, #464]
-; CHECK-NEXT:    ld1 { v1.b }[3], [x8]
-; CHECK-NEXT:    add x12, sp, #664
-; CHECK-NEXT:    add x9, sp, #472
-; CHECK-NEXT:    ld1 { v6.b }[1], [x12]
-; CHECK-NEXT:    add x8, sp, #96
-; CHECK-NEXT:    add x10, sp, #184
-; CHECK-NEXT:    add x12, sp, #288
-; CHECK-NEXT:    ld1 { v7.b }[1], [x9]
-; CHECK-NEXT:    ld1 { v3.b }[2], [x8]
-; CHECK-NEXT:    mov v0.b[3], w3
-; CHECK-NEXT:    ld1 { v1.b }[4], [x11]
-; CHECK-NEXT:    add x8, sp, #352
-; CHECK-NEXT:    ld1 { v2.b }[2], [x12]
-; CHECK-NEXT:    add x13, sp, #544
-; CHECK-NEXT:    ld1 { v5.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #672
-; CHECK-NEXT:    ld1 { v4.b }[2], [x13]
-; CHECK-NEXT:    add x9, sp, #192
-; CHECK-NEXT:    ld1 { v1.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #480
-; CHECK-NEXT:    mov v0.b[4], w4
-; CHECK-NEXT:    ld1 { v7.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #296
-; CHECK-NEXT:    ld1 { v2.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #552
-; CHECK-NEXT:    add x12, sp, #200
-; CHECK-NEXT:    ld1 { v1.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v4.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #360
-; CHECK-NEXT:    ld1 { v5.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #104
-; CHECK-NEXT:    add x9, sp, #560
-; CHECK-NEXT:    mov v0.b[5], w5
-; CHECK-NEXT:    ld1 { v3.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #368
-; CHECK-NEXT:    ld1 { v1.b }[7], [x12]
-; CHECK-NEXT:    ld1 { v4.b }[4], [x9]
-; CHECK-NEXT:    add x13, sp, #208
-; CHECK-NEXT:    ld1 { v5.b }[4], [x8]
-; CHECK-NEXT:    add x12, sp, #304
-; CHECK-NEXT:    add x8, sp, #568
-; CHECK-NEXT:    ld1 { v2.b }[4], [x12]
-; CHECK-NEXT:    add x12, sp, #16
-; CHECK-NEXT:    add x17, sp, #376
-; CHECK-NEXT:    mov v0.b[6], w6
-; CHECK-NEXT:    ld1 { v1.b }[8], [x13]
-; CHECK-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-NEXT:    add x14, sp, #216
-; CHECK-NEXT:    ld1 { v5.b }[5], [x17]
-; CHECK-NEXT:    add x13, sp, #576
-; CHECK-NEXT:    add x11, sp, #224
-; CHECK-NEXT:    add x10, sp, #232
-; CHECK-NEXT:    add x15, sp, #240
-; CHECK-NEXT:    ld1 { v1.b }[9], [x14]
-; CHECK-NEXT:    ld1 { v4.b }[6], [x13]
-; CHECK-NEXT:    add x13, sp, #384
-; CHECK-NEXT:    mov v0.b[7], w7
-; CHECK-NEXT:    ld1 { v5.b }[6], [x13]
-; CHECK-NEXT:    add x13, sp, #112
-; CHECK-NEXT:    ld1 { v3.b }[4], [x13]
-; CHECK-NEXT:    add x13, sp, #32
-; CHECK-NEXT:    add x14, sp, #584
-; CHECK-NEXT:    ld1 { v1.b }[10], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[7], [x14]
-; CHECK-NEXT:    add x11, sp, #312
-; CHECK-NEXT:    add x14, sp, #40
-; CHECK-NEXT:    ld1 { v2.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #592
-; CHECK-NEXT:    ld1 { v0.b }[8], [x12]
-; CHECK-NEXT:    add x12, sp, #24
-; CHECK-NEXT:    add x16, sp, #248
-; CHECK-NEXT:    ld1 { v1.b }[11], [x10]
-; CHECK-NEXT:    ld1 { v4.b }[8], [x11]
-; CHECK-NEXT:    add x11, sp, #400
-; CHECK-NEXT:    add x9, sp, #256
-; CHECK-NEXT:    add x8, sp, #264
-; CHECK-NEXT:    add x10, sp, #72
-; CHECK-NEXT:    ld1 { v0.b }[9], [x12]
-; CHECK-NEXT:    add x12, sp, #392
-; CHECK-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v5.b }[7], [x12]
-; CHECK-NEXT:    add x12, sp, #48
-; CHECK-NEXT:    ld1 { v1.b }[12], [x15]
-; CHECK-NEXT:    add x15, sp, #120
-; CHECK-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-NEXT:    movi v18.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.b }[10], [x13]
-; CHECK-NEXT:    ld1 { v3.b }[5], [x15]
-; CHECK-NEXT:    add x15, sp, #408
-; CHECK-NEXT:    ld1 { v5.b }[8], [x11]
-; CHECK-NEXT:    add x13, sp, #56
-; CHECK-NEXT:    ld1 { v1.b }[13], [x16]
-; CHECK-NEXT:    add x11, sp, #64
-; CHECK-NEXT:    add x16, sp, #616
-; CHECK-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.b }[11], [x14]
-; CHECK-NEXT:    add x14, sp, #600
-; CHECK-NEXT:    ld1 { v4.b }[9], [x14]
-; CHECK-NEXT:    ld1 { v5.b }[9], [x15]
-; CHECK-NEXT:    add x15, sp, #608
-; CHECK-NEXT:    ld1 { v1.b }[14], [x9]
-; CHECK-NEXT:    add x9, sp, #488
-; CHECK-NEXT:    add x14, sp, #320
-; CHECK-NEXT:    ld1 { v0.b }[12], [x12]
-; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-NEXT:    ld1 { v2.b }[6], [x14]
-; CHECK-NEXT:    ld1 { v4.b }[10], [x15]
-; CHECK-NEXT:    add x14, sp, #624
-; CHECK-NEXT:    add x9, sp, #688
-; CHECK-NEXT:    ld1 { v1.b }[15], [x8]
-; CHECK-NEXT:    add x8, sp, #432
-; CHECK-NEXT:    add x12, sp, #328
-; CHECK-NEXT:    ld1 { v0.b }[13], [x13]
-; CHECK-NEXT:    add x13, sp, #416
-; CHECK-NEXT:    ld1 { v2.b }[7], [x12]
-; CHECK-NEXT:    ld1 { v5.b }[10], [x13]
-; CHECK-NEXT:    ld1 { v4.b }[11], [x16]
-; CHECK-NEXT:    add x16, sp, #680
-; CHECK-NEXT:    ld1 { v6.b }[3], [x16]
-; CHECK-NEXT:    add x13, sp, #632
-; CHECK-NEXT:    add x12, sp, #504
-; CHECK-NEXT:    ld1 { v0.b }[14], [x11]
-; CHECK-NEXT:    add x11, sp, #424
-; CHECK-NEXT:    add x15, sp, #128
-; CHECK-NEXT:    ld1 { v5.b }[11], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[12], [x14]
-; CHECK-NEXT:    add x11, sp, #696
-; CHECK-NEXT:    ld1 { v6.b }[4], [x9]
-; CHECK-NEXT:    ld1 { v3.b }[6], [x15]
-; CHECK-NEXT:    add x9, sp, #640
-; CHECK-NEXT:    ld1 { v0.b }[15], [x10]
-; CHECK-NEXT:    add x10, sp, #496
-; CHECK-NEXT:    ld1 { v5.b }[12], [x8]
-; CHECK-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-NEXT:    ld1 { v4.b }[13], [x13]
-; CHECK-NEXT:    add x10, sp, #440
-; CHECK-NEXT:    ld1 { v6.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #512
-; CHECK-NEXT:    add x8, sp, #136
-; CHECK-NEXT:    sdot v17.4s, v0.16b, v1.16b
-; CHECK-NEXT:    ld1 { v5.b }[13], [x10]
-; CHECK-NEXT:    ld1 { v7.b }[5], [x12]
-; CHECK-NEXT:    ld1 { v4.b }[14], [x9]
-; CHECK-NEXT:    add x9, sp, #448
-; CHECK-NEXT:    add x10, sp, #704
-; CHECK-NEXT:    ld1 { v3.b }[7], [x8]
-; CHECK-NEXT:    ld1 { v6.b }[6], [x10]
-; CHECK-NEXT:    add x8, sp, #648
-; CHECK-NEXT:    add x10, sp, #520
-; CHECK-NEXT:    ld1 { v5.b }[14], [x9]
-; CHECK-NEXT:    ld1 { v7.b }[6], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[15], [x8]
-; CHECK-NEXT:    add x8, sp, #456
-; CHECK-NEXT:    add x9, sp, #712
-; CHECK-NEXT:    sdot v19.2s, v3.8b, v2.8b
-; CHECK-NEXT:    ld1 { v6.b }[7], [x9]
-; CHECK-NEXT:    addv s0, v17.4s
-; CHECK-NEXT:    ld1 { v5.b }[15], [x8]
-; CHECK-NEXT:    ld1 { v7.b }[7], [x10]
-; CHECK-NEXT:    addp v1.2s, v19.2s, v19.2s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    sdot v16.4s, v5.16b, v4.16b
-; CHECK-NEXT:    sdot v18.2s, v7.8b, v6.8b
-; CHECK-NEXT:    fmov w9, s1
-; CHECK-NEXT:    addv s2, v16.4s
-; CHECK-NEXT:    addp v3.2s, v18.2s, v18.2s
-; CHECK-NEXT:    add w8, w8, w9
-; CHECK-NEXT:    fmov w10, s2
-; CHECK-NEXT:    fmov w11, s3
-; CHECK-NEXT:    add w9, w10, w11
-; CHECK-NEXT:    add w0, w8, w9
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v24i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr b1, [sp, #144]
+; CHECK-SD-NEXT:    add x10, sp, #152
+; CHECK-SD-NEXT:    add x9, sp, #160
+; CHECK-SD-NEXT:    add x8, sp, #168
+; CHECK-SD-NEXT:    ldr b2, [sp, #272]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #280
+; CHECK-SD-NEXT:    ldr b3, [sp, #80]
+; CHECK-SD-NEXT:    mov v0.b[1], w1
+; CHECK-SD-NEXT:    ldr b4, [sp, #528]
+; CHECK-SD-NEXT:    add x10, sp, #88
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #536
+; CHECK-SD-NEXT:    ldr b5, [sp, #336]
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #344
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #176
+; CHECK-SD-NEXT:    ldr b6, [sp, #656]
+; CHECK-SD-NEXT:    mov v0.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]
+; CHECK-SD-NEXT:    ldr b7, [sp, #464]
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    add x12, sp, #664
+; CHECK-SD-NEXT:    add x9, sp, #472
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT:    add x8, sp, #96
+; CHECK-SD-NEXT:    add x10, sp, #184
+; CHECK-SD-NEXT:    add x12, sp, #288
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
+; CHECK-SD-NEXT:    mov v0.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #352
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x12]
+; CHECK-SD-NEXT:    add x13, sp, #544
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #672
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x13]
+; CHECK-SD-NEXT:    add x9, sp, #192
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #480
+; CHECK-SD-NEXT:    mov v0.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #296
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #552
+; CHECK-SD-NEXT:    add x12, sp, #200
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #360
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #104
+; CHECK-SD-NEXT:    add x9, sp, #560
+; CHECK-SD-NEXT:    mov v0.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #368
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x9]
+; CHECK-SD-NEXT:    add x13, sp, #208
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT:    add x12, sp, #304
+; CHECK-SD-NEXT:    add x8, sp, #568
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #16
+; CHECK-SD-NEXT:    add x17, sp, #376
+; CHECK-SD-NEXT:    mov v0.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
+; CHECK-SD-NEXT:    add x14, sp, #216
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x17]
+; CHECK-SD-NEXT:    add x13, sp, #576
+; CHECK-SD-NEXT:    add x11, sp, #224
+; CHECK-SD-NEXT:    add x10, sp, #232
+; CHECK-SD-NEXT:    add x15, sp, #240
+; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x14]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #384
+; CHECK-SD-NEXT:    mov v0.b[7], w7
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #112
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #32
+; CHECK-SD-NEXT:    add x14, sp, #584
+; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x11]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x14]
+; CHECK-SD-NEXT:    add x11, sp, #312
+; CHECK-SD-NEXT:    add x14, sp, #40
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #592
+; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #24
+; CHECK-SD-NEXT:    add x16, sp, #248
+; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[8], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #400
+; CHECK-SD-NEXT:    add x9, sp, #256
+; CHECK-SD-NEXT:    add x8, sp, #264
+; CHECK-SD-NEXT:    add x10, sp, #72
+; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #392
+; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #48
+; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #120
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x13]
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #408
+; CHECK-SD-NEXT:    ld1 { v5.b }[8], [x11]
+; CHECK-SD-NEXT:    add x13, sp, #56
+; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x16]
+; CHECK-SD-NEXT:    add x11, sp, #64
+; CHECK-SD-NEXT:    add x16, sp, #616
+; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #600
+; CHECK-SD-NEXT:    ld1 { v4.b }[9], [x14]
+; CHECK-SD-NEXT:    ld1 { v5.b }[9], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #608
+; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #488
+; CHECK-SD-NEXT:    add x14, sp, #320
+; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x14]
+; CHECK-SD-NEXT:    ld1 { v4.b }[10], [x15]
+; CHECK-SD-NEXT:    add x14, sp, #624
+; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #432
+; CHECK-SD-NEXT:    add x12, sp, #328
+; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #416
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x12]
+; CHECK-SD-NEXT:    ld1 { v5.b }[10], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[11], [x16]
+; CHECK-SD-NEXT:    add x16, sp, #680
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x16]
+; CHECK-SD-NEXT:    add x13, sp, #632
+; CHECK-SD-NEXT:    add x12, sp, #504
+; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #424
+; CHECK-SD-NEXT:    add x15, sp, #128
+; CHECK-SD-NEXT:    ld1 { v5.b }[11], [x11]
+; CHECK-SD-NEXT:    ld1 { v4.b }[12], [x14]
+; CHECK-SD-NEXT:    add x11, sp, #696
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x15]
+; CHECK-SD-NEXT:    add x9, sp, #640
+; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #496
+; CHECK-SD-NEXT:    ld1 { v5.b }[12], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[13], [x13]
+; CHECK-SD-NEXT:    add x10, sp, #440
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #512
+; CHECK-SD-NEXT:    add x8, sp, #136
+; CHECK-SD-NEXT:    sdot v17.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ld1 { v5.b }[13], [x10]
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[14], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #448
+; CHECK-SD-NEXT:    add x10, sp, #704
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #648
+; CHECK-SD-NEXT:    add x10, sp, #520
+; CHECK-SD-NEXT:    ld1 { v5.b }[14], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v4.b }[15], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #456
+; CHECK-SD-NEXT:    add x9, sp, #712
+; CHECK-SD-NEXT:    sdot v19.2s, v3.8b, v2.8b
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
+; CHECK-SD-NEXT:    addv s0, v17.4s
+; CHECK-SD-NEXT:    ld1 { v5.b }[15], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]
+; CHECK-SD-NEXT:    addp v1.2s, v19.2s, v19.2s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    sdot v16.4s, v5.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v18.2s, v7.8b, v6.8b
+; CHECK-SD-NEXT:    fmov w9, s1
+; CHECK-SD-NEXT:    addv s2, v16.4s
+; CHECK-SD-NEXT:    addp v3.2s, v18.2s, v18.2s
+; CHECK-SD-NEXT:    add w8, w8, w9
+; CHECK-SD-NEXT:    fmov w10, s2
+; CHECK-SD-NEXT:    fmov w11, s3
+; CHECK-SD-NEXT:    add w9, w10, w11
+; CHECK-SD-NEXT:    add w0, w8, w9
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v24i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    ldr w8, [sp, #80]
+; CHECK-GI-NEXT:    ldr w9, [sp, #88]
+; CHECK-GI-NEXT:    fmov s1, w0
+; CHECK-GI-NEXT:    ldr w11, [sp, #336]
+; CHECK-GI-NEXT:    ldr w10, [sp, #280]
+; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #96]
+; CHECK-GI-NEXT:    ldr w12, [sp, #152]
+; CHECK-GI-NEXT:    mov v1.b[1], w1
+; CHECK-GI-NEXT:    fmov s4, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #584]
+; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v0.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #272]
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #144]
+; CHECK-GI-NEXT:    mov v1.b[2], w2
+; CHECK-GI-NEXT:    mov v0.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #528]
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    mov v2.b[1], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #344]
+; CHECK-GI-NEXT:    ldr w10, [sp, #536]
+; CHECK-GI-NEXT:    fmov s5, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #288]
+; CHECK-GI-NEXT:    mov v1.b[3], w3
+; CHECK-GI-NEXT:    mov v3.b[1], w12
+; CHECK-GI-NEXT:    mov v4.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #160]
+; CHECK-GI-NEXT:    mov v5.b[1], w10
+; CHECK-GI-NEXT:    mov v2.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #104]
+; CHECK-GI-NEXT:    ldr w10, [sp, #352]
+; CHECK-GI-NEXT:    mov v1.b[4], w4
+; CHECK-GI-NEXT:    mov v3.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #544]
+; CHECK-GI-NEXT:    mov v0.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #296]
+; CHECK-GI-NEXT:    mov v4.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #360]
+; CHECK-GI-NEXT:    mov v5.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #168]
+; CHECK-GI-NEXT:    mov v2.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #112]
+; CHECK-GI-NEXT:    mov v1.b[5], w5
+; CHECK-GI-NEXT:    mov v3.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #552]
+; CHECK-GI-NEXT:    mov v0.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #304]
+; CHECK-GI-NEXT:    mov v4.b[3], w10
+; CHECK-GI-NEXT:    mov v5.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #176]
+; CHECK-GI-NEXT:    ldr w10, [sp, #368]
+; CHECK-GI-NEXT:    mov v2.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #120]
+; CHECK-GI-NEXT:    mov v1.b[6], w6
+; CHECK-GI-NEXT:    mov v3.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #560]
+; CHECK-GI-NEXT:    mov v0.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #312]
+; CHECK-GI-NEXT:    mov v4.b[4], w10
+; CHECK-GI-NEXT:    mov v5.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #184]
+; CHECK-GI-NEXT:    ldr w10, [sp, #376]
+; CHECK-GI-NEXT:    mov v2.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #128]
+; CHECK-GI-NEXT:    mov v1.b[7], w7
+; CHECK-GI-NEXT:    mov v3.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #568]
+; CHECK-GI-NEXT:    mov v0.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #320]
+; CHECK-GI-NEXT:    mov v4.b[5], w10
+; CHECK-GI-NEXT:    mov v5.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #192]
+; CHECK-GI-NEXT:    ldr w10, [sp, #384]
+; CHECK-GI-NEXT:    mov v2.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #136]
+; CHECK-GI-NEXT:    mov v3.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #576]
+; CHECK-GI-NEXT:    mov v0.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #328]
+; CHECK-GI-NEXT:    mov v4.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #200]
+; CHECK-GI-NEXT:    mov v5.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #392]
+; CHECK-GI-NEXT:    mov v2.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #464]
+; CHECK-GI-NEXT:    mov v3.b[7], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #16]
+; CHECK-GI-NEXT:    fmov s6, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #208]
+; CHECK-GI-NEXT:    mov v4.b[7], w9
+; CHECK-GI-NEXT:    mov v1.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #656]
+; CHECK-GI-NEXT:    ldr w9, [sp, #472]
+; CHECK-GI-NEXT:    mov v5.b[7], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #400]
+; CHECK-GI-NEXT:    fmov d0, d0
+; CHECK-GI-NEXT:    fmov s7, w10
+; CHECK-GI-NEXT:    mov v6.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #592]
+; CHECK-GI-NEXT:    mov v3.b[8], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #664]
+; CHECK-GI-NEXT:    ldr w8, [sp, #24]
+; CHECK-GI-NEXT:    mov v4.b[8], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #216]
+; CHECK-GI-NEXT:    fmov d2, d2
+; CHECK-GI-NEXT:    mov v5.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #480]
+; CHECK-GI-NEXT:    mov v7.b[1], w10
+; CHECK-GI-NEXT:    mov v1.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #408]
+; CHECK-GI-NEXT:    ldr w10, [sp, #600]
+; CHECK-GI-NEXT:    mov v3.b[9], w11
+; CHECK-GI-NEXT:    mov v6.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #672]
+; CHECK-GI-NEXT:    ldr w11, [sp, #32]
+; CHECK-GI-NEXT:    mov v4.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #224]
+; CHECK-GI-NEXT:    mov v5.b[9], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #488]
+; CHECK-GI-NEXT:    mov v7.b[2], w9
+; CHECK-GI-NEXT:    mov v1.b[10], w11
+; CHECK-GI-NEXT:    ldr w9, [sp, #416]
+; CHECK-GI-NEXT:    ldr w11, [sp, #608]
+; CHECK-GI-NEXT:    mov v3.b[10], w8
+; CHECK-GI-NEXT:    mov v6.b[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #680]
+; CHECK-GI-NEXT:    ldr w8, [sp, #40]
+; CHECK-GI-NEXT:    mov v4.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #232]
+; CHECK-GI-NEXT:    mov v5.b[10], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #496]
+; CHECK-GI-NEXT:    mov v7.b[3], w10
+; CHECK-GI-NEXT:    mov v1.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #424]
+; CHECK-GI-NEXT:    ldr w10, [sp, #616]
+; CHECK-GI-NEXT:    mov v3.b[11], w9
+; CHECK-GI-NEXT:    mov v6.b[4], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #688]
+; CHECK-GI-NEXT:    ldr w9, [sp, #48]
+; CHECK-GI-NEXT:    mov v4.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #240]
+; CHECK-GI-NEXT:    mov v5.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #504]
+; CHECK-GI-NEXT:    mov v7.b[4], w11
+; CHECK-GI-NEXT:    mov v1.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #432]
+; CHECK-GI-NEXT:    ldr w11, [sp, #624]
+; CHECK-GI-NEXT:    mov v3.b[12], w8
+; CHECK-GI-NEXT:    mov v6.b[5], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #696]
+; CHECK-GI-NEXT:    ldr w8, [sp, #56]
+; CHECK-GI-NEXT:    mov v4.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #248]
+; CHECK-GI-NEXT:    mov v5.b[12], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #512]
+; CHECK-GI-NEXT:    mov v7.b[5], w10
+; CHECK-GI-NEXT:    mov v1.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #440]
+; CHECK-GI-NEXT:    ldr w10, [sp, #632]
+; CHECK-GI-NEXT:    mov v3.b[13], w9
+; CHECK-GI-NEXT:    mov v6.b[6], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #704]
+; CHECK-GI-NEXT:    ldr w9, [sp, #64]
+; CHECK-GI-NEXT:    mov v4.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #256]
+; CHECK-GI-NEXT:    mov v5.b[13], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #520]
+; CHECK-GI-NEXT:    mov v7.b[6], w11
+; CHECK-GI-NEXT:    mov v1.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #448]
+; CHECK-GI-NEXT:    ldr w11, [sp, #640]
+; CHECK-GI-NEXT:    mov v3.b[14], w8
+; CHECK-GI-NEXT:    mov v6.b[7], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #712]
+; CHECK-GI-NEXT:    ldr w8, [sp, #72]
+; CHECK-GI-NEXT:    mov v4.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #264]
+; CHECK-GI-NEXT:    mov v5.b[14], w11
+; CHECK-GI-NEXT:    mov v7.b[7], w10
+; CHECK-GI-NEXT:    sdot v18.4s, v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #456]
+; CHECK-GI-NEXT:    mov v3.b[15], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #648]
+; CHECK-GI-NEXT:    fmov d6, d6
+; CHECK-GI-NEXT:    mov v4.b[15], w8
+; CHECK-GI-NEXT:    mov v5.b[15], w9
+; CHECK-GI-NEXT:    fmov d7, d7
+; CHECK-GI-NEXT:    sdot v17.4s, v1.16b, v3.16b
+; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v5.16b
+; CHECK-GI-NEXT:    sdot v16.4s, v6.16b, v7.16b
+; CHECK-GI-NEXT:    add v0.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT:    add v1.4s, v19.4s, v16.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <24 x i8> %a to <24 x i32>
   %bz = sext <24 x i8> %b to <24 x i32>
@@ -868,119 +1418,237 @@ entry:
 }
 
 define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
-; CHECK-LABEL: test_sdot_v24i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    ldr b1, [sp, #336]
-; CHECK-NEXT:    add x8, sp, #344
-; CHECK-NEXT:    add x9, sp, #400
-; CHECK-NEXT:    ldr b2, [sp, #80]
-; CHECK-NEXT:    ldr b3, [sp, #464]
-; CHECK-NEXT:    ld1 { v1.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #352
-; CHECK-NEXT:    add x10, sp, #408
-; CHECK-NEXT:    mov v0.b[1], w1
-; CHECK-NEXT:    add x11, sp, #472
-; CHECK-NEXT:    add x12, sp, #480
-; CHECK-NEXT:    ld1 { v3.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #416
-; CHECK-NEXT:    add x13, sp, #488
-; CHECK-NEXT:    ld1 { v1.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #360
-; CHECK-NEXT:    add x14, sp, #496
-; CHECK-NEXT:    movi v4.16b, #1
-; CHECK-NEXT:    movi v5.2d, #0000000000000000
-; CHECK-NEXT:    movi v6.2d, #0000000000000000
-; CHECK-NEXT:    mov v0.b[2], w2
-; CHECK-NEXT:    ld1 { v3.b }[2], [x12]
-; CHECK-NEXT:    add x12, sp, #424
-; CHECK-NEXT:    ld1 { v1.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #368
-; CHECK-NEXT:    movi v7.2d, #0000000000000000
-; CHECK-NEXT:    movi v16.8b, #1
-; CHECK-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v3.b }[3], [x13]
-; CHECK-NEXT:    add x13, sp, #432
-; CHECK-NEXT:    mov v0.b[3], w3
-; CHECK-NEXT:    ld1 { v1.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #376
-; CHECK-NEXT:    ld1 { v3.b }[4], [x14]
-; CHECK-NEXT:    ld1 { v1.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #384
-; CHECK-NEXT:    mov v0.b[4], w4
-; CHECK-NEXT:    ld1 { v1.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #392
-; CHECK-NEXT:    mov v0.b[5], w5
-; CHECK-NEXT:    ld1 { v1.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #16
-; CHECK-NEXT:    mov v0.b[6], w6
-; CHECK-NEXT:    ld1 { v1.b }[8], [x9]
-; CHECK-NEXT:    add x9, sp, #88
-; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #40
-; CHECK-NEXT:    ld1 { v1.b }[9], [x10]
-; CHECK-NEXT:    add x10, sp, #96
-; CHECK-NEXT:    mov v0.b[7], w7
-; CHECK-NEXT:    ld1 { v2.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #56
-; CHECK-NEXT:    ld1 { v1.b }[10], [x11]
-; CHECK-NEXT:    add x11, sp, #104
-; CHECK-NEXT:    ld1 { v2.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #72
-; CHECK-NEXT:    ld1 { v0.b }[8], [x8]
-; CHECK-NEXT:    add x8, sp, #24
-; CHECK-NEXT:    ld1 { v1.b }[11], [x12]
-; CHECK-NEXT:    add x12, sp, #112
-; CHECK-NEXT:    ld1 { v2.b }[4], [x12]
-; CHECK-NEXT:    add x12, sp, #440
-; CHECK-NEXT:    ld1 { v0.b }[9], [x8]
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    ld1 { v1.b }[12], [x13]
-; CHECK-NEXT:    add x13, sp, #504
-; CHECK-NEXT:    ld1 { v3.b }[5], [x13]
-; CHECK-NEXT:    add x13, sp, #512
-; CHECK-NEXT:    ld1 { v0.b }[10], [x8]
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    ld1 { v1.b }[13], [x12]
-; CHECK-NEXT:    add x12, sp, #448
-; CHECK-NEXT:    ld1 { v3.b }[6], [x13]
-; CHECK-NEXT:    ld1 { v0.b }[11], [x9]
-; CHECK-NEXT:    add x9, sp, #64
-; CHECK-NEXT:    ld1 { v1.b }[14], [x12]
-; CHECK-NEXT:    ld1 { v0.b }[12], [x8]
-; CHECK-NEXT:    add x8, sp, #120
-; CHECK-NEXT:    ld1 { v2.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #128
-; CHECK-NEXT:    ld1 { v0.b }[13], [x10]
-; CHECK-NEXT:    add x10, sp, #136
-; CHECK-NEXT:    ld1 { v2.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #456
-; CHECK-NEXT:    ld1 { v1.b }[15], [x8]
-; CHECK-NEXT:    ld1 { v0.b }[14], [x9]
-; CHECK-NEXT:    add x9, sp, #520
-; CHECK-NEXT:    ld1 { v2.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v3.b }[7], [x9]
-; CHECK-NEXT:    sdot v5.4s, v1.16b, v4.16b
-; CHECK-NEXT:    ld1 { v0.b }[15], [x11]
-; CHECK-NEXT:    sdot v17.2s, v2.8b, v16.8b
-; CHECK-NEXT:    sdot v7.2s, v3.8b, v16.8b
-; CHECK-NEXT:    sdot v6.4s, v0.16b, v4.16b
-; CHECK-NEXT:    addv s3, v5.4s
-; CHECK-NEXT:    addp v1.2s, v17.2s, v17.2s
-; CHECK-NEXT:    addp v2.2s, v7.2s, v7.2s
-; CHECK-NEXT:    fmov w10, s3
-; CHECK-NEXT:    addv s0, v6.4s
-; CHECK-NEXT:    fmov w9, s1
-; CHECK-NEXT:    fmov w11, s2
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w8, w8, w9
-; CHECK-NEXT:    add w9, w10, w11
-; CHECK-NEXT:    add w0, w8, w9
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v24i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr b1, [sp, #336]
+; CHECK-SD-NEXT:    add x8, sp, #344
+; CHECK-SD-NEXT:    add x9, sp, #400
+; CHECK-SD-NEXT:    ldr b2, [sp, #80]
+; CHECK-SD-NEXT:    ldr b3, [sp, #464]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #352
+; CHECK-SD-NEXT:    add x10, sp, #408
+; CHECK-SD-NEXT:    mov v0.b[1], w1
+; CHECK-SD-NEXT:    add x11, sp, #472
+; CHECK-SD-NEXT:    add x12, sp, #480
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #416
+; CHECK-SD-NEXT:    add x13, sp, #488
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #360
+; CHECK-SD-NEXT:    add x14, sp, #496
+; CHECK-SD-NEXT:    movi v4.16b, #1
+; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-SD-NEXT:    mov v0.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #424
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #368
+; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v16.8b, #1
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #432
+; CHECK-SD-NEXT:    mov v0.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #376
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x14]
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #384
+; CHECK-SD-NEXT:    mov v0.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #392
+; CHECK-SD-NEXT:    mov v0.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #16
+; CHECK-SD-NEXT:    mov v0.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #88
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #40
+; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #96
+; CHECK-SD-NEXT:    mov v0.b[7], w7
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #104
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #72
+; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
+; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #112
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #440
+; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #504
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #512
+; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #48
+; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #448
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #64
+; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x12]
+; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #120
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #128
+; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #136
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #456
+; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x8]
+; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #520
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v4.16b
+; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x11]
+; CHECK-SD-NEXT:    sdot v17.2s, v2.8b, v16.8b
+; CHECK-SD-NEXT:    sdot v7.2s, v3.8b, v16.8b
+; CHECK-SD-NEXT:    sdot v6.4s, v0.16b, v4.16b
+; CHECK-SD-NEXT:    addv s3, v5.4s
+; CHECK-SD-NEXT:    addp v1.2s, v17.2s, v17.2s
+; CHECK-SD-NEXT:    addp v2.2s, v7.2s, v7.2s
+; CHECK-SD-NEXT:    fmov w10, s3
+; CHECK-SD-NEXT:    addv s0, v6.4s
+; CHECK-SD-NEXT:    fmov w9, s1
+; CHECK-SD-NEXT:    fmov w11, s2
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w8, w8, w9
+; CHECK-SD-NEXT:    add w9, w10, w11
+; CHECK-SD-NEXT:    add w0, w8, w9
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v24i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    ldr w9, [sp, #336]
+; CHECK-GI-NEXT:    ldr w8, [sp, #344]
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    ldr w10, [sp, #16]
+; CHECK-GI-NEXT:    ldr w11, [sp, #88]
+; CHECK-GI-NEXT:    movi v4.8b, #1
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #464]
+; CHECK-GI-NEXT:    ldr w12, [sp, #400]
+; CHECK-GI-NEXT:    mov v0.b[1], w1
+; CHECK-GI-NEXT:    movi v5.8b, #1
+; CHECK-GI-NEXT:    movi v6.8b, #1
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #96]
+; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v1.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #352]
+; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v0.b[2], w2
+; CHECK-GI-NEXT:    mov v5.d[1], v4.d[0]
+; CHECK-GI-NEXT:    mov v6.d[1], v4.d[0]
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #360]
+; CHECK-GI-NEXT:    mov v0.b[3], w3
+; CHECK-GI-NEXT:    mov v1.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #368]
+; CHECK-GI-NEXT:    mov v0.b[4], w4
+; CHECK-GI-NEXT:    mov v1.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #376]
+; CHECK-GI-NEXT:    mov v0.b[5], w5
+; CHECK-GI-NEXT:    mov v1.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #384]
+; CHECK-GI-NEXT:    mov v0.b[6], w6
+; CHECK-GI-NEXT:    mov v1.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #392]
+; CHECK-GI-NEXT:    mov v0.b[7], w7
+; CHECK-GI-NEXT:    mov v1.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #80]
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #472]
+; CHECK-GI-NEXT:    mov v0.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #408]
+; CHECK-GI-NEXT:    mov v1.b[8], w12
+; CHECK-GI-NEXT:    mov v2.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #24]
+; CHECK-GI-NEXT:    mov v3.b[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #480]
+; CHECK-GI-NEXT:    mov v0.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #32]
+; CHECK-GI-NEXT:    mov v1.b[9], w10
+; CHECK-GI-NEXT:    mov v2.b[2], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #416]
+; CHECK-GI-NEXT:    mov v3.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #104]
+; CHECK-GI-NEXT:    ldr w11, [sp, #488]
+; CHECK-GI-NEXT:    mov v0.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #40]
+; CHECK-GI-NEXT:    mov v1.b[10], w10
+; CHECK-GI-NEXT:    mov v2.b[3], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #424]
+; CHECK-GI-NEXT:    mov v3.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #112]
+; CHECK-GI-NEXT:    ldr w11, [sp, #496]
+; CHECK-GI-NEXT:    mov v0.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #48]
+; CHECK-GI-NEXT:    mov v1.b[11], w10
+; CHECK-GI-NEXT:    mov v2.b[4], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #432]
+; CHECK-GI-NEXT:    mov v3.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #120]
+; CHECK-GI-NEXT:    ldr w11, [sp, #504]
+; CHECK-GI-NEXT:    mov v0.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #56]
+; CHECK-GI-NEXT:    mov v1.b[12], w10
+; CHECK-GI-NEXT:    mov v2.b[5], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #440]
+; CHECK-GI-NEXT:    mov v3.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #128]
+; CHECK-GI-NEXT:    ldr w11, [sp, #512]
+; CHECK-GI-NEXT:    mov v0.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #64]
+; CHECK-GI-NEXT:    mov v1.b[13], w10
+; CHECK-GI-NEXT:    mov v2.b[6], w11
+; CHECK-GI-NEXT:    ldr w10, [sp, #448]
+; CHECK-GI-NEXT:    mov v3.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #136]
+; CHECK-GI-NEXT:    ldr w11, [sp, #520]
+; CHECK-GI-NEXT:    mov v0.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #72]
+; CHECK-GI-NEXT:    mov v1.b[14], w10
+; CHECK-GI-NEXT:    mov v2.b[7], w11
+; CHECK-GI-NEXT:    mov v3.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #456]
+; CHECK-GI-NEXT:    mov v0.b[15], w8
+; CHECK-GI-NEXT:    mov v1.b[15], w9
+; CHECK-GI-NEXT:    fmov d2, d2
+; CHECK-GI-NEXT:    fmov d3, d3
+; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v5.16b
+; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v6.16b
+; CHECK-GI-NEXT:    sdot v7.4s, v2.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v17.4s, v3.16b, v4.16b
+; CHECK-GI-NEXT:    add v1.4s, v18.4s, v7.4s
+; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <24 x i8> %a to <24 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %az)
@@ -1442,17 +2110,31 @@ entry:
 }
 
 define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v32i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ldp q1, q3, [x0]
-; CHECK-NEXT:    ldp q2, q4, [x1]
-; CHECK-NEXT:    udot v0.4s, v4.16b, v3.16b
-; CHECK-NEXT:    udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q1, q3, [x0]
+; CHECK-SD-NEXT:    ldp q2, q4, [x1]
+; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q2, q3, [x0]
+; CHECK-GI-NEXT:    ldp q4, q5, [x1]
+; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b
+; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <32 x i8>, ptr %a
   %1 = zext <32 x i8> %0 to <32 x i32>
@@ -1465,16 +2147,29 @@ entry:
 }
 
 define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v32i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.16b, #1
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    udot v1.4s, v3.16b, v0.16b
-; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT:    addv s0, v1.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v32i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.16b, #1
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q2, q3, [x0]
+; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
+; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-SD-NEXT:    addv s0, v1.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v32i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.16b, #1
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q3, q4, [x0]
+; CHECK-GI-NEXT:    udot v2.4s, v3.16b, v0.16b
+; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v0.16b
+; CHECK-GI-NEXT:    add v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <32 x i8>, ptr %a1
   %1 = zext <32 x i8> %0 to <32 x i32>
@@ -1482,17 +2177,31 @@ entry:
   ret i32 %2
 }
 define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v32i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ldp q1, q3, [x0]
-; CHECK-NEXT:    ldp q2, q4, [x1]
-; CHECK-NEXT:    sdot v0.4s, v4.16b, v3.16b
-; CHECK-NEXT:    sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q1, q3, [x0]
+; CHECK-SD-NEXT:    ldp q2, q4, [x1]
+; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q2, q3, [x0]
+; CHECK-GI-NEXT:    ldp q4, q5, [x1]
+; CHECK-GI-NEXT:    sdot v1.4s, v4.16b, v2.16b
+; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <32 x i8>, ptr %a
   %1 = sext <32 x i8> %0 to <32 x i32>
@@ -1505,18 +2214,37 @@ entry:
 }
 
 define i32 @test_sdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
-; CHECK-LABEL: test_sdot_v32i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-NEXT:    sdot v17.4s, v1.16b, v3.16b
-; CHECK-NEXT:    sdot v16.4s, v5.16b, v7.16b
-; CHECK-NEXT:    sdot v17.4s, v0.16b, v2.16b
-; CHECK-NEXT:    sdot v16.4s, v4.16b, v6.16b
-; CHECK-NEXT:    add v0.4s, v17.4s, v16.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v32i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    sdot v17.4s, v1.16b, v3.16b
+; CHECK-SD-NEXT:    sdot v16.4s, v5.16b, v7.16b
+; CHECK-SD-NEXT:    sdot v17.4s, v0.16b, v2.16b
+; CHECK-SD-NEXT:    sdot v16.4s, v4.16b, v6.16b
+; CHECK-SD-NEXT:    add v0.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v32i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v2.16b
+; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v3.16b
+; CHECK-GI-NEXT:    sdot v17.4s, v5.16b, v7.16b
+; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v6.16b
+; CHECK-GI-NEXT:    add v0.4s, v16.4s, v18.4s
+; CHECK-GI-NEXT:    add v1.4s, v19.4s, v17.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <32 x i8> %a to <32 x i32>
   %bz = sext <32 x i8> %b to <32 x i32>
@@ -1531,19 +2259,39 @@ entry:
 }
 
 define i32 @test_sdot_v32i8_double_nomla(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
-; CHECK-LABEL: test_sdot_v32i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.16b, #1
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    movi v6.2d, #0000000000000000
-; CHECK-NEXT:    sdot v6.4s, v1.16b, v2.16b
-; CHECK-NEXT:    sdot v3.4s, v5.16b, v2.16b
-; CHECK-NEXT:    sdot v6.4s, v0.16b, v2.16b
-; CHECK-NEXT:    sdot v3.4s, v4.16b, v2.16b
-; CHECK-NEXT:    add v0.4s, v6.4s, v3.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v32i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v2.16b, #1
+; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-SD-NEXT:    sdot v6.4s, v1.16b, v2.16b
+; CHECK-SD-NEXT:    sdot v3.4s, v5.16b, v2.16b
+; CHECK-SD-NEXT:    sdot v6.4s, v0.16b, v2.16b
+; CHECK-SD-NEXT:    sdot v3.4s, v4.16b, v2.16b
+; CHECK-SD-NEXT:    add v0.4s, v6.4s, v3.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v32i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.16b, #1
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT:    sdot v3.4s, v0.16b, v2.16b
+; CHECK-GI-NEXT:    sdot v6.4s, v5.16b, v2.16b
+; CHECK-GI-NEXT:    sdot v7.4s, v1.16b, v2.16b
+; CHECK-GI-NEXT:    sdot v16.4s, v4.16b, v2.16b
+; CHECK-GI-NEXT:    add v0.4s, v3.4s, v7.4s
+; CHECK-GI-NEXT:    add v1.4s, v16.4s, v6.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <32 x i8> %a to <32 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az)
@@ -2119,20 +2867,43 @@ entry:
   ret i32 %x
 }
 define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v48i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ldr q1, [x0, #32]
-; CHECK-NEXT:    ldr q2, [x1, #32]
-; CHECK-NEXT:    udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT:    ldp q3, q1, [x0]
-; CHECK-NEXT:    ldp q4, q2, [x1]
-; CHECK-NEXT:    udot v0.4s, v4.16b, v3.16b
-; CHECK-NEXT:    udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v48i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldr q1, [x0, #32]
+; CHECK-SD-NEXT:    ldr q2, [x1, #32]
+; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    ldp q3, q1, [x0]
+; CHECK-SD-NEXT:    ldp q4, q2, [x1]
+; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v48i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr q7, [x0, #32]
+; CHECK-GI-NEXT:    ldp q3, q4, [x0]
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q5, q6, [x1]
+; CHECK-GI-NEXT:    ldr q16, [x1, #32]
+; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    udot v1.4s, v6.16b, v4.16b
+; CHECK-GI-NEXT:    udot v2.4s, v16.16b, v7.16b
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    addv s2, v2.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <48 x i8>, ptr %a
   %1 = zext <48 x i8> %0 to <48 x i32>
@@ -2145,18 +2916,39 @@ entry:
 }
 
 define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v48i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.16b, #1
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ldr q2, [x0, #32]
-; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT:    ldp q3, q2, [x0]
-; CHECK-NEXT:    udot v1.4s, v3.16b, v0.16b
-; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT:    addv s0, v1.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v48i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.16b, #1
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldr q2, [x0, #32]
+; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-SD-NEXT:    ldp q3, q2, [x0]
+; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
+; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-SD-NEXT:    addv s0, v1.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v48i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.16b, #1
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr q6, [x0, #32]
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q4, q5, [x0]
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v0.16b
+; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v0.16b
+; CHECK-GI-NEXT:    udot v3.4s, v6.16b, v0.16b
+; CHECK-GI-NEXT:    addv s0, v1.4s
+; CHECK-GI-NEXT:    addv s1, v2.4s
+; CHECK-GI-NEXT:    addv s2, v3.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <48 x i8>, ptr %a1
   %1 = zext <48 x i8> %0 to <48 x i32>
@@ -2164,20 +2956,43 @@ entry:
   ret i32 %2
 }
 define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v48i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ldr q1, [x0, #32]
-; CHECK-NEXT:    ldr q2, [x1, #32]
-; CHECK-NEXT:    sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT:    ldp q3, q1, [x0]
-; CHECK-NEXT:    ldp q4, q2, [x1]
-; CHECK-NEXT:    sdot v0.4s, v4.16b, v3.16b
-; CHECK-NEXT:    sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v48i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldr q1, [x0, #32]
+; CHECK-SD-NEXT:    ldr q2, [x1, #32]
+; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    ldp q3, q1, [x0]
+; CHECK-SD-NEXT:    ldp q4, q2, [x1]
+; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v48i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldr q7, [x0, #32]
+; CHECK-GI-NEXT:    ldp q3, q4, [x0]
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q5, q6, [x1]
+; CHECK-GI-NEXT:    ldr q16, [x1, #32]
+; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT:    sdot v1.4s, v6.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v2.4s, v16.16b, v7.16b
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    addv s2, v2.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <48 x i8>, ptr %a
   %1 = sext <48 x i8> %0 to <48 x i32>
@@ -2190,389 +3005,802 @@ entry:
 }
 
 define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
-; CHECK-LABEL: test_sdot_v48i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    ldr b3, [sp, #592]
-; CHECK-NEXT:    add x8, sp, #600
-; CHECK-NEXT:    ldr b6, [sp, #208]
-; CHECK-NEXT:    ldr b0, [sp, #336]
-; CHECK-NEXT:    add x9, sp, #344
-; CHECK-NEXT:    ldr b2, [sp, #464]
-; CHECK-NEXT:    ld1 { v3.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #216
-; CHECK-NEXT:    add x10, sp, #624
-; CHECK-NEXT:    ld1 { v6.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #608
-; CHECK-NEXT:    ld1 { v0.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #232
-; CHECK-NEXT:    fmov s1, w0
-; CHECK-NEXT:    ldr b7, [sp, #1360]
-; CHECK-NEXT:    ld1 { v3.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #224
-; CHECK-NEXT:    add x11, sp, #648
-; CHECK-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #616
-; CHECK-NEXT:    add x12, sp, #376
-; CHECK-NEXT:    mov v1.b[1], w1
-; CHECK-NEXT:    ldr b16, [sp, #976]
-; CHECK-NEXT:    add x14, sp, #288
-; CHECK-NEXT:    ld1 { v3.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #632
-; CHECK-NEXT:    add x15, sp, #408
-; CHECK-NEXT:    ld1 { v6.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #472
-; CHECK-NEXT:    add x13, sp, #696
-; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #240
-; CHECK-NEXT:    add x16, sp, #448
-; CHECK-NEXT:    ld1 { v3.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #352
-; CHECK-NEXT:    mov v1.b[2], w2
-; CHECK-NEXT:    ld1 { v6.b }[4], [x9]
-; CHECK-NEXT:    ld1 { v0.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #1368
-; CHECK-NEXT:    ld1 { v7.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #248
-; CHECK-NEXT:    add x9, sp, #640
-; CHECK-NEXT:    ld1 { v3.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #656
-; CHECK-NEXT:    movi v5.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v6.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #360
-; CHECK-NEXT:    mov v1.b[3], w3
-; CHECK-NEXT:    ld1 { v0.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #256
-; CHECK-NEXT:    movi v4.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v3.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #368
-; CHECK-NEXT:    ldr b17, [sp, #720]
-; CHECK-NEXT:    ld1 { v6.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #984
-; CHECK-NEXT:    ld1 { v0.b }[4], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #664
-; CHECK-NEXT:    ld1 { v3.b }[7], [x11]
-; CHECK-NEXT:    add x11, sp, #264
-; CHECK-NEXT:    mov v1.b[4], w4
-; CHECK-NEXT:    ld1 { v6.b }[7], [x11]
-; CHECK-NEXT:    add x9, sp, #672
-; CHECK-NEXT:    add x11, sp, #680
-; CHECK-NEXT:    ld1 { v0.b }[5], [x12]
-; CHECK-NEXT:    add x12, sp, #480
-; CHECK-NEXT:    ld1 { v2.b }[2], [x12]
-; CHECK-NEXT:    add x12, sp, #272
-; CHECK-NEXT:    ld1 { v3.b }[8], [x8]
-; CHECK-NEXT:    ld1 { v6.b }[8], [x12]
-; CHECK-NEXT:    add x12, sp, #384
-; CHECK-NEXT:    mov v1.b[5], w5
-; CHECK-NEXT:    ld1 { v0.b }[6], [x12]
-; CHECK-NEXT:    add x12, sp, #280
-; CHECK-NEXT:    add x8, sp, #688
-; CHECK-NEXT:    ld1 { v3.b }[9], [x10]
-; CHECK-NEXT:    add x10, sp, #1376
-; CHECK-NEXT:    ld1 { v7.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #392
-; CHECK-NEXT:    ld1 { v6.b }[9], [x12]
-; CHECK-NEXT:    ld1 { v0.b }[7], [x10]
-; CHECK-NEXT:    mov v1.b[6], w6
-; CHECK-NEXT:    add x12, sp, #704
-; CHECK-NEXT:    ld1 { v3.b }[10], [x9]
-; CHECK-NEXT:    add x9, sp, #400
-; CHECK-NEXT:    add x10, sp, #712
-; CHECK-NEXT:    ld1 { v6.b }[10], [x14]
-; CHECK-NEXT:    add x14, sp, #992
-; CHECK-NEXT:    ld1 { v0.b }[8], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[2], [x14]
-; CHECK-NEXT:    add x14, sp, #296
-; CHECK-NEXT:    ld1 { v3.b }[11], [x11]
-; CHECK-NEXT:    add x9, sp, #304
-; CHECK-NEXT:    add x11, sp, #312
-; CHECK-NEXT:    ld1 { v6.b }[11], [x14]
-; CHECK-NEXT:    mov v1.b[7], w7
-; CHECK-NEXT:    add x14, sp, #320
-; CHECK-NEXT:    ld1 { v0.b }[9], [x15]
-; CHECK-NEXT:    add x15, sp, #328
-; CHECK-NEXT:    ld1 { v3.b }[12], [x8]
-; CHECK-NEXT:    add x8, sp, #416
-; CHECK-NEXT:    ld1 { v6.b }[12], [x9]
-; CHECK-NEXT:    add x9, sp, #1384
-; CHECK-NEXT:    ld1 { v0.b }[10], [x8]
-; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #424
-; CHECK-NEXT:    ld1 { v3.b }[13], [x13]
-; CHECK-NEXT:    add x8, sp, #432
-; CHECK-NEXT:    add x13, sp, #440
-; CHECK-NEXT:    ld1 { v6.b }[13], [x11]
-; CHECK-NEXT:    add x11, sp, #16
-; CHECK-NEXT:    ld1 { v0.b }[11], [x9]
-; CHECK-NEXT:    add x9, sp, #1000
-; CHECK-NEXT:    ld1 { v1.b }[8], [x11]
-; CHECK-NEXT:    ld1 { v16.b }[3], [x9]
-; CHECK-NEXT:    ld1 { v3.b }[14], [x12]
-; CHECK-NEXT:    add x12, sp, #488
-; CHECK-NEXT:    ld1 { v6.b }[14], [x14]
-; CHECK-NEXT:    add x14, sp, #1392
-; CHECK-NEXT:    ld1 { v2.b }[3], [x12]
-; CHECK-NEXT:    ld1 { v7.b }[4], [x14]
-; CHECK-NEXT:    add x11, sp, #1008
-; CHECK-NEXT:    ld1 { v0.b }[12], [x8]
-; CHECK-NEXT:    ld1 { v16.b }[4], [x11]
-; CHECK-NEXT:    add x8, sp, #1400
-; CHECK-NEXT:    ld1 { v3.b }[15], [x10]
-; CHECK-NEXT:    add x10, sp, #496
-; CHECK-NEXT:    add x9, sp, #24
-; CHECK-NEXT:    ld1 { v6.b }[15], [x15]
-; CHECK-NEXT:    ld1 { v7.b }[5], [x8]
-; CHECK-NEXT:    ld1 { v2.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #1016
-; CHECK-NEXT:    ld1 { v16.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v0.b }[13], [x13]
-; CHECK-NEXT:    add x8, sp, #1408
-; CHECK-NEXT:    ld1 { v1.b }[9], [x9]
-; CHECK-NEXT:    add x9, sp, #504
-; CHECK-NEXT:    add x10, sp, #512
-; CHECK-NEXT:    ld1 { v7.b }[6], [x8]
-; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #1024
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    ld1 { v16.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v0.b }[14], [x16]
-; CHECK-NEXT:    ld1 { v1.b }[10], [x8]
-; CHECK-NEXT:    add x8, sp, #1416
-; CHECK-NEXT:    add x9, sp, #456
-; CHECK-NEXT:    ld1 { v7.b }[7], [x8]
-; CHECK-NEXT:    ld1 { v2.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #1032
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    ld1 { v16.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v0.b }[15], [x9]
-; CHECK-NEXT:    ld1 { v1.b }[11], [x8]
-; CHECK-NEXT:    add x8, sp, #1424
-; CHECK-NEXT:    add x9, sp, #520
-; CHECK-NEXT:    ld1 { v7.b }[8], [x8]
-; CHECK-NEXT:    ld1 { v2.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #1040
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    ld1 { v16.b }[8], [x9]
-; CHECK-NEXT:    add x10, sp, #528
-; CHECK-NEXT:    ld1 { v1.b }[12], [x8]
-; CHECK-NEXT:    add x8, sp, #1432
-; CHECK-NEXT:    sdot v5.4s, v6.16b, v3.16b
-; CHECK-NEXT:    ld1 { v7.b }[9], [x8]
-; CHECK-NEXT:    ld1 { v2.b }[8], [x10]
-; CHECK-NEXT:    add x8, sp, #1048
-; CHECK-NEXT:    ldr b3, [sp, #80]
-; CHECK-NEXT:    ld1 { v16.b }[9], [x8]
-; CHECK-NEXT:    add x10, sp, #88
-; CHECK-NEXT:    add x8, sp, #536
-; CHECK-NEXT:    add x11, sp, #1440
-; CHECK-NEXT:    add x9, sp, #56
-; CHECK-NEXT:    ld1 { v3.b }[1], [x10]
-; CHECK-NEXT:    ld1 { v2.b }[9], [x8]
-; CHECK-NEXT:    add x8, sp, #1056
-; CHECK-NEXT:    ld1 { v7.b }[10], [x11]
-; CHECK-NEXT:    ld1 { v16.b }[10], [x8]
-; CHECK-NEXT:    ld1 { v1.b }[13], [x9]
-; CHECK-NEXT:    add x9, sp, #96
-; CHECK-NEXT:    add x8, sp, #544
-; CHECK-NEXT:    add x10, sp, #1448
-; CHECK-NEXT:    ld1 { v3.b }[2], [x9]
-; CHECK-NEXT:    ld1 { v2.b }[10], [x8]
-; CHECK-NEXT:    add x8, sp, #1064
-; CHECK-NEXT:    ld1 { v7.b }[11], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[11], [x8]
-; CHECK-NEXT:    add x10, sp, #104
-; CHECK-NEXT:    add x8, sp, #552
-; CHECK-NEXT:    add x11, sp, #1456
-; CHECK-NEXT:    add x9, sp, #64
-; CHECK-NEXT:    ld1 { v3.b }[3], [x10]
-; CHECK-NEXT:    ld1 { v2.b }[11], [x8]
-; CHECK-NEXT:    add x8, sp, #1072
-; CHECK-NEXT:    ld1 { v7.b }[12], [x11]
-; CHECK-NEXT:    ld1 { v16.b }[12], [x8]
-; CHECK-NEXT:    ld1 { v1.b }[14], [x9]
-; CHECK-NEXT:    add x9, sp, #112
-; CHECK-NEXT:    add x8, sp, #560
-; CHECK-NEXT:    add x10, sp, #1464
-; CHECK-NEXT:    ld1 { v3.b }[4], [x9]
-; CHECK-NEXT:    ld1 { v2.b }[12], [x8]
-; CHECK-NEXT:    add x8, sp, #1080
-; CHECK-NEXT:    ld1 { v7.b }[13], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[13], [x8]
-; CHECK-NEXT:    add x10, sp, #120
-; CHECK-NEXT:    add x8, sp, #568
-; CHECK-NEXT:    add x11, sp, #1472
-; CHECK-NEXT:    add x9, sp, #72
-; CHECK-NEXT:    ld1 { v3.b }[5], [x10]
-; CHECK-NEXT:    ld1 { v2.b }[13], [x8]
-; CHECK-NEXT:    add x8, sp, #1088
-; CHECK-NEXT:    ld1 { v7.b }[14], [x11]
-; CHECK-NEXT:    ld1 { v16.b }[14], [x8]
-; CHECK-NEXT:    ld1 { v1.b }[15], [x9]
-; CHECK-NEXT:    add x9, sp, #128
-; CHECK-NEXT:    ldr b6, [sp, #1104]
-; CHECK-NEXT:    add x10, sp, #1480
-; CHECK-NEXT:    ld1 { v3.b }[6], [x9]
-; CHECK-NEXT:    add x8, sp, #1096
-; CHECK-NEXT:    add x9, sp, #1112
-; CHECK-NEXT:    ld1 { v7.b }[15], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[15], [x8]
-; CHECK-NEXT:    ld1 { v6.b }[1], [x9]
-; CHECK-NEXT:    add x8, sp, #728
-; CHECK-NEXT:    add x9, sp, #576
-; CHECK-NEXT:    add x10, sp, #136
-; CHECK-NEXT:    ld1 { v17.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #1120
-; CHECK-NEXT:    ld1 { v2.b }[14], [x9]
-; CHECK-NEXT:    sdot v4.4s, v16.16b, v7.16b
-; CHECK-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #736
-; CHECK-NEXT:    ldr b7, [sp, #1232]
-; CHECK-NEXT:    ldr b16, [sp, #848]
-; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v17.b }[2], [x8]
-; CHECK-NEXT:    add x9, sp, #1240
-; CHECK-NEXT:    add x10, sp, #856
-; CHECK-NEXT:    ld1 { v7.b }[1], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
-; CHECK-NEXT:    add x8, sp, #1128
-; CHECK-NEXT:    add x11, sp, #744
-; CHECK-NEXT:    ld1 { v6.b }[3], [x8]
-; CHECK-NEXT:    add x10, sp, #1248
-; CHECK-NEXT:    ld1 { v17.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #864
-; CHECK-NEXT:    add x9, sp, #144
-; CHECK-NEXT:    ld1 { v7.b }[2], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[2], [x11]
-; CHECK-NEXT:    add x8, sp, #1136
-; CHECK-NEXT:    add x12, sp, #752
-; CHECK-NEXT:    ld1 { v3.b }[8], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[4], [x8]
-; CHECK-NEXT:    ld1 { v17.b }[4], [x12]
-; CHECK-NEXT:    add x9, sp, #1256
-; CHECK-NEXT:    add x10, sp, #872
-; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[3], [x10]
-; CHECK-NEXT:    add x8, sp, #1144
-; CHECK-NEXT:    add x11, sp, #760
-; CHECK-NEXT:    ld1 { v6.b }[5], [x8]
-; CHECK-NEXT:    add x10, sp, #1264
-; CHECK-NEXT:    ld1 { v17.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #880
-; CHECK-NEXT:    add x9, sp, #152
-; CHECK-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[4], [x11]
-; CHECK-NEXT:    add x8, sp, #1152
-; CHECK-NEXT:    add x12, sp, #768
-; CHECK-NEXT:    ld1 { v3.b }[9], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[6], [x8]
-; CHECK-NEXT:    ld1 { v17.b }[6], [x12]
-; CHECK-NEXT:    add x9, sp, #1272
-; CHECK-NEXT:    add x10, sp, #888
-; CHECK-NEXT:    ld1 { v7.b }[5], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[5], [x10]
-; CHECK-NEXT:    add x8, sp, #1160
-; CHECK-NEXT:    add x11, sp, #776
-; CHECK-NEXT:    ld1 { v6.b }[7], [x8]
-; CHECK-NEXT:    add x10, sp, #1280
-; CHECK-NEXT:    ld1 { v17.b }[7], [x11]
-; CHECK-NEXT:    add x11, sp, #896
-; CHECK-NEXT:    add x9, sp, #160
-; CHECK-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[6], [x11]
-; CHECK-NEXT:    add x8, sp, #1168
-; CHECK-NEXT:    add x12, sp, #784
-; CHECK-NEXT:    ld1 { v3.b }[10], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[8], [x8]
-; CHECK-NEXT:    ld1 { v17.b }[8], [x12]
-; CHECK-NEXT:    add x9, sp, #1288
-; CHECK-NEXT:    add x10, sp, #904
-; CHECK-NEXT:    ld1 { v7.b }[7], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[7], [x10]
-; CHECK-NEXT:    add x8, sp, #1176
-; CHECK-NEXT:    add x11, sp, #792
-; CHECK-NEXT:    ld1 { v6.b }[9], [x8]
-; CHECK-NEXT:    add x10, sp, #1296
-; CHECK-NEXT:    ld1 { v17.b }[9], [x11]
-; CHECK-NEXT:    add x11, sp, #912
-; CHECK-NEXT:    add x9, sp, #168
-; CHECK-NEXT:    ld1 { v7.b }[8], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[8], [x11]
-; CHECK-NEXT:    add x8, sp, #1184
-; CHECK-NEXT:    add x12, sp, #800
-; CHECK-NEXT:    ld1 { v3.b }[11], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[10], [x8]
-; CHECK-NEXT:    ld1 { v17.b }[10], [x12]
-; CHECK-NEXT:    add x9, sp, #1304
-; CHECK-NEXT:    add x10, sp, #920
-; CHECK-NEXT:    ld1 { v7.b }[9], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[9], [x10]
-; CHECK-NEXT:    add x8, sp, #1192
-; CHECK-NEXT:    add x11, sp, #808
-; CHECK-NEXT:    ld1 { v6.b }[11], [x8]
-; CHECK-NEXT:    add x10, sp, #1312
-; CHECK-NEXT:    ld1 { v17.b }[11], [x11]
-; CHECK-NEXT:    add x11, sp, #928
-; CHECK-NEXT:    add x9, sp, #176
-; CHECK-NEXT:    ld1 { v7.b }[10], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[10], [x11]
-; CHECK-NEXT:    add x8, sp, #1200
-; CHECK-NEXT:    add x12, sp, #816
-; CHECK-NEXT:    ld1 { v3.b }[12], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[12], [x8]
-; CHECK-NEXT:    ld1 { v17.b }[12], [x12]
-; CHECK-NEXT:    add x9, sp, #1320
-; CHECK-NEXT:    add x10, sp, #936
-; CHECK-NEXT:    ld1 { v7.b }[11], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[11], [x10]
-; CHECK-NEXT:    add x8, sp, #1208
-; CHECK-NEXT:    add x11, sp, #824
-; CHECK-NEXT:    ld1 { v6.b }[13], [x8]
-; CHECK-NEXT:    add x10, sp, #1328
-; CHECK-NEXT:    ld1 { v17.b }[13], [x11]
-; CHECK-NEXT:    add x11, sp, #944
-; CHECK-NEXT:    add x9, sp, #184
-; CHECK-NEXT:    ld1 { v7.b }[12], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[12], [x11]
-; CHECK-NEXT:    add x8, sp, #1216
-; CHECK-NEXT:    add x12, sp, #832
-; CHECK-NEXT:    ld1 { v3.b }[13], [x9]
-; CHECK-NEXT:    ld1 { v6.b }[14], [x8]
-; CHECK-NEXT:    ld1 { v17.b }[14], [x12]
-; CHECK-NEXT:    add x9, sp, #1336
-; CHECK-NEXT:    add x10, sp, #952
-; CHECK-NEXT:    ld1 { v7.b }[13], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[13], [x10]
-; CHECK-NEXT:    add x8, sp, #1224
-; CHECK-NEXT:    add x11, sp, #840
-; CHECK-NEXT:    ld1 { v6.b }[15], [x8]
-; CHECK-NEXT:    add x8, sp, #192
-; CHECK-NEXT:    ld1 { v17.b }[15], [x11]
-; CHECK-NEXT:    add x10, sp, #1344
-; CHECK-NEXT:    add x11, sp, #960
-; CHECK-NEXT:    ld1 { v3.b }[14], [x8]
-; CHECK-NEXT:    ld1 { v7.b }[14], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[14], [x11]
-; CHECK-NEXT:    add x9, sp, #584
-; CHECK-NEXT:    sdot v5.4s, v1.16b, v0.16b
-; CHECK-NEXT:    add x8, sp, #200
-; CHECK-NEXT:    sdot v4.4s, v17.16b, v6.16b
-; CHECK-NEXT:    ld1 { v2.b }[15], [x9]
-; CHECK-NEXT:    add x9, sp, #1352
-; CHECK-NEXT:    add x10, sp, #968
-; CHECK-NEXT:    ld1 { v3.b }[15], [x8]
-; CHECK-NEXT:    ld1 { v7.b }[15], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[15], [x10]
-; CHECK-NEXT:    sdot v5.4s, v3.16b, v2.16b
-; CHECK-NEXT:    sdot v4.4s, v16.16b, v7.16b
-; CHECK-NEXT:    add v0.4s, v5.4s, v4.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v48i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    ldr b3, [sp, #592]
+; CHECK-SD-NEXT:    add x8, sp, #600
+; CHECK-SD-NEXT:    ldr b6, [sp, #208]
+; CHECK-SD-NEXT:    ldr b0, [sp, #336]
+; CHECK-SD-NEXT:    add x9, sp, #344
+; CHECK-SD-NEXT:    ldr b2, [sp, #464]
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #216
+; CHECK-SD-NEXT:    add x10, sp, #624
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #608
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #232
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    ldr b7, [sp, #1360]
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #224
+; CHECK-SD-NEXT:    add x11, sp, #648
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #616
+; CHECK-SD-NEXT:    add x12, sp, #376
+; CHECK-SD-NEXT:    mov v1.b[1], w1
+; CHECK-SD-NEXT:    ldr b16, [sp, #976]
+; CHECK-SD-NEXT:    add x14, sp, #288
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #632
+; CHECK-SD-NEXT:    add x15, sp, #408
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #472
+; CHECK-SD-NEXT:    add x13, sp, #696
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #240
+; CHECK-SD-NEXT:    add x16, sp, #448
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #352
+; CHECK-SD-NEXT:    mov v1.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #1368
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #248
+; CHECK-SD-NEXT:    add x9, sp, #640
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #656
+; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #360
+; CHECK-SD-NEXT:    mov v1.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #256
+; CHECK-SD-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #368
+; CHECK-SD-NEXT:    ldr b17, [sp, #720]
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #984
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #664
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #264
+; CHECK-SD-NEXT:    mov v1.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #672
+; CHECK-SD-NEXT:    add x11, sp, #680
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #480
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #272
+; CHECK-SD-NEXT:    ld1 { v3.b }[8], [x8]
+; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #384
+; CHECK-SD-NEXT:    mov v1.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #280
+; CHECK-SD-NEXT:    add x8, sp, #688
+; CHECK-SD-NEXT:    ld1 { v3.b }[9], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #1376
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #392
+; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x12]
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x10]
+; CHECK-SD-NEXT:    mov v1.b[6], w6
+; CHECK-SD-NEXT:    add x12, sp, #704
+; CHECK-SD-NEXT:    ld1 { v3.b }[10], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #400
+; CHECK-SD-NEXT:    add x10, sp, #712
+; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #992
+; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #296
+; CHECK-SD-NEXT:    ld1 { v3.b }[11], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #304
+; CHECK-SD-NEXT:    add x11, sp, #312
+; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x14]
+; CHECK-SD-NEXT:    mov v1.b[7], w7
+; CHECK-SD-NEXT:    add x14, sp, #320
+; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #328
+; CHECK-SD-NEXT:    ld1 { v3.b }[12], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #416
+; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #1384
+; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #424
+; CHECK-SD-NEXT:    ld1 { v3.b }[13], [x13]
+; CHECK-SD-NEXT:    add x8, sp, #432
+; CHECK-SD-NEXT:    add x13, sp, #440
+; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #16
+; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #1000
+; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x11]
+; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[14], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #488
+; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #1392
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x14]
+; CHECK-SD-NEXT:    add x11, sp, #1008
+; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x8]
+; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #1400
+; CHECK-SD-NEXT:    ld1 { v3.b }[15], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #496
+; CHECK-SD-NEXT:    add x9, sp, #24
+; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x15]
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #1016
+; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x13]
+; CHECK-SD-NEXT:    add x8, sp, #1408
+; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #504
+; CHECK-SD-NEXT:    add x10, sp, #512
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #1024
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x16]
+; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1416
+; CHECK-SD-NEXT:    add x9, sp, #456
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #1032
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x9]
+; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1424
+; CHECK-SD-NEXT:    add x9, sp, #520
+; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #1040
+; CHECK-SD-NEXT:    add x8, sp, #48
+; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x9]
+; CHECK-SD-NEXT:    add x10, sp, #528
+; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1432
+; CHECK-SD-NEXT:    sdot v5.4s, v6.16b, v3.16b
+; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[8], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1048
+; CHECK-SD-NEXT:    ldr b3, [sp, #80]
+; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #88
+; CHECK-SD-NEXT:    add x8, sp, #536
+; CHECK-SD-NEXT:    add x11, sp, #1440
+; CHECK-SD-NEXT:    add x9, sp, #56
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v2.b }[9], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1056
+; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x11]
+; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #96
+; CHECK-SD-NEXT:    add x8, sp, #544
+; CHECK-SD-NEXT:    add x10, sp, #1448
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x9]
+; CHECK-SD-NEXT:    ld1 { v2.b }[10], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1064
+; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #104
+; CHECK-SD-NEXT:    add x8, sp, #552
+; CHECK-SD-NEXT:    add x11, sp, #1456
+; CHECK-SD-NEXT:    add x9, sp, #64
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x10]
+; CHECK-SD-NEXT:    ld1 { v2.b }[11], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1072
+; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x11]
+; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #112
+; CHECK-SD-NEXT:    add x8, sp, #560
+; CHECK-SD-NEXT:    add x10, sp, #1464
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v2.b }[12], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1080
+; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #120
+; CHECK-SD-NEXT:    add x8, sp, #568
+; CHECK-SD-NEXT:    add x11, sp, #1472
+; CHECK-SD-NEXT:    add x9, sp, #72
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v2.b }[13], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1088
+; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x11]
+; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #128
+; CHECK-SD-NEXT:    ldr b6, [sp, #1104]
+; CHECK-SD-NEXT:    add x10, sp, #1480
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #1096
+; CHECK-SD-NEXT:    add x9, sp, #1112
+; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x8]
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #728
+; CHECK-SD-NEXT:    add x9, sp, #576
+; CHECK-SD-NEXT:    add x10, sp, #136
+; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1120
+; CHECK-SD-NEXT:    ld1 { v2.b }[14], [x9]
+; CHECK-SD-NEXT:    sdot v4.4s, v16.16b, v7.16b
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #736
+; CHECK-SD-NEXT:    ldr b7, [sp, #1232]
+; CHECK-SD-NEXT:    ldr b16, [sp, #848]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x8]
+; CHECK-SD-NEXT:    add x9, sp, #1240
+; CHECK-SD-NEXT:    add x10, sp, #856
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1128
+; CHECK-SD-NEXT:    add x11, sp, #744
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #1248
+; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #864
+; CHECK-SD-NEXT:    add x9, sp, #144
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #1136
+; CHECK-SD-NEXT:    add x12, sp, #752
+; CHECK-SD-NEXT:    ld1 { v3.b }[8], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #1256
+; CHECK-SD-NEXT:    add x10, sp, #872
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1144
+; CHECK-SD-NEXT:    add x11, sp, #760
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #1264
+; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #880
+; CHECK-SD-NEXT:    add x9, sp, #152
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #1152
+; CHECK-SD-NEXT:    add x12, sp, #768
+; CHECK-SD-NEXT:    ld1 { v3.b }[9], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #1272
+; CHECK-SD-NEXT:    add x10, sp, #888
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1160
+; CHECK-SD-NEXT:    add x11, sp, #776
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #1280
+; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #896
+; CHECK-SD-NEXT:    add x9, sp, #160
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #1168
+; CHECK-SD-NEXT:    add x12, sp, #784
+; CHECK-SD-NEXT:    ld1 { v3.b }[10], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x8]
+; CHECK-SD-NEXT:    ld1 { v17.b }[8], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #1288
+; CHECK-SD-NEXT:    add x10, sp, #904
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1176
+; CHECK-SD-NEXT:    add x11, sp, #792
+; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #1296
+; CHECK-SD-NEXT:    ld1 { v17.b }[9], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #912
+; CHECK-SD-NEXT:    add x9, sp, #168
+; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #1184
+; CHECK-SD-NEXT:    add x12, sp, #800
+; CHECK-SD-NEXT:    ld1 { v3.b }[11], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x8]
+; CHECK-SD-NEXT:    ld1 { v17.b }[10], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #1304
+; CHECK-SD-NEXT:    add x10, sp, #920
+; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1192
+; CHECK-SD-NEXT:    add x11, sp, #808
+; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #1312
+; CHECK-SD-NEXT:    ld1 { v17.b }[11], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #928
+; CHECK-SD-NEXT:    add x9, sp, #176
+; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #1200
+; CHECK-SD-NEXT:    add x12, sp, #816
+; CHECK-SD-NEXT:    ld1 { v3.b }[12], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x8]
+; CHECK-SD-NEXT:    ld1 { v17.b }[12], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #1320
+; CHECK-SD-NEXT:    add x10, sp, #936
+; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1208
+; CHECK-SD-NEXT:    add x11, sp, #824
+; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #1328
+; CHECK-SD-NEXT:    ld1 { v17.b }[13], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #944
+; CHECK-SD-NEXT:    add x9, sp, #184
+; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #1216
+; CHECK-SD-NEXT:    add x12, sp, #832
+; CHECK-SD-NEXT:    ld1 { v3.b }[13], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x8]
+; CHECK-SD-NEXT:    ld1 { v17.b }[14], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #1336
+; CHECK-SD-NEXT:    add x10, sp, #952
+; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x10]
+; CHECK-SD-NEXT:    add x8, sp, #1224
+; CHECK-SD-NEXT:    add x11, sp, #840
+; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #192
+; CHECK-SD-NEXT:    ld1 { v17.b }[15], [x11]
+; CHECK-SD-NEXT:    add x10, sp, #1344
+; CHECK-SD-NEXT:    add x11, sp, #960
+; CHECK-SD-NEXT:    ld1 { v3.b }[14], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x10]
+; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #584
+; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v0.16b
+; CHECK-SD-NEXT:    add x8, sp, #200
+; CHECK-SD-NEXT:    sdot v4.4s, v17.16b, v6.16b
+; CHECK-SD-NEXT:    ld1 { v2.b }[15], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #1352
+; CHECK-SD-NEXT:    add x10, sp, #968
+; CHECK-SD-NEXT:    ld1 { v3.b }[15], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x9]
+; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x10]
+; CHECK-SD-NEXT:    sdot v5.4s, v3.16b, v2.16b
+; CHECK-SD-NEXT:    sdot v4.4s, v16.16b, v7.16b
+; CHECK-SD-NEXT:    add v0.4s, v5.4s, v4.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v48i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    ldr w11, [sp, #80]
+; CHECK-GI-NEXT:    ldr w10, [sp, #208]
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    ldr w8, [sp, #88]
+; CHECK-GI-NEXT:    ldr w12, [sp, #344]
+; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT:    fmov s1, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #336]
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #464]
+; CHECK-GI-NEXT:    ldr w9, [sp, #216]
+; CHECK-GI-NEXT:    mov v0.b[1], w1
+; CHECK-GI-NEXT:    fmov s3, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #600]
+; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v1.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #592]
+; CHECK-GI-NEXT:    fmov s4, w10
+; CHECK-GI-NEXT:    mov v2.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #472]
+; CHECK-GI-NEXT:    ldr w10, [sp, #608]
+; CHECK-GI-NEXT:    mov v3.b[1], w12
+; CHECK-GI-NEXT:    fmov s5, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #96]
+; CHECK-GI-NEXT:    mov v4.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #224]
+; CHECK-GI-NEXT:    mov v0.b[2], w2
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #352]
+; CHECK-GI-NEXT:    ldr w12, [sp, #848]
+; CHECK-GI-NEXT:    mov v2.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #480]
+; CHECK-GI-NEXT:    mov v5.b[1], w11
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #104]
+; CHECK-GI-NEXT:    ldr w11, [sp, #16]
+; CHECK-GI-NEXT:    mov v4.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #232]
+; CHECK-GI-NEXT:    mov v0.b[3], w3
+; CHECK-GI-NEXT:    mov v1.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #360]
+; CHECK-GI-NEXT:    fmov s7, w12
+; CHECK-GI-NEXT:    mov v2.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #488]
+; CHECK-GI-NEXT:    mov v5.b[2], w10
+; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #112]
+; CHECK-GI-NEXT:    ldr w10, [sp, #616]
+; CHECK-GI-NEXT:    mov v4.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #240]
+; CHECK-GI-NEXT:    mov v0.b[4], w4
+; CHECK-GI-NEXT:    mov v1.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #368]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1112]
+; CHECK-GI-NEXT:    mov v2.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #496]
+; CHECK-GI-NEXT:    mov v5.b[3], w10
+; CHECK-GI-NEXT:    mov v3.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #120]
+; CHECK-GI-NEXT:    ldr w10, [sp, #624]
+; CHECK-GI-NEXT:    mov v4.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #248]
+; CHECK-GI-NEXT:    mov v0.b[5], w5
+; CHECK-GI-NEXT:    mov v1.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #376]
+; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v2.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #504]
+; CHECK-GI-NEXT:    mov v5.b[4], w10
+; CHECK-GI-NEXT:    mov v3.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #128]
+; CHECK-GI-NEXT:    ldr w10, [sp, #632]
+; CHECK-GI-NEXT:    mov v4.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #256]
+; CHECK-GI-NEXT:    mov v0.b[6], w6
+; CHECK-GI-NEXT:    mov v1.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #384]
+; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v2.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #512]
+; CHECK-GI-NEXT:    mov v5.b[5], w10
+; CHECK-GI-NEXT:    mov v3.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #136]
+; CHECK-GI-NEXT:    ldr w10, [sp, #640]
+; CHECK-GI-NEXT:    mov v4.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #264]
+; CHECK-GI-NEXT:    mov v0.b[7], w7
+; CHECK-GI-NEXT:    mov v1.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #392]
+; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v2.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #520]
+; CHECK-GI-NEXT:    mov v5.b[6], w10
+; CHECK-GI-NEXT:    mov v3.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #144]
+; CHECK-GI-NEXT:    ldr w10, [sp, #648]
+; CHECK-GI-NEXT:    mov v4.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #272]
+; CHECK-GI-NEXT:    mov v0.b[8], w11
+; CHECK-GI-NEXT:    mov v1.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #400]
+; CHECK-GI-NEXT:    ldr w11, [sp, #24]
+; CHECK-GI-NEXT:    mov v2.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #528]
+; CHECK-GI-NEXT:    mov v5.b[7], w10
+; CHECK-GI-NEXT:    mov v3.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #152]
+; CHECK-GI-NEXT:    ldr w10, [sp, #656]
+; CHECK-GI-NEXT:    mov v4.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #280]
+; CHECK-GI-NEXT:    mov v0.b[9], w11
+; CHECK-GI-NEXT:    mov v1.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #408]
+; CHECK-GI-NEXT:    ldr w11, [sp, #32]
+; CHECK-GI-NEXT:    mov v2.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #536]
+; CHECK-GI-NEXT:    mov v5.b[8], w10
+; CHECK-GI-NEXT:    mov v3.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #160]
+; CHECK-GI-NEXT:    ldr w10, [sp, #664]
+; CHECK-GI-NEXT:    mov v4.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #288]
+; CHECK-GI-NEXT:    mov v0.b[10], w11
+; CHECK-GI-NEXT:    mov v1.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #416]
+; CHECK-GI-NEXT:    ldr w11, [sp, #40]
+; CHECK-GI-NEXT:    mov v2.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #544]
+; CHECK-GI-NEXT:    mov v5.b[9], w10
+; CHECK-GI-NEXT:    mov v3.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #168]
+; CHECK-GI-NEXT:    ldr w10, [sp, #672]
+; CHECK-GI-NEXT:    mov v4.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #296]
+; CHECK-GI-NEXT:    mov v0.b[11], w11
+; CHECK-GI-NEXT:    mov v1.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #424]
+; CHECK-GI-NEXT:    ldr w11, [sp, #48]
+; CHECK-GI-NEXT:    mov v2.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #552]
+; CHECK-GI-NEXT:    mov v5.b[10], w10
+; CHECK-GI-NEXT:    mov v3.b[11], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #176]
+; CHECK-GI-NEXT:    ldr w10, [sp, #680]
+; CHECK-GI-NEXT:    mov v4.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #304]
+; CHECK-GI-NEXT:    mov v0.b[12], w11
+; CHECK-GI-NEXT:    mov v1.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #432]
+; CHECK-GI-NEXT:    ldr w11, [sp, #56]
+; CHECK-GI-NEXT:    mov v2.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #560]
+; CHECK-GI-NEXT:    mov v5.b[11], w10
+; CHECK-GI-NEXT:    mov v3.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #184]
+; CHECK-GI-NEXT:    ldr w10, [sp, #688]
+; CHECK-GI-NEXT:    mov v4.b[12], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #312]
+; CHECK-GI-NEXT:    mov v0.b[13], w11
+; CHECK-GI-NEXT:    mov v1.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #440]
+; CHECK-GI-NEXT:    ldr w11, [sp, #64]
+; CHECK-GI-NEXT:    mov v2.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #568]
+; CHECK-GI-NEXT:    mov v5.b[12], w10
+; CHECK-GI-NEXT:    mov v3.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #192]
+; CHECK-GI-NEXT:    ldr w10, [sp, #696]
+; CHECK-GI-NEXT:    mov v4.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #320]
+; CHECK-GI-NEXT:    mov v0.b[14], w11
+; CHECK-GI-NEXT:    mov v1.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #448]
+; CHECK-GI-NEXT:    ldr w11, [sp, #72]
+; CHECK-GI-NEXT:    mov v2.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #576]
+; CHECK-GI-NEXT:    mov v5.b[13], w10
+; CHECK-GI-NEXT:    mov v3.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #720]
+; CHECK-GI-NEXT:    ldr w10, [sp, #704]
+; CHECK-GI-NEXT:    mov v4.b[14], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #728]
+; CHECK-GI-NEXT:    mov v0.b[15], w11
+; CHECK-GI-NEXT:    fmov s6, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #328]
+; CHECK-GI-NEXT:    ldr w11, [sp, #456]
+; CHECK-GI-NEXT:    mov v5.b[14], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #200]
+; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v2.b[15], w8
+; CHECK-GI-NEXT:    mov v3.b[15], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #736]
+; CHECK-GI-NEXT:    mov v6.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #584]
+; CHECK-GI-NEXT:    ldr w8, [sp, #856]
+; CHECK-GI-NEXT:    mov v1.b[15], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #712]
+; CHECK-GI-NEXT:    mov v4.b[15], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #976]
+; CHECK-GI-NEXT:    mov v7.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1232]
+; CHECK-GI-NEXT:    mov v5.b[15], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #984]
+; CHECK-GI-NEXT:    mov v6.b[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1104]
+; CHECK-GI-NEXT:    fmov s16, w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1360]
+; CHECK-GI-NEXT:    fmov s18, w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1368]
+; CHECK-GI-NEXT:    fmov s17, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1240]
+; CHECK-GI-NEXT:    sdot v20.4s, v0.16b, v3.16b
+; CHECK-GI-NEXT:    mov v16.b[1], w10
+; CHECK-GI-NEXT:    fmov s19, w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #864]
+; CHECK-GI-NEXT:    mov v18.b[1], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #992]
+; CHECK-GI-NEXT:    ldr w9, [sp, #1120]
+; CHECK-GI-NEXT:    mov v17.b[1], w12
+; CHECK-GI-NEXT:    mov v7.b[2], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1248]
+; CHECK-GI-NEXT:    mov v19.b[1], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #744]
+; CHECK-GI-NEXT:    sdot v21.4s, v1.16b, v4.16b
+; CHECK-GI-NEXT:    mov v16.b[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #872]
+; CHECK-GI-NEXT:    addv s0, v20.4s
+; CHECK-GI-NEXT:    mov v6.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1000]
+; CHECK-GI-NEXT:    mov v18.b[2], w10
+; CHECK-GI-NEXT:    mov v17.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1376]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1128]
+; CHECK-GI-NEXT:    mov v7.b[3], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #880]
+; CHECK-GI-NEXT:    addv s1, v21.4s
+; CHECK-GI-NEXT:    mov v19.b[2], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #752]
+; CHECK-GI-NEXT:    mov v16.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1256]
+; CHECK-GI-NEXT:    sdot v25.4s, v2.16b, v5.16b
+; CHECK-GI-NEXT:    mov v17.b[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1384]
+; CHECK-GI-NEXT:    mov v6.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1008]
+; CHECK-GI-NEXT:    mov v18.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1136]
+; CHECK-GI-NEXT:    mov v19.b[3], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #760]
+; CHECK-GI-NEXT:    mov v7.b[4], w11
+; CHECK-GI-NEXT:    mov v16.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1264]
+; CHECK-GI-NEXT:    ldr w11, [sp, #888]
+; CHECK-GI-NEXT:    mov v17.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1392]
+; CHECK-GI-NEXT:    mov v6.b[5], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1016]
+; CHECK-GI-NEXT:    mov v18.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1144]
+; CHECK-GI-NEXT:    mov v19.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #768]
+; CHECK-GI-NEXT:    mov v7.b[5], w11
+; CHECK-GI-NEXT:    mov v16.b[5], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1272]
+; CHECK-GI-NEXT:    ldr w11, [sp, #896]
+; CHECK-GI-NEXT:    mov v17.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1400]
+; CHECK-GI-NEXT:    mov v6.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1024]
+; CHECK-GI-NEXT:    mov v18.b[5], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1152]
+; CHECK-GI-NEXT:    mov v19.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #776]
+; CHECK-GI-NEXT:    mov v7.b[6], w11
+; CHECK-GI-NEXT:    mov v16.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1280]
+; CHECK-GI-NEXT:    ldr w11, [sp, #904]
+; CHECK-GI-NEXT:    mov v17.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1408]
+; CHECK-GI-NEXT:    mov v6.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1032]
+; CHECK-GI-NEXT:    mov v18.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1160]
+; CHECK-GI-NEXT:    mov v19.b[6], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #784]
+; CHECK-GI-NEXT:    mov v7.b[7], w11
+; CHECK-GI-NEXT:    mov v16.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1288]
+; CHECK-GI-NEXT:    ldr w11, [sp, #912]
+; CHECK-GI-NEXT:    mov v17.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1416]
+; CHECK-GI-NEXT:    mov v6.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1040]
+; CHECK-GI-NEXT:    mov v18.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1168]
+; CHECK-GI-NEXT:    mov v19.b[7], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #792]
+; CHECK-GI-NEXT:    mov v7.b[8], w11
+; CHECK-GI-NEXT:    mov v16.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1296]
+; CHECK-GI-NEXT:    ldr w11, [sp, #920]
+; CHECK-GI-NEXT:    mov v17.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1424]
+; CHECK-GI-NEXT:    mov v6.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1048]
+; CHECK-GI-NEXT:    mov v18.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1176]
+; CHECK-GI-NEXT:    mov v19.b[8], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #800]
+; CHECK-GI-NEXT:    mov v7.b[9], w11
+; CHECK-GI-NEXT:    mov v16.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1304]
+; CHECK-GI-NEXT:    ldr w11, [sp, #928]
+; CHECK-GI-NEXT:    mov v17.b[9], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1432]
+; CHECK-GI-NEXT:    mov v6.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1056]
+; CHECK-GI-NEXT:    mov v18.b[9], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1184]
+; CHECK-GI-NEXT:    mov v19.b[9], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #808]
+; CHECK-GI-NEXT:    mov v7.b[10], w11
+; CHECK-GI-NEXT:    mov v16.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1312]
+; CHECK-GI-NEXT:    ldr w11, [sp, #936]
+; CHECK-GI-NEXT:    mov v17.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1440]
+; CHECK-GI-NEXT:    mov v6.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1064]
+; CHECK-GI-NEXT:    mov v18.b[10], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1192]
+; CHECK-GI-NEXT:    mov v19.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #816]
+; CHECK-GI-NEXT:    mov v7.b[11], w11
+; CHECK-GI-NEXT:    mov v16.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1320]
+; CHECK-GI-NEXT:    ldr w11, [sp, #944]
+; CHECK-GI-NEXT:    mov v17.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1448]
+; CHECK-GI-NEXT:    mov v6.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1072]
+; CHECK-GI-NEXT:    mov v18.b[11], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1200]
+; CHECK-GI-NEXT:    mov v19.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #824]
+; CHECK-GI-NEXT:    mov v7.b[12], w11
+; CHECK-GI-NEXT:    mov v16.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1328]
+; CHECK-GI-NEXT:    ldr w11, [sp, #952]
+; CHECK-GI-NEXT:    mov v17.b[12], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1456]
+; CHECK-GI-NEXT:    mov v6.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1080]
+; CHECK-GI-NEXT:    mov v18.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1208]
+; CHECK-GI-NEXT:    mov v19.b[12], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #832]
+; CHECK-GI-NEXT:    mov v7.b[13], w11
+; CHECK-GI-NEXT:    mov v16.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1336]
+; CHECK-GI-NEXT:    ldr w11, [sp, #960]
+; CHECK-GI-NEXT:    mov v17.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1464]
+; CHECK-GI-NEXT:    mov v6.b[14], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1088]
+; CHECK-GI-NEXT:    mov v18.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #1216]
+; CHECK-GI-NEXT:    mov v19.b[13], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #840]
+; CHECK-GI-NEXT:    mov v7.b[14], w11
+; CHECK-GI-NEXT:    mov v16.b[14], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #1344]
+; CHECK-GI-NEXT:    ldr w11, [sp, #968]
+; CHECK-GI-NEXT:    mov v17.b[14], w9
+; CHECK-GI-NEXT:    mov v6.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1096]
+; CHECK-GI-NEXT:    mov v18.b[14], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #1472]
+; CHECK-GI-NEXT:    ldr w10, [sp, #1224]
+; CHECK-GI-NEXT:    mov v7.b[15], w11
+; CHECK-GI-NEXT:    addv s4, v25.4s
+; CHECK-GI-NEXT:    mov v16.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #1352]
+; CHECK-GI-NEXT:    mov v19.b[14], w9
+; CHECK-GI-NEXT:    mov v17.b[15], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #1480]
+; CHECK-GI-NEXT:    mov v18.b[15], w8
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w11, s4
+; CHECK-GI-NEXT:    mov v19.b[15], w9
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    sdot v22.4s, v6.16b, v17.16b
+; CHECK-GI-NEXT:    sdot v23.4s, v7.16b, v18.16b
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    sdot v24.4s, v16.16b, v19.16b
+; CHECK-GI-NEXT:    add w8, w8, w11
+; CHECK-GI-NEXT:    addv s2, v22.4s
+; CHECK-GI-NEXT:    addv s3, v23.4s
+; CHECK-GI-NEXT:    addv s5, v24.4s
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    fmov w10, s3
+; CHECK-GI-NEXT:    add w9, w9, w10
+; CHECK-GI-NEXT:    fmov w10, s5
+; CHECK-GI-NEXT:    add w9, w9, w10
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <48 x i8> %a to <48 x i32>
   %bz = sext <48 x i8> %b to <48 x i32>
@@ -2587,204 +3815,426 @@ entry:
 }
 
 define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
-; CHECK-LABEL: test_sdot_v48i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    ldr b5, [sp, #208]
-; CHECK-NEXT:    add x8, sp, #216
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    ldr b4, [sp, #976]
-; CHECK-NEXT:    add x9, sp, #984
-; CHECK-NEXT:    add x12, sp, #328
-; CHECK-NEXT:    ld1 { v5.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #224
-; CHECK-NEXT:    movi v1.16b, #1
-; CHECK-NEXT:    mov v0.b[1], w1
-; CHECK-NEXT:    ld1 { v4.b }[1], [x9]
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    add x11, sp, #992
-; CHECK-NEXT:    ldr b6, [sp, #720]
-; CHECK-NEXT:    ldr b7, [sp, #80]
-; CHECK-NEXT:    ld1 { v5.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #232
-; CHECK-NEXT:    add x13, sp, #88
-; CHECK-NEXT:    ld1 { v4.b }[2], [x11]
-; CHECK-NEXT:    ld1 { v7.b }[1], [x13]
-; CHECK-NEXT:    add x13, sp, #856
-; CHECK-NEXT:    mov v0.b[2], w2
-; CHECK-NEXT:    add x14, sp, #1008
-; CHECK-NEXT:    add x15, sp, #872
-; CHECK-NEXT:    ld1 { v5.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #240
-; CHECK-NEXT:    add x16, sp, #888
-; CHECK-NEXT:    add x10, sp, #16
-; CHECK-NEXT:    add x9, sp, #24
-; CHECK-NEXT:    add x11, sp, #40
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v5.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #248
-; CHECK-NEXT:    mov v0.b[3], w3
-; CHECK-NEXT:    ld1 { v5.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #256
-; CHECK-NEXT:    mov v0.b[4], w4
-; CHECK-NEXT:    ld1 { v5.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #264
-; CHECK-NEXT:    mov v0.b[5], w5
-; CHECK-NEXT:    ld1 { v5.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #272
-; CHECK-NEXT:    ld1 { v5.b }[8], [x8]
-; CHECK-NEXT:    add x8, sp, #280
-; CHECK-NEXT:    mov v0.b[6], w6
-; CHECK-NEXT:    ld1 { v5.b }[9], [x8]
-; CHECK-NEXT:    add x8, sp, #288
-; CHECK-NEXT:    mov v0.b[7], w7
-; CHECK-NEXT:    ld1 { v5.b }[10], [x8]
-; CHECK-NEXT:    add x8, sp, #296
-; CHECK-NEXT:    ld1 { v0.b }[8], [x10]
-; CHECK-NEXT:    add x10, sp, #128
-; CHECK-NEXT:    ld1 { v5.b }[11], [x8]
-; CHECK-NEXT:    add x8, sp, #304
-; CHECK-NEXT:    ld1 { v0.b }[9], [x9]
-; CHECK-NEXT:    add x9, sp, #136
-; CHECK-NEXT:    ld1 { v5.b }[12], [x8]
-; CHECK-NEXT:    add x8, sp, #312
-; CHECK-NEXT:    ld1 { v5.b }[13], [x8]
-; CHECK-NEXT:    add x8, sp, #320
-; CHECK-NEXT:    ld1 { v5.b }[14], [x8]
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    ld1 { v0.b }[10], [x8]
-; CHECK-NEXT:    add x8, sp, #144
-; CHECK-NEXT:    ld1 { v5.b }[15], [x12]
-; CHECK-NEXT:    add x12, sp, #728
-; CHECK-NEXT:    ld1 { v6.b }[1], [x12]
-; CHECK-NEXT:    add x12, sp, #1000
-; CHECK-NEXT:    ld1 { v0.b }[11], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[3], [x12]
-; CHECK-NEXT:    add x12, sp, #736
-; CHECK-NEXT:    add x11, sp, #920
-; CHECK-NEXT:    sdot v3.4s, v5.16b, v1.16b
-; CHECK-NEXT:    ldr b5, [sp, #848]
-; CHECK-NEXT:    ld1 { v6.b }[2], [x12]
-; CHECK-NEXT:    add x12, sp, #48
-; CHECK-NEXT:    ld1 { v5.b }[1], [x13]
-; CHECK-NEXT:    add x13, sp, #744
-; CHECK-NEXT:    ld1 { v4.b }[4], [x14]
-; CHECK-NEXT:    add x14, sp, #96
-; CHECK-NEXT:    ld1 { v0.b }[12], [x12]
-; CHECK-NEXT:    ld1 { v6.b }[3], [x13]
-; CHECK-NEXT:    add x13, sp, #864
-; CHECK-NEXT:    ld1 { v7.b }[2], [x14]
-; CHECK-NEXT:    add x14, sp, #1016
-; CHECK-NEXT:    ld1 { v5.b }[2], [x13]
-; CHECK-NEXT:    add x13, sp, #752
-; CHECK-NEXT:    ld1 { v4.b }[5], [x14]
-; CHECK-NEXT:    add x14, sp, #104
-; CHECK-NEXT:    ld1 { v6.b }[4], [x13]
-; CHECK-NEXT:    add x13, sp, #1024
-; CHECK-NEXT:    ld1 { v7.b }[3], [x14]
-; CHECK-NEXT:    ld1 { v5.b }[3], [x15]
-; CHECK-NEXT:    add x15, sp, #760
-; CHECK-NEXT:    add x14, sp, #112
-; CHECK-NEXT:    ld1 { v4.b }[6], [x13]
-; CHECK-NEXT:    add x13, sp, #880
-; CHECK-NEXT:    ld1 { v6.b }[5], [x15]
-; CHECK-NEXT:    add x15, sp, #1032
-; CHECK-NEXT:    ld1 { v7.b }[4], [x14]
-; CHECK-NEXT:    ld1 { v5.b }[4], [x13]
-; CHECK-NEXT:    add x14, sp, #768
-; CHECK-NEXT:    add x13, sp, #120
-; CHECK-NEXT:    ld1 { v4.b }[7], [x15]
-; CHECK-NEXT:    add x15, sp, #1040
-; CHECK-NEXT:    ld1 { v6.b }[6], [x14]
-; CHECK-NEXT:    ld1 { v7.b }[5], [x13]
-; CHECK-NEXT:    add x13, sp, #776
-; CHECK-NEXT:    ld1 { v5.b }[5], [x16]
-; CHECK-NEXT:    add x14, sp, #1048
-; CHECK-NEXT:    ld1 { v4.b }[8], [x15]
-; CHECK-NEXT:    add x15, sp, #896
-; CHECK-NEXT:    ld1 { v6.b }[7], [x13]
-; CHECK-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #784
-; CHECK-NEXT:    ld1 { v5.b }[6], [x15]
-; CHECK-NEXT:    add x13, sp, #1056
-; CHECK-NEXT:    ld1 { v4.b }[9], [x14]
-; CHECK-NEXT:    add x14, sp, #904
-; CHECK-NEXT:    ld1 { v6.b }[8], [x10]
-; CHECK-NEXT:    ld1 { v7.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #792
-; CHECK-NEXT:    ld1 { v5.b }[7], [x14]
-; CHECK-NEXT:    add x10, sp, #1064
-; CHECK-NEXT:    ld1 { v4.b }[10], [x13]
-; CHECK-NEXT:    add x13, sp, #912
-; CHECK-NEXT:    ld1 { v6.b }[9], [x9]
-; CHECK-NEXT:    ld1 { v7.b }[8], [x8]
-; CHECK-NEXT:    add x9, sp, #800
-; CHECK-NEXT:    ld1 { v5.b }[8], [x13]
-; CHECK-NEXT:    add x8, sp, #152
-; CHECK-NEXT:    ld1 { v4.b }[11], [x10]
-; CHECK-NEXT:    add x10, sp, #1072
-; CHECK-NEXT:    ld1 { v6.b }[10], [x9]
-; CHECK-NEXT:    ld1 { v7.b }[9], [x8]
-; CHECK-NEXT:    add x9, sp, #808
-; CHECK-NEXT:    ld1 { v5.b }[9], [x11]
-; CHECK-NEXT:    add x8, sp, #56
-; CHECK-NEXT:    ld1 { v4.b }[12], [x10]
-; CHECK-NEXT:    add x10, sp, #160
-; CHECK-NEXT:    ld1 { v0.b }[13], [x8]
-; CHECK-NEXT:    ld1 { v6.b }[11], [x9]
-; CHECK-NEXT:    add x9, sp, #928
-; CHECK-NEXT:    ld1 { v7.b }[10], [x10]
-; CHECK-NEXT:    add x10, sp, #1080
-; CHECK-NEXT:    ld1 { v5.b }[10], [x9]
-; CHECK-NEXT:    add x8, sp, #816
-; CHECK-NEXT:    ld1 { v4.b }[13], [x10]
-; CHECK-NEXT:    add x9, sp, #168
-; CHECK-NEXT:    add x10, sp, #176
-; CHECK-NEXT:    ld1 { v6.b }[12], [x8]
-; CHECK-NEXT:    add x8, sp, #936
-; CHECK-NEXT:    ld1 { v7.b }[11], [x9]
-; CHECK-NEXT:    add x9, sp, #1088
-; CHECK-NEXT:    ld1 { v5.b }[11], [x8]
-; CHECK-NEXT:    add x8, sp, #64
-; CHECK-NEXT:    ld1 { v4.b }[14], [x9]
-; CHECK-NEXT:    add x9, sp, #824
-; CHECK-NEXT:    ld1 { v0.b }[14], [x8]
-; CHECK-NEXT:    ld1 { v6.b }[13], [x9]
-; CHECK-NEXT:    add x9, sp, #944
-; CHECK-NEXT:    ld1 { v7.b }[12], [x10]
-; CHECK-NEXT:    add x10, sp, #1096
-; CHECK-NEXT:    ld1 { v5.b }[12], [x9]
-; CHECK-NEXT:    add x8, sp, #832
-; CHECK-NEXT:    ld1 { v4.b }[15], [x10]
-; CHECK-NEXT:    add x9, sp, #184
-; CHECK-NEXT:    add x10, sp, #72
-; CHECK-NEXT:    ld1 { v6.b }[14], [x8]
-; CHECK-NEXT:    add x8, sp, #952
-; CHECK-NEXT:    ld1 { v7.b }[13], [x9]
-; CHECK-NEXT:    ld1 { v5.b }[13], [x8]
-; CHECK-NEXT:    add x8, sp, #840
-; CHECK-NEXT:    ld1 { v0.b }[15], [x10]
-; CHECK-NEXT:    sdot v2.4s, v4.16b, v1.16b
-; CHECK-NEXT:    add x9, sp, #192
-; CHECK-NEXT:    ld1 { v6.b }[15], [x8]
-; CHECK-NEXT:    add x8, sp, #960
-; CHECK-NEXT:    ld1 { v7.b }[14], [x9]
-; CHECK-NEXT:    ld1 { v5.b }[14], [x8]
-; CHECK-NEXT:    sdot v3.4s, v0.16b, v1.16b
-; CHECK-NEXT:    add x8, sp, #200
-; CHECK-NEXT:    add x9, sp, #968
-; CHECK-NEXT:    sdot v2.4s, v6.16b, v1.16b
-; CHECK-NEXT:    ld1 { v7.b }[15], [x8]
-; CHECK-NEXT:    ld1 { v5.b }[15], [x9]
-; CHECK-NEXT:    sdot v3.4s, v7.16b, v1.16b
-; CHECK-NEXT:    sdot v2.4s, v5.16b, v1.16b
-; CHECK-NEXT:    add v0.4s, v3.4s, v2.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w29, -16
+; CHECK-SD-NEXT:    ldr b5, [sp, #208]
+; CHECK-SD-NEXT:    add x8, sp, #216
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr b4, [sp, #976]
+; CHECK-SD-NEXT:    add x9, sp, #984
+; CHECK-SD-NEXT:    add x12, sp, #328
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #224
+; CHECK-SD-NEXT:    movi v1.16b, #1
+; CHECK-SD-NEXT:    mov v0.b[1], w1
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x9]
+; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT:    add x11, sp, #992
+; CHECK-SD-NEXT:    ldr b6, [sp, #720]
+; CHECK-SD-NEXT:    ldr b7, [sp, #80]
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #232
+; CHECK-SD-NEXT:    add x13, sp, #88
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #856
+; CHECK-SD-NEXT:    mov v0.b[2], w2
+; CHECK-SD-NEXT:    add x14, sp, #1008
+; CHECK-SD-NEXT:    add x15, sp, #872
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #240
+; CHECK-SD-NEXT:    add x16, sp, #888
+; CHECK-SD-NEXT:    add x10, sp, #16
+; CHECK-SD-NEXT:    add x9, sp, #24
+; CHECK-SD-NEXT:    add x11, sp, #40
+; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #248
+; CHECK-SD-NEXT:    mov v0.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #256
+; CHECK-SD-NEXT:    mov v0.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #264
+; CHECK-SD-NEXT:    mov v0.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #272
+; CHECK-SD-NEXT:    ld1 { v5.b }[8], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #280
+; CHECK-SD-NEXT:    mov v0.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v5.b }[9], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #288
+; CHECK-SD-NEXT:    mov v0.b[7], w7
+; CHECK-SD-NEXT:    ld1 { v5.b }[10], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #296
+; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #128
+; CHECK-SD-NEXT:    ld1 { v5.b }[11], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #304
+; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #136
+; CHECK-SD-NEXT:    ld1 { v5.b }[12], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #312
+; CHECK-SD-NEXT:    ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #320
+; CHECK-SD-NEXT:    ld1 { v5.b }[14], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #144
+; CHECK-SD-NEXT:    ld1 { v5.b }[15], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #728
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #1000
+; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x11]
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #736
+; CHECK-SD-NEXT:    add x11, sp, #920
+; CHECK-SD-NEXT:    sdot v3.4s, v5.16b, v1.16b
+; CHECK-SD-NEXT:    ldr b5, [sp, #848]
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #48
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #744
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #96
+; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x12]
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #864
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #1016
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #752
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #104
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #1024
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x14]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #760
+; CHECK-SD-NEXT:    add x14, sp, #112
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #880
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #1032
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x14]
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x13]
+; CHECK-SD-NEXT:    add x14, sp, #768
+; CHECK-SD-NEXT:    add x13, sp, #120
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #1040
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x14]
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #776
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x16]
+; CHECK-SD-NEXT:    add x14, sp, #1048
+; CHECK-SD-NEXT:    ld1 { v4.b }[8], [x15]
+; CHECK-SD-NEXT:    add x15, sp, #896
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x13]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #784
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x15]
+; CHECK-SD-NEXT:    add x13, sp, #1056
+; CHECK-SD-NEXT:    ld1 { v4.b }[9], [x14]
+; CHECK-SD-NEXT:    add x14, sp, #904
+; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x10]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #792
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x14]
+; CHECK-SD-NEXT:    add x10, sp, #1064
+; CHECK-SD-NEXT:    ld1 { v4.b }[10], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #912
+; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x8]
+; CHECK-SD-NEXT:    add x9, sp, #800
+; CHECK-SD-NEXT:    ld1 { v5.b }[8], [x13]
+; CHECK-SD-NEXT:    add x8, sp, #152
+; CHECK-SD-NEXT:    ld1 { v4.b }[11], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #1072
+; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x8]
+; CHECK-SD-NEXT:    add x9, sp, #808
+; CHECK-SD-NEXT:    ld1 { v5.b }[9], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #56
+; CHECK-SD-NEXT:    ld1 { v4.b }[12], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #160
+; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x8]
+; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #928
+; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #1080
+; CHECK-SD-NEXT:    ld1 { v5.b }[10], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #816
+; CHECK-SD-NEXT:    ld1 { v4.b }[13], [x10]
+; CHECK-SD-NEXT:    add x9, sp, #168
+; CHECK-SD-NEXT:    add x10, sp, #176
+; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #936
+; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #1088
+; CHECK-SD-NEXT:    ld1 { v5.b }[11], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #64
+; CHECK-SD-NEXT:    ld1 { v4.b }[14], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #824
+; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x8]
+; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #944
+; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #1096
+; CHECK-SD-NEXT:    ld1 { v5.b }[12], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #832
+; CHECK-SD-NEXT:    ld1 { v4.b }[15], [x10]
+; CHECK-SD-NEXT:    add x9, sp, #184
+; CHECK-SD-NEXT:    add x10, sp, #72
+; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #952
+; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #840
+; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x10]
+; CHECK-SD-NEXT:    sdot v2.4s, v4.16b, v1.16b
+; CHECK-SD-NEXT:    add x9, sp, #192
+; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #960
+; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[14], [x8]
+; CHECK-SD-NEXT:    sdot v3.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    add x8, sp, #200
+; CHECK-SD-NEXT:    add x9, sp, #968
+; CHECK-SD-NEXT:    sdot v2.4s, v6.16b, v1.16b
+; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[15], [x9]
+; CHECK-SD-NEXT:    sdot v3.4s, v7.16b, v1.16b
+; CHECK-SD-NEXT:    sdot v2.4s, v5.16b, v1.16b
+; CHECK-SD-NEXT:    add v0.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    ldr w10, [sp, #80]
+; CHECK-GI-NEXT:    ldr w11, [sp, #208]
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    ldr w9, [sp, #88]
+; CHECK-GI-NEXT:    ldr w12, [sp, #728]
+; CHECK-GI-NEXT:    movi v6.16b, #1
+; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    fmov s2, w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #720]
+; CHECK-GI-NEXT:    ldr w10, [sp, #216]
+; CHECK-GI-NEXT:    mov v0.b[1], w1
+; CHECK-GI-NEXT:    ldr w13, [sp, #856]
+; CHECK-GI-NEXT:    fmov s3, w11
+; CHECK-GI-NEXT:    ldr w8, [sp, #96]
+; CHECK-GI-NEXT:    ldr w11, [sp, #224]
+; CHECK-GI-NEXT:    mov v1.b[1], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #848]
+; CHECK-GI-NEXT:    mov v2.b[1], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #976]
+; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT:    fmov s4, w9
+; CHECK-GI-NEXT:    mov v3.b[1], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #984]
+; CHECK-GI-NEXT:    fmov s5, w10
+; CHECK-GI-NEXT:    mov v0.b[2], w2
+; CHECK-GI-NEXT:    ldr w10, [sp, #736]
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #864]
+; CHECK-GI-NEXT:    mov v2.b[2], w11
+; CHECK-GI-NEXT:    mov v4.b[1], w13
+; CHECK-GI-NEXT:    ldr w11, [sp, #992]
+; CHECK-GI-NEXT:    ldr w12, [sp, #776]
+; CHECK-GI-NEXT:    mov v5.b[1], w9
+; CHECK-GI-NEXT:    mov v3.b[2], w10
+; CHECK-GI-NEXT:    ldr w9, [sp, #104]
+; CHECK-GI-NEXT:    ldr w10, [sp, #232]
+; CHECK-GI-NEXT:    mov v0.b[3], w3
+; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v1.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #872]
+; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v4.b[2], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #744]
+; CHECK-GI-NEXT:    mov v2.b[3], w10
+; CHECK-GI-NEXT:    mov v5.b[2], w11
+; CHECK-GI-NEXT:    ldr w11, [sp, #1000]
+; CHECK-GI-NEXT:    ldr w10, [sp, #240]
+; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #112]
+; CHECK-GI-NEXT:    mov v0.b[4], w4
+; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT:    mov v4.b[3], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #752]
+; CHECK-GI-NEXT:    mov v1.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #880]
+; CHECK-GI-NEXT:    mov v5.b[3], w11
+; CHECK-GI-NEXT:    mov v2.b[4], w10
+; CHECK-GI-NEXT:    mov v3.b[4], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #120]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1008]
+; CHECK-GI-NEXT:    ldr w10, [sp, #248]
+; CHECK-GI-NEXT:    mov v0.b[5], w5
+; CHECK-GI-NEXT:    mov v4.b[4], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #760]
+; CHECK-GI-NEXT:    mov v1.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #888]
+; CHECK-GI-NEXT:    mov v5.b[4], w11
+; CHECK-GI-NEXT:    mov v2.b[5], w10
+; CHECK-GI-NEXT:    mov v3.b[5], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #128]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1016]
+; CHECK-GI-NEXT:    ldr w10, [sp, #256]
+; CHECK-GI-NEXT:    mov v0.b[6], w6
+; CHECK-GI-NEXT:    mov v4.b[5], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #768]
+; CHECK-GI-NEXT:    mov v1.b[6], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #896]
+; CHECK-GI-NEXT:    mov v5.b[5], w11
+; CHECK-GI-NEXT:    mov v2.b[6], w10
+; CHECK-GI-NEXT:    mov v3.b[6], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #136]
+; CHECK-GI-NEXT:    ldr w11, [sp, #1024]
+; CHECK-GI-NEXT:    ldr w10, [sp, #264]
+; CHECK-GI-NEXT:    mov v0.b[7], w7
+; CHECK-GI-NEXT:    mov v4.b[6], w8
+; CHECK-GI-NEXT:    mov v1.b[7], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #904]
+; CHECK-GI-NEXT:    mov v5.b[6], w11
+; CHECK-GI-NEXT:    mov v2.b[7], w10
+; CHECK-GI-NEXT:    ldr w8, [sp, #16]
+; CHECK-GI-NEXT:    mov v3.b[7], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #144]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1032]
+; CHECK-GI-NEXT:    mov v0.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #784]
+; CHECK-GI-NEXT:    ldr w11, [sp, #272]
+; CHECK-GI-NEXT:    mov v4.b[7], w9
+; CHECK-GI-NEXT:    mov v1.b[8], w10
+; CHECK-GI-NEXT:    ldr w10, [sp, #912]
+; CHECK-GI-NEXT:    mov v5.b[7], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #24]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1040]
+; CHECK-GI-NEXT:    mov v3.b[8], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #152]
+; CHECK-GI-NEXT:    mov v2.b[8], w11
+; CHECK-GI-NEXT:    mov v0.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #792]
+; CHECK-GI-NEXT:    ldr w11, [sp, #280]
+; CHECK-GI-NEXT:    mov v4.b[8], w10
+; CHECK-GI-NEXT:    mov v1.b[9], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #920]
+; CHECK-GI-NEXT:    mov v5.b[8], w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #32]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1048]
+; CHECK-GI-NEXT:    mov v3.b[9], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #160]
+; CHECK-GI-NEXT:    mov v2.b[9], w11
+; CHECK-GI-NEXT:    mov v0.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #800]
+; CHECK-GI-NEXT:    ldr w11, [sp, #288]
+; CHECK-GI-NEXT:    mov v4.b[9], w10
+; CHECK-GI-NEXT:    mov v1.b[10], w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #928]
+; CHECK-GI-NEXT:    mov v5.b[9], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #40]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1056]
+; CHECK-GI-NEXT:    mov v3.b[10], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #168]
+; CHECK-GI-NEXT:    mov v2.b[10], w11
+; CHECK-GI-NEXT:    mov v0.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #808]
+; CHECK-GI-NEXT:    ldr w11, [sp, #296]
+; CHECK-GI-NEXT:    mov v4.b[10], w10
+; CHECK-GI-NEXT:    mov v1.b[11], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #936]
+; CHECK-GI-NEXT:    mov v5.b[10], w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #48]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1064]
+; CHECK-GI-NEXT:    mov v3.b[11], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #176]
+; CHECK-GI-NEXT:    mov v2.b[11], w11
+; CHECK-GI-NEXT:    mov v0.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #816]
+; CHECK-GI-NEXT:    ldr w11, [sp, #304]
+; CHECK-GI-NEXT:    mov v4.b[11], w10
+; CHECK-GI-NEXT:    mov v1.b[12], w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #944]
+; CHECK-GI-NEXT:    mov v5.b[11], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #56]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1072]
+; CHECK-GI-NEXT:    mov v3.b[12], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #184]
+; CHECK-GI-NEXT:    mov v2.b[12], w11
+; CHECK-GI-NEXT:    mov v0.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #824]
+; CHECK-GI-NEXT:    ldr w11, [sp, #312]
+; CHECK-GI-NEXT:    mov v4.b[12], w10
+; CHECK-GI-NEXT:    mov v1.b[13], w8
+; CHECK-GI-NEXT:    ldr w10, [sp, #952]
+; CHECK-GI-NEXT:    mov v5.b[12], w12
+; CHECK-GI-NEXT:    ldr w8, [sp, #64]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1080]
+; CHECK-GI-NEXT:    mov v3.b[13], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #192]
+; CHECK-GI-NEXT:    mov v2.b[13], w11
+; CHECK-GI-NEXT:    mov v0.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #832]
+; CHECK-GI-NEXT:    ldr w11, [sp, #320]
+; CHECK-GI-NEXT:    mov v4.b[13], w10
+; CHECK-GI-NEXT:    mov v1.b[14], w9
+; CHECK-GI-NEXT:    ldr w10, [sp, #960]
+; CHECK-GI-NEXT:    mov v5.b[13], w12
+; CHECK-GI-NEXT:    ldr w9, [sp, #72]
+; CHECK-GI-NEXT:    ldr w12, [sp, #1088]
+; CHECK-GI-NEXT:    mov v3.b[14], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #200]
+; CHECK-GI-NEXT:    mov v2.b[14], w11
+; CHECK-GI-NEXT:    mov v0.b[15], w9
+; CHECK-GI-NEXT:    ldr w9, [sp, #840]
+; CHECK-GI-NEXT:    ldr w11, [sp, #328]
+; CHECK-GI-NEXT:    mov v4.b[14], w10
+; CHECK-GI-NEXT:    mov v1.b[15], w8
+; CHECK-GI-NEXT:    ldr w8, [sp, #968]
+; CHECK-GI-NEXT:    mov v5.b[14], w12
+; CHECK-GI-NEXT:    ldr w10, [sp, #1096]
+; CHECK-GI-NEXT:    mov v3.b[15], w9
+; CHECK-GI-NEXT:    mov v2.b[15], w11
+; CHECK-GI-NEXT:    sdot v7.4s, v0.16b, v6.16b
+; CHECK-GI-NEXT:    mov v4.b[15], w8
+; CHECK-GI-NEXT:    sdot v16.4s, v1.16b, v6.16b
+; CHECK-GI-NEXT:    mov v5.b[15], w10
+; CHECK-GI-NEXT:    sdot v17.4s, v3.16b, v6.16b
+; CHECK-GI-NEXT:    sdot v20.4s, v2.16b, v6.16b
+; CHECK-GI-NEXT:    addv s0, v7.4s
+; CHECK-GI-NEXT:    sdot v18.4s, v4.16b, v6.16b
+; CHECK-GI-NEXT:    addv s1, v16.4s
+; CHECK-GI-NEXT:    sdot v19.4s, v5.16b, v6.16b
+; CHECK-GI-NEXT:    addv s2, v17.4s
+; CHECK-GI-NEXT:    addv s4, v20.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    addv s3, v18.4s
+; CHECK-GI-NEXT:    addv s5, v19.4s
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    add w8, w8, w9
+; CHECK-GI-NEXT:    add w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s5
+; CHECK-GI-NEXT:    add w9, w10, w11
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <48 x i8> %a to <48 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)
@@ -2795,23 +4245,45 @@ entry:
 }
 
 define i32 @test_udot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v64i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ldp q2, q3, [x0, #32]
-; CHECK-NEXT:    ldp q4, q5, [x1, #32]
-; CHECK-NEXT:    udot v1.4s, v5.16b, v3.16b
-; CHECK-NEXT:    udot v0.4s, v4.16b, v2.16b
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    ldp q4, q5, [x1]
-; CHECK-NEXT:    udot v1.4s, v5.16b, v3.16b
-; CHECK-NEXT:    udot v0.4s, v4.16b, v2.16b
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v64i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q2, q3, [x0, #32]
+; CHECK-SD-NEXT:    ldp q4, q5, [x1, #32]
+; CHECK-SD-NEXT:    udot v1.4s, v5.16b, v3.16b
+; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v2.16b
+; CHECK-SD-NEXT:    ldp q2, q3, [x0]
+; CHECK-SD-NEXT:    ldp q4, q5, [x1]
+; CHECK-SD-NEXT:    udot v1.4s, v5.16b, v3.16b
+; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v2.16b
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v64i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q1, q2, [x0]
+; CHECK-GI-NEXT:    ldp q6, q7, [x0, #32]
+; CHECK-GI-NEXT:    ldp q16, q17, [x1]
+; CHECK-GI-NEXT:    ldp q18, q19, [x1, #32]
+; CHECK-GI-NEXT:    udot v0.4s, v16.16b, v1.16b
+; CHECK-GI-NEXT:    udot v4.4s, v17.16b, v2.16b
+; CHECK-GI-NEXT:    udot v5.4s, v18.16b, v6.16b
+; CHECK-GI-NEXT:    udot v3.4s, v19.16b, v7.16b
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    add v1.4s, v5.4s, v3.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <64 x i8>, ptr %a
   %1 = zext <64 x i8> %0 to <64 x i32>
@@ -2824,21 +4296,41 @@ entry:
 }
 
 define i32 @test_udot_v64i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v64i8_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.16b, #1
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-NEXT:    ldp q3, q4, [x0, #32]
-; CHECK-NEXT:    udot v2.4s, v4.16b, v0.16b
-; CHECK-NEXT:    udot v1.4s, v3.16b, v0.16b
-; CHECK-NEXT:    ldp q3, q4, [x0]
-; CHECK-NEXT:    udot v2.4s, v4.16b, v0.16b
-; CHECK-NEXT:    udot v1.4s, v3.16b, v0.16b
-; CHECK-NEXT:    add v0.4s, v1.4s, v2.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_udot_v64i8_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.16b, #1
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q3, q4, [x0, #32]
+; CHECK-SD-NEXT:    udot v2.4s, v4.16b, v0.16b
+; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
+; CHECK-SD-NEXT:    ldp q3, q4, [x0]
+; CHECK-SD-NEXT:    udot v2.4s, v4.16b, v0.16b
+; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b
+; CHECK-SD-NEXT:    add v0.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_udot_v64i8_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.16b, #1
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q5, q6, [x0]
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q7, q16, [x0, #32]
+; CHECK-GI-NEXT:    udot v1.4s, v5.16b, v0.16b
+; CHECK-GI-NEXT:    udot v3.4s, v6.16b, v0.16b
+; CHECK-GI-NEXT:    udot v2.4s, v16.16b, v0.16b
+; CHECK-GI-NEXT:    udot v4.4s, v7.16b, v0.16b
+; CHECK-GI-NEXT:    add v0.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    add v1.4s, v4.4s, v2.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <64 x i8>, ptr %a1
   %1 = zext <64 x i8> %0 to <64 x i32>
@@ -2846,23 +4338,45 @@ entry:
   ret i32 %2
 }
 define i32 @test_sdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v64i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ldp q2, q3, [x0, #32]
-; CHECK-NEXT:    ldp q4, q5, [x1, #32]
-; CHECK-NEXT:    sdot v1.4s, v5.16b, v3.16b
-; CHECK-NEXT:    sdot v0.4s, v4.16b, v2.16b
-; CHECK-NEXT:    ldp q2, q3, [x0]
-; CHECK-NEXT:    ldp q4, q5, [x1]
-; CHECK-NEXT:    sdot v1.4s, v5.16b, v3.16b
-; CHECK-NEXT:    sdot v0.4s, v4.16b, v2.16b
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    add w0, w8, w2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v64i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q2, q3, [x0, #32]
+; CHECK-SD-NEXT:    ldp q4, q5, [x1, #32]
+; CHECK-SD-NEXT:    sdot v1.4s, v5.16b, v3.16b
+; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v2.16b
+; CHECK-SD-NEXT:    ldp q2, q3, [x0]
+; CHECK-SD-NEXT:    ldp q4, q5, [x1]
+; CHECK-SD-NEXT:    sdot v1.4s, v5.16b, v3.16b
+; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v2.16b
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w8, s0
+; CHECK-SD-NEXT:    add w0, w8, w2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v64i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q1, q2, [x0]
+; CHECK-GI-NEXT:    ldp q6, q7, [x0, #32]
+; CHECK-GI-NEXT:    ldp q16, q17, [x1]
+; CHECK-GI-NEXT:    ldp q18, q19, [x1, #32]
+; CHECK-GI-NEXT:    sdot v0.4s, v16.16b, v1.16b
+; CHECK-GI-NEXT:    sdot v4.4s, v17.16b, v2.16b
+; CHECK-GI-NEXT:    sdot v5.4s, v18.16b, v6.16b
+; CHECK-GI-NEXT:    sdot v3.4s, v19.16b, v7.16b
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT:    add v1.4s, v5.4s, v3.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    add w0, w8, w2
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load <64 x i8>, ptr %a
   %1 = sext <64 x i8> %0 to <64 x i32>
@@ -2875,30 +4389,65 @@ entry:
 }
 
 define i32 @test_sdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
-; CHECK-LABEL: test_sdot_v64i8_double:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-NEXT:    movi v18.2d, #0000000000000000
-; CHECK-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-NEXT:    ldp q20, q21, [sp, #96]
-; CHECK-NEXT:    ldp q22, q23, [sp, #32]
-; CHECK-NEXT:    sdot v16.4s, v3.16b, v7.16b
-; CHECK-NEXT:    sdot v18.4s, v2.16b, v6.16b
-; CHECK-NEXT:    sdot v19.4s, v23.16b, v21.16b
-; CHECK-NEXT:    sdot v17.4s, v22.16b, v20.16b
-; CHECK-NEXT:    ldp q2, q3, [sp, #64]
-; CHECK-NEXT:    ldp q6, q7, [sp]
-; CHECK-NEXT:    sdot v16.4s, v1.16b, v5.16b
-; CHECK-NEXT:    sdot v18.4s, v0.16b, v4.16b
-; CHECK-NEXT:    sdot v19.4s, v7.16b, v3.16b
-; CHECK-NEXT:    sdot v17.4s, v6.16b, v2.16b
-; CHECK-NEXT:    add v0.4s, v18.4s, v16.4s
-; CHECK-NEXT:    add v1.4s, v17.4s, v19.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v64i8_double:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q20, q21, [sp, #96]
+; CHECK-SD-NEXT:    ldp q22, q23, [sp, #32]
+; CHECK-SD-NEXT:    sdot v16.4s, v3.16b, v7.16b
+; CHECK-SD-NEXT:    sdot v18.4s, v2.16b, v6.16b
+; CHECK-SD-NEXT:    sdot v19.4s, v23.16b, v21.16b
+; CHECK-SD-NEXT:    sdot v17.4s, v22.16b, v20.16b
+; CHECK-SD-NEXT:    ldp q2, q3, [sp, #64]
+; CHECK-SD-NEXT:    ldp q6, q7, [sp]
+; CHECK-SD-NEXT:    sdot v16.4s, v1.16b, v5.16b
+; CHECK-SD-NEXT:    sdot v18.4s, v0.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v19.4s, v7.16b, v3.16b
+; CHECK-SD-NEXT:    sdot v17.4s, v6.16b, v2.16b
+; CHECK-SD-NEXT:    add v0.4s, v18.4s, v16.4s
+; CHECK-SD-NEXT:    add v1.4s, v17.4s, v19.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v64i8_double:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v21.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v23.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-GI-NEXT:    movi v24.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v25.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v26.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v27.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q19, q20, [sp, #32]
+; CHECK-GI-NEXT:    sdot v18.4s, v0.16b, v4.16b
+; CHECK-GI-NEXT:    ldp q0, q4, [sp, #64]
+; CHECK-GI-NEXT:    sdot v21.4s, v1.16b, v5.16b
+; CHECK-GI-NEXT:    ldp q1, q5, [sp, #96]
+; CHECK-GI-NEXT:    sdot v22.4s, v2.16b, v6.16b
+; CHECK-GI-NEXT:    sdot v23.4s, v3.16b, v7.16b
+; CHECK-GI-NEXT:    sdot v24.4s, v16.16b, v0.16b
+; CHECK-GI-NEXT:    sdot v26.4s, v17.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v27.4s, v19.16b, v1.16b
+; CHECK-GI-NEXT:    sdot v25.4s, v20.16b, v5.16b
+; CHECK-GI-NEXT:    add v0.4s, v18.4s, v21.4s
+; CHECK-GI-NEXT:    add v1.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT:    add v2.4s, v24.4s, v26.4s
+; CHECK-GI-NEXT:    add v3.4s, v27.4s, v25.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <64 x i8> %a to <64 x i32>
   %bz = sext <64 x i8> %b to <64 x i32>
@@ -2913,29 +4462,63 @@ entry:
 }
 
 define i32 @test_sdot_v64i8_double_nomla(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
-; CHECK-LABEL: test_sdot_v64i8_double_nomla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v4.16b, #1
-; CHECK-NEXT:    movi v5.2d, #0000000000000000
-; CHECK-NEXT:    movi v6.2d, #0000000000000000
-; CHECK-NEXT:    movi v7.2d, #0000000000000000
-; CHECK-NEXT:    ldp q17, q18, [sp, #32]
-; CHECK-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-NEXT:    sdot v5.4s, v3.16b, v4.16b
-; CHECK-NEXT:    sdot v6.4s, v17.16b, v4.16b
-; CHECK-NEXT:    sdot v7.4s, v2.16b, v4.16b
-; CHECK-NEXT:    ldp q2, q3, [sp]
-; CHECK-NEXT:    sdot v16.4s, v18.16b, v4.16b
-; CHECK-NEXT:    sdot v5.4s, v1.16b, v4.16b
-; CHECK-NEXT:    sdot v6.4s, v2.16b, v4.16b
-; CHECK-NEXT:    sdot v7.4s, v0.16b, v4.16b
-; CHECK-NEXT:    sdot v16.4s, v3.16b, v4.16b
-; CHECK-NEXT:    add v0.4s, v7.4s, v5.4s
-; CHECK-NEXT:    add v1.4s, v6.4s, v16.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_sdot_v64i8_double_nomla:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v4.16b, #1
+; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-SD-NEXT:    ldp q17, q18, [sp, #32]
+; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT:    sdot v5.4s, v3.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v6.4s, v17.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v7.4s, v2.16b, v4.16b
+; CHECK-SD-NEXT:    ldp q2, q3, [sp]
+; CHECK-SD-NEXT:    sdot v16.4s, v18.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v6.4s, v2.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v7.4s, v0.16b, v4.16b
+; CHECK-SD-NEXT:    sdot v16.4s, v3.16b, v4.16b
+; CHECK-SD-NEXT:    add v0.4s, v7.4s, v5.4s
+; CHECK-SD-NEXT:    add v1.4s, v6.4s, v16.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    addv s0, v0.4s
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_sdot_v64i8_double_nomla:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v4.16b, #1
+; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-GI-NEXT:    ldp q21, q22, [sp]
+; CHECK-GI-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT:    sdot v5.4s, v0.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v6.4s, v1.16b, v4.16b
+; CHECK-GI-NEXT:    ldp q0, q1, [sp, #32]
+; CHECK-GI-NEXT:    sdot v7.4s, v2.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v16.4s, v3.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v17.4s, v21.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v19.4s, v22.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v20.4s, v0.16b, v4.16b
+; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v4.16b
+; CHECK-GI-NEXT:    add v0.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT:    add v1.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    add v2.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT:    add v3.4s, v20.4s, v18.4s
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    addv s0, v0.4s
+; CHECK-GI-NEXT:    addv s1, v1.4s
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    add w0, w8, w9
+; CHECK-GI-NEXT:    ret
 entry:
   %az = sext <64 x i8> %a to <64 x i32>
   %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %az)
diff --git a/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll b/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll
index b3692f8f46fbc92c92427d29211d5af4bdc734c7..93a50ec305e1e5b6d6ca24f4ab09f6961e43a1f7 100644
--- a/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll
+++ b/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll
@@ -1,12 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64 -global-isel -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define <4 x i16> @addls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: addls_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addls_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    saddl v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addls_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <8 x i8> %s0 to <8 x i16>
   %s1s = sext <8 x i8> %s1 to <8 x i16>
@@ -17,12 +25,18 @@ entry:
 }
 
 define <4 x i16> @addws_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: addws_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    saddw v0.8h, v0.8h, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addws_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    saddw v0.8h, v0.8h, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addws_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = sext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -31,11 +45,18 @@ entry:
 }
 
 define <4 x i16> @addlu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: addlu_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addlu_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addlu_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <8 x i8> %s0 to <8 x i16>
   %s1s = zext <8 x i8> %s1 to <8 x i16>
@@ -46,12 +67,18 @@ entry:
 }
 
 define <4 x i16> @addwu_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: addwu_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    uaddw v0.8h, v0.8h, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addwu_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    uaddw v0.8h, v0.8h, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addwu_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = zext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -60,11 +87,18 @@ entry:
 }
 
 define <4 x i16> @subls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: subls_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ssubl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subls_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ssubl v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subls_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <8 x i8> %s0 to <8 x i16>
   %s1s = sext <8 x i8> %s1 to <8 x i16>
@@ -75,12 +109,18 @@ entry:
 }
 
 define <4 x i16> @subws_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: subws_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ssubw v0.8h, v0.8h, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subws_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    ssubw v0.8h, v0.8h, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subws_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = sext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -89,11 +129,18 @@ entry:
 }
 
 define <4 x i16> @sublu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: sublu_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    usubl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sublu_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    usubl v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sublu_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <8 x i8> %s0 to <8 x i16>
   %s1s = zext <8 x i8> %s1 to <8 x i16>
@@ -104,12 +151,18 @@ entry:
 }
 
 define <4 x i16> @subwu_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: subwu_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    usubw v0.8h, v0.8h, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subwu_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    usubw v0.8h, v0.8h, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subwu_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = zext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -118,11 +171,18 @@ entry:
 }
 
 define <4 x i16> @mulls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: mulls_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mulls_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulls_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <8 x i8> %s0 to <8 x i16>
   %s1s = sext <8 x i8> %s1 to <8 x i16>
@@ -146,11 +206,18 @@ entry:
 }
 
 define <4 x i16> @mullu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
-; CHECK-LABEL: mullu_v8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mullu_v8i8_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mullu_v8i8_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <8 x i8> %s0 to <8 x i16>
   %s1s = zext <8 x i8> %s1 to <8 x i16>
@@ -174,11 +241,18 @@ entry:
 }
 
 define <2 x i32> @addls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: addls_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addls_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    saddl v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addls_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <4 x i16> %s0 to <4 x i32>
   %s1s = sext <4 x i16> %s1 to <4 x i32>
@@ -189,12 +263,18 @@ entry:
 }
 
 define <2 x i32> @addws_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: addws_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    saddw v0.4s, v0.4s, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addws_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    saddw v0.4s, v0.4s, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addws_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = sext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
@@ -203,11 +283,18 @@ entry:
 }
 
 define <2 x i32> @addlu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: addlu_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addlu_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    uaddl v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addlu_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <4 x i16> %s0 to <4 x i32>
   %s1s = zext <4 x i16> %s1 to <4 x i32>
@@ -218,12 +305,18 @@ entry:
 }
 
 define <2 x i32> @addwu_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: addwu_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    uaddw v0.4s, v0.4s, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addwu_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addwu_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = zext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
@@ -232,11 +325,18 @@ entry:
 }
 
 define <2 x i32> @subls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: subls_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ssubl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subls_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ssubl v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subls_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <4 x i16> %s0 to <4 x i32>
   %s1s = sext <4 x i16> %s1 to <4 x i32>
@@ -247,12 +347,18 @@ entry:
 }
 
 define <2 x i32> @subws_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: subws_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ssubw v0.4s, v0.4s, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subws_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    ssubw v0.4s, v0.4s, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subws_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = sext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
@@ -261,11 +367,18 @@ entry:
 }
 
 define <2 x i32> @sublu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: sublu_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    usubl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sublu_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    usubl v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sublu_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <4 x i16> %s0 to <4 x i32>
   %s1s = zext <4 x i16> %s1 to <4 x i32>
@@ -276,12 +389,18 @@ entry:
 }
 
 define <2 x i32> @subwu_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: subwu_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    usubw v0.4s, v0.4s, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subwu_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    usubw v0.4s, v0.4s, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subwu_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = zext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
@@ -290,11 +409,18 @@ entry:
 }
 
 define <2 x i32> @mulls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: mulls_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mulls_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulls_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <4 x i16> %s0 to <4 x i32>
   %s1s = sext <4 x i16> %s1 to <4 x i32>
@@ -318,11 +444,18 @@ entry:
 }
 
 define <2 x i32> @mullu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
-; CHECK-LABEL: mullu_v4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mullu_v4i16_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mullu_v4i16_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <4 x i16> %s0 to <4 x i32>
   %s1s = zext <4 x i16> %s1 to <4 x i32>
@@ -346,11 +479,21 @@ entry:
 }
 
 define <1 x i64> @addls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: addls_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addls_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    saddl v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addls_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <2 x i32> %s0 to <2 x i64>
   %s1s = sext <2 x i32> %s1 to <2 x i64>
@@ -361,12 +504,21 @@ entry:
 }
 
 define <1 x i64> @addws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: addws_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    saddw v0.2d, v0.2d, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addws_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    saddw v0.2d, v0.2d, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addws_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = sext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
@@ -375,11 +527,21 @@ entry:
 }
 
 define <1 x i64> @addlu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: addlu_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addlu_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    uaddl v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addlu_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <2 x i32> %s0 to <2 x i64>
   %s1s = zext <2 x i32> %s1 to <2 x i64>
@@ -390,12 +552,21 @@ entry:
 }
 
 define <1 x i64> @addwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: addwu_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    uaddw v0.2d, v0.2d, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: addwu_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    uaddw v0.2d, v0.2d, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: addwu_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = zext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
@@ -404,11 +575,21 @@ entry:
 }
 
 define <1 x i64> @subls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: subls_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ssubl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subls_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ssubl v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subls_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <2 x i32> %s0 to <2 x i64>
   %s1s = sext <2 x i32> %s1 to <2 x i64>
@@ -419,12 +600,21 @@ entry:
 }
 
 define <1 x i64> @subws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: subws_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ssubw v0.2d, v0.2d, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subws_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    ssubw v0.2d, v0.2d, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subws_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = sext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
@@ -433,11 +623,21 @@ entry:
 }
 
 define <1 x i64> @sublu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: sublu_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    usubl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sublu_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    usubl v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sublu_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <2 x i32> %s0 to <2 x i64>
   %s1s = zext <2 x i32> %s1 to <2 x i64>
@@ -448,12 +648,21 @@ entry:
 }
 
 define <1 x i64> @subwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: subwu_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    usubw v0.2d, v0.2d, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: subwu_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    usubw v0.2d, v0.2d, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: subwu_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    sub x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = zext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
@@ -462,11 +671,21 @@ entry:
 }
 
 define <1 x i64> @mulls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: mulls_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mulls_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulls_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = sext <2 x i32> %s0 to <2 x i64>
   %s1s = sext <2 x i32> %s1 to <2 x i64>
@@ -477,15 +696,24 @@ entry:
 }
 
 define <1 x i64> @mulws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: mulws_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    fmov x9, d1
-; CHECK-NEXT:    mul x8, x8, x9
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mulws_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    fmov x8, d0
+; CHECK-SD-NEXT:    fmov x9, d1
+; CHECK-SD-NEXT:    mul x8, x8, x9
+; CHECK-SD-NEXT:    fmov d0, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulws_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = sext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
@@ -494,11 +722,21 @@ entry:
 }
 
 define <1 x i64> @mullu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: mullu_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    umull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mullu_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mullu_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s0s = zext <2 x i32> %s0 to <2 x i64>
   %s1s = zext <2 x i32> %s1 to <2 x i64>
@@ -509,15 +747,24 @@ entry:
 }
 
 define <1 x i64> @mulwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
-; CHECK-LABEL: mulwu_v2i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    fmov x9, d1
-; CHECK-NEXT:    mul x8, x8, x9
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mulwu_v2i32_0:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    fmov x8, d0
+; CHECK-SD-NEXT:    fmov x9, d1
+; CHECK-SD-NEXT:    mul x8, x8, x9
+; CHECK-SD-NEXT:    fmov d0, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulwu_v2i32_0:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    ret
 entry:
   %s1s = zext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
index f7e95008b7123722c7521ba3806f238bc99d858f..7b173bda56155338eefc8c6e9076390da938710c 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-armpl.ll
@@ -15,7 +15,7 @@ declare <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float>)
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [60 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vacosq_f64, ptr @armpl_vacosq_f32, ptr @armpl_svacos_f64_x, ptr @armpl_svacos_f32_x, ptr @armpl_vasinq_f64, ptr @armpl_vasinq_f32, ptr @armpl_svasin_f64_x, ptr @armpl_svasin_f32_x, ptr @armpl_vatanq_f64, ptr @armpl_vatanq_f32, ptr @armpl_svatan_f64_x, ptr @armpl_svatan_f32_x, ptr @armpl_vcoshq_f64, ptr @armpl_vcoshq_f32, ptr @armpl_svcosh_f64_x, ptr @armpl_svcosh_f32_x, ptr @armpl_vsinhq_f64, ptr @armpl_vsinhq_f32, ptr @armpl_svsinh_f64_x, ptr @armpl_svsinh_f32_x, ptr @armpl_vtanhq_f64, ptr @armpl_vtanhq_f32, ptr @armpl_svtanh_f64_x, ptr @armpl_svtanh_f32_x], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [64 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vpowq_f64, ptr @armpl_vpowq_f32, ptr @armpl_svpow_f64_x, ptr @armpl_svpow_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vtanq_f64, ptr @armpl_vtanq_f32, ptr @armpl_svtan_f64_x, ptr @armpl_svtan_f32_x, ptr @armpl_vacosq_f64, ptr @armpl_vacosq_f32, ptr @armpl_svacos_f64_x, ptr @armpl_svacos_f32_x, ptr @armpl_vasinq_f64, ptr @armpl_vasinq_f32, ptr @armpl_svasin_f64_x, ptr @armpl_svasin_f32_x, ptr @armpl_vatanq_f64, ptr @armpl_vatanq_f32, ptr @armpl_svatan_f64_x, ptr @armpl_svatan_f32_x, ptr @armpl_vcoshq_f64, ptr @armpl_vcoshq_f32, ptr @armpl_svcosh_f64_x, ptr @armpl_svcosh_f32_x, ptr @armpl_vsinhq_f64, ptr @armpl_vsinhq_f32, ptr @armpl_svsinh_f64_x, ptr @armpl_svsinh_f32_x, ptr @armpl_vtanhq_f64, ptr @armpl_vtanhq_f32, ptr @armpl_svtanh_f64_x, ptr @armpl_svtanh_f32_x], section "llvm.metadata"
 
 ;.
 define <2 x double> @llvm_cos_f64(<2 x double> %in) {
@@ -333,17 +333,10 @@ declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>)
 declare <vscale x 2 x double> @llvm.pow.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
 declare <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 
-;
-; There is a bug in the replace-with-veclib pass, and for intrinsics which take
-; more than one arguments, but has just one overloaded type, it incorrectly
-; reconstructs the scalar name, for pow specifically it is searching for:
-; llvm.pow.f64.f64 and llvm.pow.f32.f32
-;
-
 define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %power) {
 ; CHECK-LABEL: define <2 x double> @llvm_pow_f64
 ; CHECK-SAME: (<2 x double> [[IN:%.*]], <2 x double> [[POWER:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @llvm.pow.v2f64(<2 x double> [[IN]], <2 x double> [[POWER]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @armpl_vpowq_f64(<2 x double> [[IN]], <2 x double> [[POWER]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %in, <2 x double> %power)
@@ -353,7 +346,7 @@ define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %power) {
 define <4 x float> @llvm_pow_f32(<4 x float> %in, <4 x float> %power) {
 ; CHECK-LABEL: define <4 x float> @llvm_pow_f32
 ; CHECK-SAME: (<4 x float> [[IN:%.*]], <4 x float> [[POWER:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @llvm.pow.v4f32(<4 x float> [[IN]], <4 x float> [[POWER]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @armpl_vpowq_f32(<4 x float> [[IN]], <4 x float> [[POWER]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %in, <4 x float> %power)
@@ -363,7 +356,7 @@ define <4 x float> @llvm_pow_f32(<4 x float> %in, <4 x float> %power) {
 define <vscale x 2 x double> @llvm_pow_vscale_f64(<vscale x 2 x double> %in, <vscale x 2 x double> %power) #0 {
 ; CHECK-LABEL: define <vscale x 2 x double> @llvm_pow_vscale_f64
 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x double> [[POWER:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.pow.nxv2f64(<vscale x 2 x double> [[IN]], <vscale x 2 x double> [[POWER]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x double> [[POWER]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 ;
   %1 = call fast <vscale x 2 x double> @llvm.pow.nxv2f64(<vscale x 2 x double> %in, <vscale x 2 x double> %power)
@@ -373,7 +366,7 @@ define <vscale x 2 x double> @llvm_pow_vscale_f64(<vscale x 2 x double> %in, <vs
 define <vscale x 4 x float> @llvm_pow_vscale_f32(<vscale x 4 x float> %in, <vscale x 4 x float> %power) #0 {
 ; CHECK-LABEL: define <vscale x 4 x float> @llvm_pow_vscale_f32
 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x float> [[POWER:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> [[IN]], <vscale x 4 x float> [[POWER]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x float> [[POWER]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 ;
   %1 = call fast <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %in, <vscale x 4 x float> %power)
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
index f3f27344ad80e3d694d4a4fc95a724961161466c..155587026b464b2dffd64141e29d8a33df0fbfbf 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef-scalable.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [30 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxv_acos, ptr @_ZGVsMxv_acosf, ptr @_ZGVsMxv_asin, ptr @_ZGVsMxv_asinf, ptr @_ZGVsMxv_atan, ptr @_ZGVsMxv_atanf, ptr @_ZGVsMxv_cosh, ptr @_ZGVsMxv_coshf, ptr @_ZGVsMxv_sinh, ptr @_ZGVsMxv_sinhf, ptr @_ZGVsMxv_tanh, ptr @_ZGVsMxv_tanhf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [32 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxvv_pow, ptr @_ZGVsMxvv_powf, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf, ptr @_ZGVsMxv_tan, ptr @_ZGVsMxv_tanf, ptr @_ZGVsMxv_acos, ptr @_ZGVsMxv_acosf, ptr @_ZGVsMxv_asin, ptr @_ZGVsMxv_asinf, ptr @_ZGVsMxv_atan, ptr @_ZGVsMxv_atanf, ptr @_ZGVsMxv_cosh, ptr @_ZGVsMxv_coshf, ptr @_ZGVsMxv_sinh, ptr @_ZGVsMxv_sinhf, ptr @_ZGVsMxv_tanh, ptr @_ZGVsMxv_tanhf], section "llvm.metadata"
 ;.
 define <vscale x 2 x double> @llvm_ceil_vscale_f64(<vscale x 2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_vscale_f64(
@@ -278,7 +278,7 @@ define <vscale x 4 x float> @llvm_nearbyint_vscale_f32(<vscale x 4 x float> %in)
 
 define <vscale x 2 x double> @llvm_pow_vscale_f64(<vscale x 2 x double> %in, <vscale x 2 x double> %pow) {
 ; CHECK-LABEL: @llvm_pow_vscale_f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.pow.nxv2f64(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x double> [[POW:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x double> [[POW:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 ;
   %1 = call fast <vscale x 2 x double> @llvm.pow.nxv2f64(<vscale x 2 x double> %in, <vscale x 2 x double> %pow)
@@ -287,7 +287,7 @@ define <vscale x 2 x double> @llvm_pow_vscale_f64(<vscale x 2 x double> %in, <vs
 
 define <vscale x 4 x float> @llvm_pow_vscale_f32(<vscale x 4 x float> %in, <vscale x 4 x float> %pow) {
 ; CHECK-LABEL: @llvm_pow_vscale_f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x float> [[POW:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x float> [[POW:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 ;
   %1 = call fast <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %in, <vscale x 4 x float> %pow)
diff --git a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
index 59c2f94e1676333c245617ce009b319420daf3ba..b89bf3d6f2ca97a9f7e66247445adc5212343cf6 100644
--- a/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
+++ b/llvm/test/CodeGen/AArch64/replace-with-veclib-sleef.ll
@@ -4,7 +4,7 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ;.
-; CHECK: @llvm.compiler.used = appending global [30 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2v_acos, ptr @_ZGVnN4v_acosf, ptr @_ZGVnN2v_asin, ptr @_ZGVnN4v_asinf, ptr @_ZGVnN2v_atan, ptr @_ZGVnN4v_atanf, ptr @_ZGVnN2v_cosh, ptr @_ZGVnN4v_coshf, ptr @_ZGVnN2v_sinh, ptr @_ZGVnN4v_sinhf, ptr @_ZGVnN2v_tanh, ptr @_ZGVnN4v_tanhf], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [32 x ptr] [ptr @_ZGVnN2v_cos, ptr @_ZGVnN4v_cosf, ptr @_ZGVnN2v_exp, ptr @_ZGVnN4v_expf, ptr @_ZGVnN2v_exp10, ptr @_ZGVnN4v_exp10f, ptr @_ZGVnN2v_exp2, ptr @_ZGVnN4v_exp2f, ptr @_ZGVnN2v_log, ptr @_ZGVnN4v_logf, ptr @_ZGVnN2v_log10, ptr @_ZGVnN4v_log10f, ptr @_ZGVnN2v_log2, ptr @_ZGVnN4v_log2f, ptr @_ZGVnN2vv_pow, ptr @_ZGVnN4vv_powf, ptr @_ZGVnN2v_sin, ptr @_ZGVnN4v_sinf, ptr @_ZGVnN2v_tan, ptr @_ZGVnN4v_tanf, ptr @_ZGVnN2v_acos, ptr @_ZGVnN4v_acosf, ptr @_ZGVnN2v_asin, ptr @_ZGVnN4v_asinf, ptr @_ZGVnN2v_atan, ptr @_ZGVnN4v_atanf, ptr @_ZGVnN2v_cosh, ptr @_ZGVnN4v_coshf, ptr @_ZGVnN2v_sinh, ptr @_ZGVnN4v_sinhf, ptr @_ZGVnN2v_tanh, ptr @_ZGVnN4v_tanhf], section "llvm.metadata"
 ;.
 define <2 x double> @llvm_ceil_f64(<2 x double> %in) {
 ; CHECK-LABEL: @llvm_ceil_f64(
@@ -278,7 +278,7 @@ define <4 x float> @llvm_nearbyint_f32(<4 x float> %in) {
 
 define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %pow) {
 ; CHECK-LABEL: @llvm_pow_f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @llvm.pow.v2f64(<2 x double> [[IN:%.*]], <2 x double> [[POW:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <2 x double> @_ZGVnN2vv_pow(<2 x double> [[IN:%.*]], <2 x double> [[POW:%.*]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %in, <2 x double> %pow)
@@ -287,7 +287,7 @@ define <2 x double> @llvm_pow_f64(<2 x double> %in, <2 x double> %pow) {
 
 define <4 x float> @llvm_pow_f32(<4 x float> %in, <4 x float> %pow) {
 ; CHECK-LABEL: @llvm_pow_f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @llvm.pow.v4f32(<4 x float> [[IN:%.*]], <4 x float> [[POW:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast <4 x float> @_ZGVnN4vv_powf(<4 x float> [[IN:%.*]], <4 x float> [[POW:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %in, <4 x float> %pow)
diff --git a/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll b/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll
index 8724e7c1c368d97c1f9679c79ba950a56adb1449..5adeef3ab7455233b78724be90da11e1707bd578 100644
--- a/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll
+++ b/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll
@@ -1066,6 +1066,44 @@ define void @streaming_compatible_no_sve(i32 noundef %x) #4 {
   ret void
 }
 
+; The algorithm that fixes up the offsets of the callee-save/restore
+; instructions must jump over the instructions that instantiate the current
+; 'VG' value. We must make sure that it doesn't consider any RDSVL in
+; user-code as if it is part of the frame-setup when doing so.
+define void @test_rdsvl_right_after_prologue(i64 %x0) nounwind {
+; NO-SVE-CHECK-LABEL: test_rdsvl_right_after_prologue:
+; NO-SVE-CHECK:     // %bb.0:
+; NO-SVE-CHECK-NEXT: stp     d15, d14, [sp, #-96]!           // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT: stp     d13, d12, [sp, #16]             // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT: mov     x9, x0
+; NO-SVE-CHECK-NEXT: stp     d11, d10, [sp, #32]             // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT: stp     d9, d8, [sp, #48]               // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT: stp     x29, x30, [sp, #64]             // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT: bl      __arm_get_current_vg
+; NO-SVE-CHECK-NEXT: str     x0, [sp, #80]                   // 8-byte Folded Spill
+; NO-SVE-CHECK-NEXT: mov     x0, x9
+; NO-SVE-CHECK-NEXT: rdsvl   x8, #1
+; NO-SVE-CHECK-NEXT: add     x29, sp, #64
+; NO-SVE-CHECK-NEXT: lsr     x8, x8, #3
+; NO-SVE-CHECK-NEXT: mov     x1, x0
+; NO-SVE-CHECK-NEXT: smstart sm
+; NO-SVE-CHECK-NEXT: mov     x0, x8
+; NO-SVE-CHECK-NEXT: bl      bar
+; NO-SVE-CHECK-NEXT: smstop  sm
+; NO-SVE-CHECK-NEXT: ldp     x29, x30, [sp, #64]             // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT: ldp     d9, d8, [sp, #48]               // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT: ldp     d11, d10, [sp, #32]             // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT: ldp     d13, d12, [sp, #16]             // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT: ldp     d15, d14, [sp], #96             // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT: ret
+  %some_alloc = alloca i64, align 8
+  %rdsvl = tail call i64 @llvm.aarch64.sme.cntsd()
+  call void @bar(i64 %rdsvl, i64 %x0) "aarch64_pstate_sm_enabled"
+  ret void
+}
+
+declare void @bar(i64, i64)
+
 ; Ensure we still emit async unwind information with -fno-asynchronous-unwind-tables
 ; if the function contains a streaming-mode change.
 
diff --git a/llvm/test/CodeGen/AArch64/stack-protector-trap-unreachable.ll b/llvm/test/CodeGen/AArch64/stack-protector-trap-unreachable.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a0bfd7fe06aa3d0af885dc060c09e73276d491a9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stack-protector-trap-unreachable.ll
@@ -0,0 +1,33 @@
+;; Make sure we emit trap instructions after stack protector checks iff NoTrapAfterNoReturn is false.
+
+; RUN: llc -mtriple=aarch64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=aarch64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=aarch64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,TRAP_UNREACHABLE %s
+
+;; Make sure FastISel doesn't break anything.
+; RUN: llc -mtriple=aarch64 -fast-isel -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=aarch64 -fast-isel -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=aarch64 -fast-isel -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,TRAP_UNREACHABLE %s
+
+; CHECK-LABEL: Machine code for function test
+; CHECK: bb.0.entry:
+; CHECK:  BL {{.}}__stack_chk_fail
+; CHECK-NEXT: ADJCALLSTACKUP
+; TRAP_UNREACHABLE-NEXT: BRK 1
+; NO_TRAP_UNREACHABLE-NOT: BRK 1
+; NO_TRAP_UNREACHABLE-EMPTY:
+
+define void @test() nounwind ssp {
+entry:
+  %buf = alloca [8 x i8]
+  %result = call i32(ptr) @callee(ptr %buf) nounwind
+  ret void
+}
+
+declare i32 @callee(ptr) nounwind
diff --git a/llvm/test/CodeGen/AArch64/statepoint-twoaddr.mir b/llvm/test/CodeGen/AArch64/statepoint-twoaddr.mir
index c1ddc9c14d814b4ebcd64a5c63c482e99da79841..51e9ed6fef2d3aa789651cf73f56fe87abaf7a83 100644
--- a/llvm/test/CodeGen/AArch64/statepoint-twoaddr.mir
+++ b/llvm/test/CodeGen/AArch64/statepoint-twoaddr.mir
@@ -1,7 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64-unknown-linux -run-pass=twoaddressinstruction -verify-machineinstrs %s -o - | FileCheck %s
 # RUN: llc -mtriple=aarch64-unknown-linux --passes=two-address-instruction -verify-each %s -o - | FileCheck %s
-# REQUIRES: aarch64-registered-target
 
 # Verify that the register class is correctly constrained after the twoaddress replacement
 ---
diff --git a/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll b/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll
index e1d883b0e789973a445129655a8b11495726474c..905d110e001c853b3c169635cc3fd3d11b7577c8 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
+; RUN: llc -use-constant-fp-for-scalable-splat < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
diff --git a/llvm/test/CodeGen/AArch64/sve-int-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-imm.ll
index 4be1abe84200814d85faa2bfab7ebbc0f9188b23..47f4f0181dfbdf47766937b247982a1c644be1df 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-imm.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -use-constant-int-for-scalable-splat < %s | FileCheck %s
 
 ;
 ; SVE Arith Vector Immediate Unpredicated CodeGen
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index f46e6ae989ff20ca07f0b56ec3b8331a8755523c..184aa0226fe774010fa84e8c17ec23ba4a146e88 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-BASE
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-DOT
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-BASE
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - -mattr=+dotprod 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-DOT
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-BASE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-DOT
 
 define i32 @addv_v2i32(<2 x i32> %a) {
 ; CHECK-LABEL: addv_v2i32:
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll b/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
index 823fdfb4809768cf0874457919b73f2f705970eb..830f1efbdf67f662e56640011085797cfd8f9373 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
@@ -1,4 +1,3 @@
-; REQUIRES: arm-registered-target
 ; REQUIRES: asserts
 ; RUN: llc -o /dev/null %s -debug-only=legalize-types 2>&1 | FileCheck %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
index ce528467cd35b476d53ccb93f92098e32278f262..6e2e88f22600a8835ffccaed814a31f06b822a2a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
@@ -2428,11 +2428,54 @@ define void @store_load_i64_unaligned(ptr addrspace(5) nocapture %arg) {
 ; UNALIGNED_GFX9-LABEL: store_load_i64_unaligned:
 ; UNALIGNED_GFX9:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v1, 15
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; UNALIGNED_GFX9-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v4, 15
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, 4, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v2, 2, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v3, 1, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v0, v4, off
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX9-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v6, 6, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v3, v4, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v5, 3, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v2, v4, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v5, v4, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v7, 5, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v1, v4, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v7, v4, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v8, 7, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v6, v4, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v8, v4, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v0, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr6
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr1
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr3
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr5
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr8
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr7
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr2
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr0
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v3, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v2, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v5, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v1, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v7, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v6, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v4, v8, off glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2441,30 +2484,143 @@ define void @store_load_i64_unaligned(ptr addrspace(5) nocapture %arg) {
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; UNALIGNED_GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v3, 4, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v5, 2, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v0, v1, off
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off glc dlc
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, 3, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v6, 5, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v7, 6, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v8, 7, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v4, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v5, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v1, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v3, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v6, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v7, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v8, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v0, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v4, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v5, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v1, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v3, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v6, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v7, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v8, off glc dlc
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX940-LABEL: store_load_i64_unaligned:
 ; UNALIGNED_GFX940:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX940-NEXT:    v_mov_b64_e32 v[2:3], 15
-; UNALIGNED_GFX940-NEXT:    scratch_store_dwordx2 v0, v[2:3], off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v4, 15
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, 4, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v2, 2, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v3, 1, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v0, v4, off sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX940-NEXT:    scratch_load_dwordx2 v[0:1], v0, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v6, 6, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v3, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v5, 3, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v2, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v5, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v7, 5, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v1, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v7, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v8, 7, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v6, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v8, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v0, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr6
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr1
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr3
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr5
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr8
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr7
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr2
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr0
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v2, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v5, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v1, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v7, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v6, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v4, v8, off sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX11-LABEL: store_load_i64_unaligned:
 ; UNALIGNED_GFX11:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX11-NEXT:    v_mov_b32_e32 v1, 15
-; UNALIGNED_GFX11-NEXT:    v_mov_b32_e32 v2, 0
-; UNALIGNED_GFX11-NEXT:    scratch_store_b64 v0, v[1:2], off dlc
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_mov_b32 v2, 0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v4, 1, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v3, 4, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v5, 2, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v0, v1, off dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT:    scratch_load_b64 v[0:1], v0, off glc dlc
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v6, 5, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v7, 6, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v8, 7, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v4, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v5, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v1, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v3, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v6, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v7, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v8, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v0, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v4, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v5, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v1, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v3, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v6, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v7, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v8, off glc dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2475,12 +2631,39 @@ define void @store_load_i64_unaligned(ptr addrspace(5) nocapture %arg) {
 ; UNALIGNED_GFX12-NEXT:    s_wait_samplecnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_wait_kmcnt 0x0
-; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v1, 15
-; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; UNALIGNED_GFX12-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_mov_b32 v2, 0
 ; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT:    scratch_store_b64 v0, v[1:2], off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v1, off scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT:    scratch_load_b64 v[0:1], v0, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v0, v0, off offset:7 scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_setpc_b64 s[30:31]
 bb:
@@ -2572,59 +2755,293 @@ define void @store_load_v3i32_unaligned(ptr addrspace(5) nocapture %arg) {
 ; UNALIGNED_GFX9-LABEL: store_load_v3i32_unaligned:
 ; UNALIGNED_GFX9:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX9-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX9-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX9-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v3, s2
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; UNALIGNED_GFX9-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v3, 1
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v1, 2
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v2, 2, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v4, 1, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v0, v3, off
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX9-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v6, 4, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v7, 6, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v9, 8, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v10, 10, v0
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v12, 3
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v4, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v5, 3, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v2, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v5, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v8, 5, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v6, v1, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v8, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, 7, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v7, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v1, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v11, 9, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v9, v12, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v11, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v12, 11, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v10, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v12, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v0, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr5
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr9
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr1
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr8
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr2
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr12
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr4
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr11
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr7
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr6
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr10
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr0
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v4, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v2, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v5, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v6, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v8, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v7, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v1, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v9, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v11, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v10, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v12, off glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX10-LABEL: store_load_v3i32_unaligned:
 ; UNALIGNED_GFX10:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX10-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX10-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX10-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v3, s2
-; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, s1
-; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, s0
-; UNALIGNED_GFX10-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, 1
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v3, 0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v5, 1, v0
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, 2
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v4, 2, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v0, v1, off
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX10-NEXT:    scratch_load_dwordx3 v[0:2], v0, off glc dlc
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, 3, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v6, 4, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v7, 5, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v5, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v4, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v8, 6, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v1, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v6, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v7, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v2, 7, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v9, 8, v0
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v10, 3
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v11, 9, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v12, 10, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v13, 11, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v8, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v2, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v9, v10, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v11, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v12, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v13, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v0, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v5, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v4, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v1, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v6, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v7, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v8, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v2, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v9, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v11, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v12, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v13, off glc dlc
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX940-LABEL: store_load_v3i32_unaligned:
 ; UNALIGNED_GFX940:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX940-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX940-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX940-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v4, s2
-; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v3, s1
-; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; UNALIGNED_GFX940-NEXT:    scratch_store_dwordx3 v0, v[2:4], off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v3, 1
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v1, 2
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v2, 2, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v4, 1, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v0, v3, off sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX940-NEXT:    scratch_load_dwordx3 v[0:2], v0, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v6, 4, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v7, 6, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v9, 8, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v10, 10, v0
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v12, 3
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v4, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v5, 3, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v2, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v5, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v8, 5, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v6, v1, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v8, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, 7, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v7, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v1, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v11, 9, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v9, v12, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v11, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v12, 11, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v10, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v12, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v0, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr5
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr9
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr1
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr8
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr2
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr12
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr4
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr11
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr7
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr6
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr10
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr0
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v2, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v5, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v6, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v8, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v7, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v1, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v9, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v11, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v10, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v12, off sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX11-LABEL: store_load_v3i32_unaligned:
 ; UNALIGNED_GFX11:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX11-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX11-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX11-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; UNALIGNED_GFX11-NEXT:    v_mov_b32_e32 v1, s0
-; UNALIGNED_GFX11-NEXT:    scratch_store_b96 v0, v[1:3], off dlc
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 2
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_add_nc_u32 v4, 2, v0
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v10, 3 :: v_dual_add_nc_u32 v5, 1, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v0, v1, off dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT:    scratch_load_b96 v[0:2], v0, off glc dlc
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v6, 4, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v7, 5, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v5, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v4, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v8, 6, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v1, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v6, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v7, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v2, 7, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v9, 8, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v11, 9, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v12, 10, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v13, 11, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v8, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v2, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v9, v10, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v11, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v12, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v13, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v0, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v5, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v4, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v1, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v6, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v7, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v8, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v2, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v9, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v11, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v12, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v13, off glc dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2635,16 +3052,57 @@ define void @store_load_v3i32_unaligned(ptr addrspace(5) nocapture %arg) {
 ; UNALIGNED_GFX12-NEXT:    s_wait_samplecnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_wait_kmcnt 0x0
-; UNALIGNED_GFX12-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX12-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX12-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX12-NEXT:    s_wait_alu 0xfffe
-; UNALIGNED_GFX12-NEXT:    v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v1, s0
+; UNALIGNED_GFX12-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
+; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v3, 2
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v1, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v1, 3
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v3, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v1, off offset:8 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:9 scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT:    scratch_store_b96 v0, v[1:3], off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:10 scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT:    scratch_load_b96 v[0:2], v0, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:11 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:8 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:9 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:10 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v0, v0, off offset:11 scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_setpc_b64 s[30:31]
 bb:
@@ -2742,64 +3200,382 @@ define void @store_load_v4i32_unaligned(ptr addrspace(5) nocapture %arg) {
 ; UNALIGNED_GFX9-LABEL: store_load_v4i32_unaligned:
 ; UNALIGNED_GFX9:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX9-NEXT:    s_mov_b32 s3, 4
-; UNALIGNED_GFX9-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX9-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX9-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v4, s3
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v3, s2
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; UNALIGNED_GFX9-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
-; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX9-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v3, 1
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v1, 2
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v2, 2, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v4, 1, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v0, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v6, 4
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v7, 4, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v8, 6, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v10, 8, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v11, 10, v0
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v13, 3
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v14, 12, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v15, 14, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v4, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v5, 3, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v2, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v5, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v9, 5, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v7, v1, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v9, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, 7, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v8, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v1, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v12, 9, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v10, v13, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v12, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v13, 11, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v11, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v13, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v16, 13, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v14, v6, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v16, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v6, 15, v0
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v15, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_store_byte v6, v3, off
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v0, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v4, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v2, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v5, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v7, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v9, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v8, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v1, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v10, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v12, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v11, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v13, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v14, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v16, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v3, v15, off glc
+; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr11
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr4
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr15
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr10
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr7
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr13
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr5
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr14
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr12
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr8
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr2
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr1
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr9
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr16
+; UNALIGNED_GFX9-NEXT:    ; kill: killed $vgpr0
+; UNALIGNED_GFX9-NEXT:    scratch_load_ubyte v0, v6, off glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX10-LABEL: store_load_v4i32_unaligned:
 ; UNALIGNED_GFX10:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX10-NEXT:    s_mov_b32 s3, 4
-; UNALIGNED_GFX10-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX10-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX10-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v4, s3
-; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v3, s2
-; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, s1
-; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, s0
-; UNALIGNED_GFX10-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, 1
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, 2
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v3, 0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v6, 4, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v0, v1, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, 3, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v5, 2, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v7, 5, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v4, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v5, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v1, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v9, 6, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v6, v2, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v7, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v2, 7, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v10, 8, v0
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v11, 3
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v12, 9, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v9, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v13, 10, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v2, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v10, v11, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v12, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v11, 11, v0
+; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v8, 4
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v14, 12, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v15, 13, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v16, 14, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v17, 15, v0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v13, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v11, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v14, v8, off
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX10-NEXT:    scratch_load_dwordx4 v[0:3], v0, off glc dlc
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v15, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v16, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_store_byte v17, v3, off
+; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v0, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v4, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v5, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v1, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v6, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v7, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v9, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v2, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v10, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v12, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v13, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v11, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v14, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v15, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v16, off glc dlc
+; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX10-NEXT:    scratch_load_ubyte v0, v17, off glc dlc
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX940-LABEL: store_load_v4i32_unaligned:
 ; UNALIGNED_GFX940:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX940-NEXT:    s_mov_b32 s3, 4
-; UNALIGNED_GFX940-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX940-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX940-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX940-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
-; UNALIGNED_GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
-; UNALIGNED_GFX940-NEXT:    scratch_store_dwordx4 v0, v[2:5], off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v3, 1
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v1, 2
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v2, 2, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v4, 1, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v0, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v6, 4
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v7, 4, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v8, 6, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v10, 8, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v11, 10, v0
+; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v13, 3
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v14, 12, v0
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v15, 14, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v4, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v5, 3, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v2, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v5, v3, off sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX940-NEXT:    scratch_load_dwordx4 v[0:3], v0, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v9, 5, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v7, v1, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v9, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, 7, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v8, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v1, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v12, 9, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v10, v13, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v12, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v13, 11, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v11, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v13, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v16, 13, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v14, v6, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v16, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v6, 15, v0
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v15, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_store_byte v6, v3, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v0, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v4, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v2, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v5, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v7, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v9, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v8, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v1, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v10, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v12, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v11, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v13, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v14, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v16, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v3, v15, off sc0 sc1
+; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr11
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr4
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr15
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr10
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr7
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr13
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr5
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr14
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr12
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr8
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr2
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr1
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr9
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr16
+; UNALIGNED_GFX940-NEXT:    ; kill: killed $vgpr0
+; UNALIGNED_GFX940-NEXT:    scratch_load_ubyte v0, v6, off sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; UNALIGNED_GFX11-LABEL: store_load_v4i32_unaligned:
 ; UNALIGNED_GFX11:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX11-NEXT:    s_mov_b32 s3, 4
-; UNALIGNED_GFX11-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX11-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX11-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v3, s2
-; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; UNALIGNED_GFX11-NEXT:    scratch_store_b128 v0, v[1:4], off dlc
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 2
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_add_nc_u32 v4, 1, v0
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v11, 3 :: v_dual_add_nc_u32 v6, 4, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v0, v1, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, 3, v0
+; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v8, 4 :: v_dual_add_nc_u32 v5, 2, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v7, 5, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v4, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v5, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v1, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v9, 6, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v6, v2, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v7, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v2, 7, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v10, 8, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v12, 9, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v9, v3, off dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT:    scratch_load_b128 v[0:3], v0, off glc dlc
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v13, 10, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v2, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v10, v11, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v12, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v11, 11, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v14, 12, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v15, 13, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v16, 14, v0
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v17, 15, v0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v13, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v11, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v14, v8, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v15, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v16, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_store_b8 v17, v3, off dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v0, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v4, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v5, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v1, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v6, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v7, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v9, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v2, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v10, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v12, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v13, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v11, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v14, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v15, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v16, off glc dlc
+; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX11-NEXT:    scratch_load_u8 v0, v17, off glc dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2810,17 +3586,74 @@ define void @store_load_v4i32_unaligned(ptr addrspace(5) nocapture %arg) {
 ; UNALIGNED_GFX12-NEXT:    s_wait_samplecnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_wait_kmcnt 0x0
-; UNALIGNED_GFX12-NEXT:    s_mov_b32 s3, 4
-; UNALIGNED_GFX12-NEXT:    s_mov_b32 s2, 3
-; UNALIGNED_GFX12-NEXT:    s_mov_b32 s1, 2
-; UNALIGNED_GFX12-NEXT:    s_mov_b32 s0, 1
-; UNALIGNED_GFX12-NEXT:    s_wait_alu 0xfffe
-; UNALIGNED_GFX12-NEXT:    v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v3, s2
-; UNALIGNED_GFX12-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; UNALIGNED_GFX12-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
+; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v3, 2
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v1, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v3, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v1, 3
+; UNALIGNED_GFX12-NEXT:    v_mov_b32_e32 v3, 4
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v1, off offset:8 scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT:    scratch_store_b128 v0, v[1:4], off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:9 scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT:    scratch_load_b128 v[0:3], v0, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:10 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:11 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v3, off offset:12 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:13 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:14 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_store_b8 v0, v2, off offset:15 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_storecnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:8 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:9 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:10 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:11 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:12 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:13 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v1, v0, off offset:14 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
+; UNALIGNED_GFX12-NEXT:    scratch_load_u8 v0, v0, off offset:15 scope:SCOPE_SYS
 ; UNALIGNED_GFX12-NEXT:    s_wait_loadcnt 0x0
 ; UNALIGNED_GFX12-NEXT:    s_setpc_b64 s[30:31]
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index b1d7d36f9912e785aac7b038b4eceed0b9c921fd..032ca7c0d4fee974b65c6465ec9911400bf108b2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -483,40 +483,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s16_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX12-LABEL: name: test_load_flat_s16_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s16_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -664,40 +646,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s32_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX12-LABEL: name: test_load_flat_s32_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 2)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s32_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -798,70 +762,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s32_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX12-LABEL: name: test_load_flat_s32_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s32_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -1247,76 +1163,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[OR2]](s64)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load (s64), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s64_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[OR2]](s64)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load (s64), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
     ;
     ; GFX12-LABEL: name: test_load_flat_s64_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[OR2]](s64)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load (s64), align 2)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s64_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -1485,130 +1347,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[OR6]](s64)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load (s64), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s64_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[OR6]](s64)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load (s64), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
     ;
     ; GFX12-LABEL: name: test_load_flat_s64_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[OR6]](s64)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load (s64), align 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s64_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -2075,87 +1829,24 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p0) :: (load (<3 x s32>), align 2)
+    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
     ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s96_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p0) :: (load (<3 x s32>), align 2)
+    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
     ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; GFX12-LABEL: name: test_load_flat_s96_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
-    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p0) :: (load (<3 x s32>), align 2)
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
     ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s96_align2
@@ -2369,165 +2060,24 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX9PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
-    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p0) :: (load (<3 x s32>), align 1)
+    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
     ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s96_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX11PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX11PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
-    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p0) :: (load (<3 x s32>), align 1)
+    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
     ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; GFX12-LABEL: name: test_load_flat_s96_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
-    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p0) :: (load (<3 x s32>), align 1)
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
     ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s96_align1
@@ -3334,210 +2884,24 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX9PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-    ; GFX9PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX9PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX9PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX9PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX9PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
-    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p0) :: (load (<4 x s32>), align 1)
+    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
     ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_s128_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX11PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX11PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX11PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-    ; GFX11PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX11PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX11PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX11PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX11PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX11PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
-    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p0) :: (load (<4 x s32>), align 1)
+    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
     ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
     ; GFX12-LABEL: name: test_load_flat_s128_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX12-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-    ; GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
-    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p0) :: (load (<4 x s32>), align 1)
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
     ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_s128_align1
@@ -4132,133 +3496,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR6]](s64)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p0) :: (load (p1), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_p1_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR6]](s64)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p0) :: (load (p1), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
     ;
     ; GFX12-LABEL: name: test_load_flat_p1_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR6]](s64)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p0) :: (load (p1), align 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_p1_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -4662,79 +3915,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR2]](s64)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p4)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p0) :: (load (p4), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_p4_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR2]](s64)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p4)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p0) :: (load (p4), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
     ;
     ; GFX12-LABEL: name: test_load_flat_p4_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR2]](s64)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p4)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p0) :: (load (p4), align 2)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_p4_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -4906,133 +4102,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR6]](s64)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p4)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p0) :: (load (p4), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_p4_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR6]](s64)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p4)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p0) :: (load (p4), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
     ;
     ; GFX12-LABEL: name: test_load_flat_p4_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR6]](s64)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p4)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p0) :: (load (p4), align 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_p4_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -5274,43 +4359,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR]](s32)
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p0) :: (load (p5), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_p5_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR]](s32)
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p0) :: (load (p5), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX12-LABEL: name: test_load_flat_p5_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR]](s32)
-    ; GFX12-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p0) :: (load (p5), align 2)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_p5_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -5416,73 +4480,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR2]](s32)
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p0) :: (load (p5), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_p5_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR2]](s32)
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p0) :: (load (p5), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX12-LABEL: name: test_load_flat_p5_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR2]](s32)
-    ; GFX12-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p0) :: (load (p5), align 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_p5_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -5732,40 +4745,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2s8_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2s8_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2s8_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -6158,121 +5153,106 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 2)
+    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
-    ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C3]](s32)
+    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C2]](s32)
+    ; GFX9PLUS-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
     ; GFX9PLUS-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; GFX9PLUS-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
+    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9PLUS-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
     ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9PLUS-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C4]]
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C5]](s16)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
+    ; GFX9PLUS-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL1]]
     ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; GFX9PLUS-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
+    ; GFX9PLUS-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C3]]
     ; GFX9PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; GFX9PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR4]](s32)
+    ; GFX9PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
+    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
+    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
+    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR3]](s32)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v3s8_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 2)
+    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
-    ; GFX11PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C3]](s32)
+    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX11PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C2]](s32)
+    ; GFX11PLUS-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
     ; GFX11PLUS-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; GFX11PLUS-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
+    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX11PLUS-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
     ; GFX11PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX11PLUS-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C4]]
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C5]](s16)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
+    ; GFX11PLUS-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL1]]
     ; GFX11PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; GFX11PLUS-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
+    ; GFX11PLUS-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C3]]
     ; GFX11PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; GFX11PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; GFX11PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR4]](s32)
+    ; GFX11PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
+    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
+    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
+    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; GFX11PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR3]](s32)
     ;
     ; GFX12-LABEL: name: test_load_flat_v3s8_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 2)
+    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
-    ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C1]](s32)
-    ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C3]](s32)
+    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C2]](s32)
+    ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
     ; GFX12-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; GFX12-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
+    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX12-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
     ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX12-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C4]]
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C5]](s16)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
+    ; GFX12-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL1]]
     ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; GFX12-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
+    ; GFX12-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C3]]
     ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; GFX12-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; GFX12-NEXT: $vgpr0 = COPY [[OR4]](s32)
+    ; GFX12-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
+    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
+    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
+    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
+    ; GFX12-NEXT: $vgpr0 = COPY [[OR3]](s32)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v3s8_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -6503,40 +5483,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v4s8_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX12-LABEL: name: test_load_flat_v4s8_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 2)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v4s8_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -6638,70 +5600,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v4s8_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX12-LABEL: name: test_load_flat_v4s8_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v4s8_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -7185,40 +6099,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load (<2 x s16>), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2s16_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load (<2 x s16>), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2s16_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX12-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load (<2 x s16>), align 2)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2s16_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -7327,70 +6223,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load (<2 x s16>), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2s16_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX11PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load (<2 x s16>), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2s16_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX12-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load (<2 x s16>), align 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2s16_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -8291,36 +7139,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2, align 1)
+    ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4, align 1)
+    ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX9PLUS-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
     ; GFX9PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
+    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
     ; GFX9PLUS-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
     ; GFX9PLUS-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
@@ -8334,36 +7168,22 @@ body: |
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX11PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2, align 1)
+    ; GFX11PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4, align 1)
+    ; GFX11PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX11PLUS-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; GFX11PLUS-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
     ; GFX11PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
+    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX11PLUS-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
     ; GFX11PLUS-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX11PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
     ; GFX11PLUS-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
@@ -8377,36 +7197,22 @@ body: |
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16), align 1)
+    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2, align 1)
+    ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4, align 1)
+    ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
     ; GFX12-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; GFX12-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
     ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
+    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
     ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
     ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
@@ -8765,70 +7571,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX9PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load (<4 x s16>), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v4s16_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX11PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load (<4 x s16>), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v4s16_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load (<4 x s16>), align 2)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v4s16_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -9005,124 +7763,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR3]](s32)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX9PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load (<4 x s16>), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v4s16_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX11PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR3]](s32)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX11PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load (<4 x s16>), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v4s16_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR3]](s32)
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load (<4 x s16>), align 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v4s16_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -10686,133 +9342,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR3]](s32)
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s16) from unknown-address + 12)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s16) from unknown-address + 14)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR4]](s32)
-    ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL5]], [[ZEXT1]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR2]](s64), [[OR5]](s64)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 2)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2s64_align2
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR3]](s32)
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s16) from unknown-address + 12)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s16) from unknown-address + 14)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR4]](s32)
-    ; GFX11PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL5]], [[ZEXT1]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR2]](s64), [[OR5]](s64)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 2)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2s64_align2
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s16))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR3]](s32)
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s16) from unknown-address + 12)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s16) from unknown-address + 14)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR4]](s32)
-    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL5]], [[ZEXT1]]
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR2]](s64), [[OR5]](s64)
-    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 2)
+    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2s64_align2
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -11100,235 +9645,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX9PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX9PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX9PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX9PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX9PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX9PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX9PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX9PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX9PLUS-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX9PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX9PLUS-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2s64_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX11PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX11PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX11PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX11PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX11PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX11PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX11PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX11PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX11PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX11PLUS-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX11PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX11PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX11PLUS-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2s64_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX12-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX12-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX12-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX12-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX12-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
-    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2s64_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -12078,342 +10410,42 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
     ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX9PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX9PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX9PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX9PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX9PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX9PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX9PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX9PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX9PLUS-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX9PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX9PLUS-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX9PLUS-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; GFX9PLUS-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
-    ; GFX9PLUS-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p0) :: (load (s8) from unknown-address + 17)
-    ; GFX9PLUS-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[ZEXTLOAD12]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p0) :: (load (s8) from unknown-address + 18)
-    ; GFX9PLUS-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD17]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p0) :: (load (s8) from unknown-address + 19)
-    ; GFX9PLUS-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD14]]
-    ; GFX9PLUS-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[OR15]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[OR14]]
-    ; GFX9PLUS-NEXT: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[OR16]](s32)
-    ; GFX9PLUS-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
-    ; GFX9PLUS-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p0) :: (load (s8) from unknown-address + 21)
-    ; GFX9PLUS-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[ZEXTLOAD15]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p0) :: (load (s8) from unknown-address + 22)
-    ; GFX9PLUS-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD21]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p0) :: (load (s8) from unknown-address + 23)
-    ; GFX9PLUS-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[SHL18]], [[ZEXTLOAD17]]
-    ; GFX9PLUS-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[OR18]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[SHL19]], [[OR17]]
-    ; GFX9PLUS-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[OR19]](s32)
-    ; GFX9PLUS-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[SHL20:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT2]], [[COPY2]](s32)
-    ; GFX9PLUS-NEXT: [[OR20:%[0-9]+]]:_(s64) = G_OR [[SHL20]], [[ZEXT2]]
+    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p0) :: (load (s64) from unknown-address + 16, align 1)
+    ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[LOAD]](<2 x s64>)
     ; GFX9PLUS-NEXT: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[DEF]](<4 x s64>)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64), [[OR20]](s64), [[UV3]](s64)
+    ; GFX9PLUS-NEXT: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64), [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[DEF]](<4 x s64>)
+    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[UV]](s64), [[UV1]](s64), [[LOAD1]](s64), [[UV5]](s64)
     ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v3s64_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
     ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX11PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX11PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX11PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX11PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX11PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX11PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX11PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX11PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX11PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX11PLUS-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX11PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX11PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX11PLUS-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX11PLUS-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; GFX11PLUS-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
-    ; GFX11PLUS-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p0) :: (load (s8) from unknown-address + 17)
-    ; GFX11PLUS-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[ZEXTLOAD12]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p0) :: (load (s8) from unknown-address + 18)
-    ; GFX11PLUS-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD17]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p0) :: (load (s8) from unknown-address + 19)
-    ; GFX11PLUS-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD14]]
-    ; GFX11PLUS-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[OR15]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[OR14]]
-    ; GFX11PLUS-NEXT: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[OR16]](s32)
-    ; GFX11PLUS-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
-    ; GFX11PLUS-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p0) :: (load (s8) from unknown-address + 21)
-    ; GFX11PLUS-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[ZEXTLOAD15]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p0) :: (load (s8) from unknown-address + 22)
-    ; GFX11PLUS-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD21]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p0) :: (load (s8) from unknown-address + 23)
-    ; GFX11PLUS-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[SHL18]], [[ZEXTLOAD17]]
-    ; GFX11PLUS-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[OR18]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[SHL19]], [[OR17]]
-    ; GFX11PLUS-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[OR19]](s32)
-    ; GFX11PLUS-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[SHL20:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT2]], [[COPY2]](s32)
-    ; GFX11PLUS-NEXT: [[OR20:%[0-9]+]]:_(s64) = G_OR [[SHL20]], [[ZEXT2]]
+    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p0) :: (load (s64) from unknown-address + 16, align 1)
+    ; GFX11PLUS-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[LOAD]](<2 x s64>)
     ; GFX11PLUS-NEXT: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX11PLUS-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[DEF]](<4 x s64>)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64), [[OR20]](s64), [[UV3]](s64)
+    ; GFX11PLUS-NEXT: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64), [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[DEF]](<4 x s64>)
+    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[UV]](s64), [[UV1]](s64), [[LOAD1]](s64), [[UV5]](s64)
     ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v3s64_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
     ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX12-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX12-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX12-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX12-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX12-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX12-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; GFX12-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
-    ; GFX12-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p0) :: (load (s8) from unknown-address + 17)
-    ; GFX12-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[ZEXTLOAD12]]
-    ; GFX12-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p0) :: (load (s8) from unknown-address + 18)
-    ; GFX12-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD17]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p0) :: (load (s8) from unknown-address + 19)
-    ; GFX12-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD14]]
-    ; GFX12-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[OR15]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[OR14]]
-    ; GFX12-NEXT: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[OR16]](s32)
-    ; GFX12-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
-    ; GFX12-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p0) :: (load (s8) from unknown-address + 21)
-    ; GFX12-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[ZEXTLOAD15]]
-    ; GFX12-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p0) :: (load (s8) from unknown-address + 22)
-    ; GFX12-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD21]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p0) :: (load (s8) from unknown-address + 23)
-    ; GFX12-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[SHL18]], [[ZEXTLOAD17]]
-    ; GFX12-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[OR18]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[SHL19]], [[OR17]]
-    ; GFX12-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[OR19]](s32)
-    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX12-NEXT: [[SHL20:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT2]], [[COPY2]](s32)
-    ; GFX12-NEXT: [[OR20:%[0-9]+]]:_(s64) = G_OR [[SHL20]], [[ZEXT2]]
+    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p0) :: (load (s64) from unknown-address + 16, align 1)
+    ; GFX12-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[LOAD]](<2 x s64>)
     ; GFX12-NEXT: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX12-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[DEF]](<4 x s64>)
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64), [[OR20]](s64), [[UV3]](s64)
+    ; GFX12-NEXT: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64), [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[DEF]](<4 x s64>)
+    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[UV]](s64), [[UV1]](s64), [[LOAD1]](s64), [[UV5]](s64)
     ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v3s64_align1
@@ -13306,441 +11338,33 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
     ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX9PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX9PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX9PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX9PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX9PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX9PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX9PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX9PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX9PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX9PLUS-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX9PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX9PLUS-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
-    ; GFX9PLUS-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; GFX9PLUS-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
-    ; GFX9PLUS-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p0) :: (load (s8) from unknown-address + 17)
-    ; GFX9PLUS-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[ZEXTLOAD12]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p0) :: (load (s8) from unknown-address + 18)
-    ; GFX9PLUS-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD17]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p0) :: (load (s8) from unknown-address + 19)
-    ; GFX9PLUS-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD14]]
-    ; GFX9PLUS-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[OR15]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[OR14]]
-    ; GFX9PLUS-NEXT: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[OR16]](s32)
-    ; GFX9PLUS-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
-    ; GFX9PLUS-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p0) :: (load (s8) from unknown-address + 21)
-    ; GFX9PLUS-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[ZEXTLOAD15]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p0) :: (load (s8) from unknown-address + 22)
-    ; GFX9PLUS-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD21]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p0) :: (load (s8) from unknown-address + 23)
-    ; GFX9PLUS-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[SHL18]], [[ZEXTLOAD17]]
-    ; GFX9PLUS-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[OR18]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[SHL19]], [[OR17]]
-    ; GFX9PLUS-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[OR19]](s32)
-    ; GFX9PLUS-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[SHL20:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT2]], [[COPY2]](s32)
-    ; GFX9PLUS-NEXT: [[OR20:%[0-9]+]]:_(s64) = G_OR [[SHL20]], [[ZEXT2]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD23:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C6]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD18:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD23]](p0) :: (load (s8) from unknown-address + 24)
-    ; GFX9PLUS-NEXT: [[PTR_ADD24:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD19:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD24]](p0) :: (load (s8) from unknown-address + 25)
-    ; GFX9PLUS-NEXT: [[SHL21:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD19]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR21:%[0-9]+]]:_(s32) = G_OR [[SHL21]], [[ZEXTLOAD18]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD25:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD20:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD25]](p0) :: (load (s8) from unknown-address + 26)
-    ; GFX9PLUS-NEXT: [[PTR_ADD26:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD25]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD26]](p0) :: (load (s8) from unknown-address + 27)
-    ; GFX9PLUS-NEXT: [[SHL22:%[0-9]+]]:_(s32) = G_SHL [[LOAD6]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR22:%[0-9]+]]:_(s32) = G_OR [[SHL22]], [[ZEXTLOAD20]]
-    ; GFX9PLUS-NEXT: [[SHL23:%[0-9]+]]:_(s32) = G_SHL [[OR22]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[SHL23]], [[OR21]]
-    ; GFX9PLUS-NEXT: [[ZEXT3:%[0-9]+]]:_(s64) = G_ZEXT [[OR23]](s32)
-    ; GFX9PLUS-NEXT: [[PTR_ADD27:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD21:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD27]](p0) :: (load (s8) from unknown-address + 28)
-    ; GFX9PLUS-NEXT: [[PTR_ADD28:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD27]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD22:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD28]](p0) :: (load (s8) from unknown-address + 29)
-    ; GFX9PLUS-NEXT: [[SHL24:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD22]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR24:%[0-9]+]]:_(s32) = G_OR [[SHL24]], [[ZEXTLOAD21]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD29:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD27]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD23:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD29]](p0) :: (load (s8) from unknown-address + 30)
-    ; GFX9PLUS-NEXT: [[PTR_ADD30:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD29]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD30]](p0) :: (load (s8) from unknown-address + 31)
-    ; GFX9PLUS-NEXT: [[SHL25:%[0-9]+]]:_(s32) = G_SHL [[LOAD7]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR25:%[0-9]+]]:_(s32) = G_OR [[SHL25]], [[ZEXTLOAD23]]
-    ; GFX9PLUS-NEXT: [[SHL26:%[0-9]+]]:_(s32) = G_SHL [[OR25]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR26:%[0-9]+]]:_(s32) = G_OR [[SHL26]], [[OR24]]
-    ; GFX9PLUS-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[OR26]](s32)
-    ; GFX9PLUS-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX9PLUS-NEXT: [[SHL27:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT3]], [[COPY3]](s32)
-    ; GFX9PLUS-NEXT: [[OR27:%[0-9]+]]:_(s64) = G_OR [[SHL27]], [[ZEXT3]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR20]](s64), [[OR27]](s64)
-    ; GFX9PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s64>), [[BUILD_VECTOR1]](<2 x s64>)
+    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[PTR_ADD]](p0) :: (load (<2 x s64>) from unknown-address + 16, align 1)
+    ; GFX9PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>)
     ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<4 x s64>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v4s64_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
     ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX11PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX11PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX11PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX11PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX11PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX11PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX11PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX11PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX11PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX11PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX11PLUS-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX11PLUS-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX11PLUS-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX11PLUS-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
-    ; GFX11PLUS-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; GFX11PLUS-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
-    ; GFX11PLUS-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p0) :: (load (s8) from unknown-address + 17)
-    ; GFX11PLUS-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[ZEXTLOAD12]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p0) :: (load (s8) from unknown-address + 18)
-    ; GFX11PLUS-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD17]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p0) :: (load (s8) from unknown-address + 19)
-    ; GFX11PLUS-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD14]]
-    ; GFX11PLUS-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[OR15]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[OR14]]
-    ; GFX11PLUS-NEXT: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[OR16]](s32)
-    ; GFX11PLUS-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
-    ; GFX11PLUS-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p0) :: (load (s8) from unknown-address + 21)
-    ; GFX11PLUS-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[ZEXTLOAD15]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p0) :: (load (s8) from unknown-address + 22)
-    ; GFX11PLUS-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD21]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p0) :: (load (s8) from unknown-address + 23)
-    ; GFX11PLUS-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[SHL18]], [[ZEXTLOAD17]]
-    ; GFX11PLUS-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[OR18]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[SHL19]], [[OR17]]
-    ; GFX11PLUS-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[OR19]](s32)
-    ; GFX11PLUS-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[SHL20:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT2]], [[COPY2]](s32)
-    ; GFX11PLUS-NEXT: [[OR20:%[0-9]+]]:_(s64) = G_OR [[SHL20]], [[ZEXT2]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD23:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C6]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD18:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD23]](p0) :: (load (s8) from unknown-address + 24)
-    ; GFX11PLUS-NEXT: [[PTR_ADD24:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD19:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD24]](p0) :: (load (s8) from unknown-address + 25)
-    ; GFX11PLUS-NEXT: [[SHL21:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD19]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR21:%[0-9]+]]:_(s32) = G_OR [[SHL21]], [[ZEXTLOAD18]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD25:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD20:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD25]](p0) :: (load (s8) from unknown-address + 26)
-    ; GFX11PLUS-NEXT: [[PTR_ADD26:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD25]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD26]](p0) :: (load (s8) from unknown-address + 27)
-    ; GFX11PLUS-NEXT: [[SHL22:%[0-9]+]]:_(s32) = G_SHL [[LOAD6]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR22:%[0-9]+]]:_(s32) = G_OR [[SHL22]], [[ZEXTLOAD20]]
-    ; GFX11PLUS-NEXT: [[SHL23:%[0-9]+]]:_(s32) = G_SHL [[OR22]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[SHL23]], [[OR21]]
-    ; GFX11PLUS-NEXT: [[ZEXT3:%[0-9]+]]:_(s64) = G_ZEXT [[OR23]](s32)
-    ; GFX11PLUS-NEXT: [[PTR_ADD27:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD21:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD27]](p0) :: (load (s8) from unknown-address + 28)
-    ; GFX11PLUS-NEXT: [[PTR_ADD28:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD27]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD22:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD28]](p0) :: (load (s8) from unknown-address + 29)
-    ; GFX11PLUS-NEXT: [[SHL24:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD22]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR24:%[0-9]+]]:_(s32) = G_OR [[SHL24]], [[ZEXTLOAD21]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD29:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD27]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD23:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD29]](p0) :: (load (s8) from unknown-address + 30)
-    ; GFX11PLUS-NEXT: [[PTR_ADD30:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD29]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD30]](p0) :: (load (s8) from unknown-address + 31)
-    ; GFX11PLUS-NEXT: [[SHL25:%[0-9]+]]:_(s32) = G_SHL [[LOAD7]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR25:%[0-9]+]]:_(s32) = G_OR [[SHL25]], [[ZEXTLOAD23]]
-    ; GFX11PLUS-NEXT: [[SHL26:%[0-9]+]]:_(s32) = G_SHL [[OR25]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR26:%[0-9]+]]:_(s32) = G_OR [[SHL26]], [[OR24]]
-    ; GFX11PLUS-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[OR26]](s32)
-    ; GFX11PLUS-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX11PLUS-NEXT: [[SHL27:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT3]], [[COPY3]](s32)
-    ; GFX11PLUS-NEXT: [[OR27:%[0-9]+]]:_(s64) = G_OR [[SHL27]], [[ZEXT3]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR20]](s64), [[OR27]](s64)
-    ; GFX11PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s64>), [[BUILD_VECTOR1]](<2 x s64>)
+    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[PTR_ADD]](p0) :: (load (<2 x s64>) from unknown-address + 16, align 1)
+    ; GFX11PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>)
     ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<4 x s64>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v4s64_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p0) :: (load (<2 x s64>), align 1)
+    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
     ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
-    ; GFX12-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
-    ; GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
-    ; GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
-    ; GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
-    ; GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
-    ; GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
-    ; GFX12-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
-    ; GFX12-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
-    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX12-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
-    ; GFX12-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
-    ; GFX12-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; GFX12-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
-    ; GFX12-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p0) :: (load (s8) from unknown-address + 17)
-    ; GFX12-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[ZEXTLOAD12]]
-    ; GFX12-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p0) :: (load (s8) from unknown-address + 18)
-    ; GFX12-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD17]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p0) :: (load (s8) from unknown-address + 19)
-    ; GFX12-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD14]]
-    ; GFX12-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[OR15]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[OR14]]
-    ; GFX12-NEXT: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[OR16]](s32)
-    ; GFX12-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
-    ; GFX12-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p0) :: (load (s8) from unknown-address + 21)
-    ; GFX12-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[ZEXTLOAD15]]
-    ; GFX12-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p0) :: (load (s8) from unknown-address + 22)
-    ; GFX12-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD21]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p0) :: (load (s8) from unknown-address + 23)
-    ; GFX12-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[SHL18]], [[ZEXTLOAD17]]
-    ; GFX12-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[OR18]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[SHL19]], [[OR17]]
-    ; GFX12-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[OR19]](s32)
-    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX12-NEXT: [[SHL20:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT2]], [[COPY2]](s32)
-    ; GFX12-NEXT: [[OR20:%[0-9]+]]:_(s64) = G_OR [[SHL20]], [[ZEXT2]]
-    ; GFX12-NEXT: [[PTR_ADD23:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD15]], [[C6]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD18:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD23]](p0) :: (load (s8) from unknown-address + 24)
-    ; GFX12-NEXT: [[PTR_ADD24:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD19:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD24]](p0) :: (load (s8) from unknown-address + 25)
-    ; GFX12-NEXT: [[SHL21:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD19]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR21:%[0-9]+]]:_(s32) = G_OR [[SHL21]], [[ZEXTLOAD18]]
-    ; GFX12-NEXT: [[PTR_ADD25:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD20:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD25]](p0) :: (load (s8) from unknown-address + 26)
-    ; GFX12-NEXT: [[PTR_ADD26:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD25]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD26]](p0) :: (load (s8) from unknown-address + 27)
-    ; GFX12-NEXT: [[SHL22:%[0-9]+]]:_(s32) = G_SHL [[LOAD6]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR22:%[0-9]+]]:_(s32) = G_OR [[SHL22]], [[ZEXTLOAD20]]
-    ; GFX12-NEXT: [[SHL23:%[0-9]+]]:_(s32) = G_SHL [[OR22]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[SHL23]], [[OR21]]
-    ; GFX12-NEXT: [[ZEXT3:%[0-9]+]]:_(s64) = G_ZEXT [[OR23]](s32)
-    ; GFX12-NEXT: [[PTR_ADD27:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD23]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD21:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD27]](p0) :: (load (s8) from unknown-address + 28)
-    ; GFX12-NEXT: [[PTR_ADD28:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD27]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD22:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD28]](p0) :: (load (s8) from unknown-address + 29)
-    ; GFX12-NEXT: [[SHL24:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD22]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR24:%[0-9]+]]:_(s32) = G_OR [[SHL24]], [[ZEXTLOAD21]]
-    ; GFX12-NEXT: [[PTR_ADD29:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD27]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD23:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD29]](p0) :: (load (s8) from unknown-address + 30)
-    ; GFX12-NEXT: [[PTR_ADD30:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD29]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD30]](p0) :: (load (s8) from unknown-address + 31)
-    ; GFX12-NEXT: [[SHL25:%[0-9]+]]:_(s32) = G_SHL [[LOAD7]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR25:%[0-9]+]]:_(s32) = G_OR [[SHL25]], [[ZEXTLOAD23]]
-    ; GFX12-NEXT: [[SHL26:%[0-9]+]]:_(s32) = G_SHL [[OR25]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR26:%[0-9]+]]:_(s32) = G_OR [[SHL26]], [[OR24]]
-    ; GFX12-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[OR26]](s32)
-    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
-    ; GFX12-NEXT: [[SHL27:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT3]], [[COPY3]](s32)
-    ; GFX12-NEXT: [[OR27:%[0-9]+]]:_(s64) = G_OR [[SHL27]], [[ZEXT3]]
-    ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR20]](s64), [[OR27]](s64)
-    ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s64>), [[BUILD_VECTOR1]](<2 x s64>)
+    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[PTR_ADD]](p0) :: (load (<2 x s64>) from unknown-address + 16, align 1)
+    ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>)
     ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<4 x s64>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v4s64_align1
@@ -14762,210 +12386,24 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX9PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX9PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX9PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX9PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX9PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-    ; GFX9PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX9PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX9PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX9PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX9PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
-    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p1>) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p0) :: (load (<4 x s32>), align 1)
+    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p1>) = G_BITCAST [[LOAD]](<4 x s32>)
     ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](<2 x p1>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2p1_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX11PLUS-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX11PLUS-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX11PLUS-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX11PLUS-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX11PLUS-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX11PLUS-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX11PLUS-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-    ; GFX11PLUS-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX11PLUS-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX11PLUS-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX11PLUS-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX11PLUS-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX11PLUS-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
-    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p1>) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p0) :: (load (<4 x s32>), align 1)
+    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p1>) = G_BITCAST [[LOAD]](<4 x s32>)
     ; GFX11PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](<2 x p1>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2p1_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX12-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-    ; GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
-    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p1>) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p0) :: (load (<4 x s32>), align 1)
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p1>) = G_BITCAST [[LOAD]](<4 x s32>)
     ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](<2 x p1>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2p1_align1
@@ -15422,124 +12860,22 @@ body: |
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX9PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX9PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX9PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX9PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
-    ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX9PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX9PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX9PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX9PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 1)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2p3_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX11PLUS-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX11PLUS-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX11PLUS-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX11PLUS-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX11PLUS-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX11PLUS-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX11PLUS-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX11PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
-    ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX11PLUS-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX11PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX11PLUS-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX11PLUS-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX11PLUS-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX11PLUS-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX11PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 1)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2p3_align1
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load (s8))
-    ; GFX12-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX12-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
-    ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
-    ; GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
-    ; GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2p3_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index 741f878c86f8b64f80e7f0d2f6b4522d65ab13c9..6d93112aae1a06bc8d4d8ee35e8a73a55b0043cb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -636,27 +636,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX10-LABEL: name: test_load_private_s16_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11-LABEL: name: test_load_private_s16_align1
     ; GFX11: liveins: $vgpr0
@@ -702,15 +690,27 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s16_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[OR]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(s16) = G_LOAD %0 :: (load (s16), align 1, addrspace 5)
     %2:_(s32) = G_ANYEXT %1
@@ -853,27 +853,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX10-LABEL: name: test_load_private_s32_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11-LABEL: name: test_load_private_s32_align2
     ; GFX11: liveins: $vgpr0
@@ -919,15 +907,27 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s32_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[OR]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(s32) = G_LOAD %0 :: (load (s32), align 2, addrspace 5)
     $vgpr0 = COPY %1
@@ -1012,47 +1012,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX10-LABEL: name: test_load_private_s32_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](s32)
     ;
     ; GFX11-LABEL: name: test_load_private_s32_align1
     ; GFX11: liveins: $vgpr0
@@ -1118,15 +1086,47 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[OR2]](s32)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s32_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[OR2]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(s32) = G_LOAD %0 :: (load (s32), align 1, addrspace 5)
     $vgpr0 = COPY %1
@@ -1529,39 +1529,27 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
-    ; GFX9-NEXT: $vgpr0 = COPY [[OR1]](s32)
+    ; GFX9-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; GFX10-LABEL: name: test_load_private_s24_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
-    ; GFX10-NEXT: $vgpr0 = COPY [[OR1]](s32)
+    ; GFX10-NEXT: $vgpr0 = COPY [[OR]](s32)
     ;
     ; GFX11-LABEL: name: test_load_private_s24_align1
     ; GFX11: liveins: $vgpr0
@@ -1631,27 +1619,39 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
     ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[OR1]](s32)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s24_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
     ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[OR1]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(s24) = G_LOAD %0 :: (load (s24), align 1, addrspace 5)
     %2:_(s32) = G_ANYEXT %1
@@ -2147,42 +2147,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
     ;
     ; GFX10-LABEL: name: test_load_private_s64_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
     ;
     ; GFX11-LABEL: name: test_load_private_s64_align2
@@ -2245,15 +2225,51 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p5) :: (load (s64), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[OR2]](s64)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s64_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p5) :: (load (s64), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[OR2]](s64)
     %0:_(p5) = COPY $vgpr0
     %1:_(s64) = G_LOAD %0 :: (load (s64), align 2, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -2386,78 +2402,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
     ;
     ; GFX10-LABEL: name: test_load_private_s64_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
     ;
     ; GFX11-LABEL: name: test_load_private_s64_align1
@@ -2556,15 +2516,87 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p5) :: (load (s64), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[OR6]](s64)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s64_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p5) :: (load (s64), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[OR6]](s64)
     %0:_(p5) = COPY $vgpr0
     %1:_(s64) = G_LOAD %0 :: (load (s64), align 1, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -2742,53 +2774,14 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
@@ -2796,53 +2789,14 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
@@ -2974,16 +2928,108 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s96_align16
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     %0:_(p5) = COPY $vgpr0
     %1:_(s96) = G_LOAD %0 :: (load (s96), align 1, addrspace 5)
@@ -3381,28 +3427,14 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
@@ -3410,28 +3442,14 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
@@ -3513,16 +3531,58 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s96_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     %0:_(p5) = COPY $vgpr0
     %1:_(s96) = G_LOAD %0 :: (load (s96), align 2, addrspace 5)
@@ -3701,53 +3761,14 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
@@ -3755,53 +3776,14 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
@@ -3933,16 +3915,108 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s96_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BITCAST]](s96)
     %0:_(p5) = COPY $vgpr0
     %1:_(s96) = G_LOAD %0 :: (load (s96), align 1, addrspace 5)
@@ -4166,68 +4240,17 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
@@ -4235,68 +4258,17 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX10-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
@@ -4458,16 +4430,138 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s128_align16
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     %0:_(p5) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load (s128), align 1, addrspace 5)
@@ -4928,35 +5022,17 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 2, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
@@ -4964,35 +5040,17 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 2, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
@@ -5088,16 +5146,72 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s128_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     %0:_(p5) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load (s128), align 2, addrspace 5)
@@ -5321,68 +5435,17 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
@@ -5390,68 +5453,17 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX10-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
@@ -5613,16 +5625,138 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_s128_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s128) = G_BITCAST [[BUILD_VECTOR]](<4 x s32>)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](s128)
     %0:_(p5) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load (s128), align 1, addrspace 5)
@@ -5932,42 +6066,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
     ;
     ; GFX10-LABEL: name: test_load_private_p1_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
     ;
     ; GFX11-LABEL: name: test_load_private_p1_align2
@@ -6030,15 +6144,53 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p5) :: (load (p1), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
+    ; UNALIGNED_GFX11-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR2]](s64)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_p1_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p5) :: (load (p1), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[ZEXT]]
+    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR2]](s64)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
     %0:_(p5) = COPY $vgpr0
     %1:_(p1) = G_LOAD %0 :: (load (p1), align 2, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -6171,78 +6323,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
     ;
     ; GFX10-LABEL: name: test_load_private_p1_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
     ;
     ; GFX11-LABEL: name: test_load_private_p1_align1
@@ -6341,15 +6437,89 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p5) :: (load (p1), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
+    ; UNALIGNED_GFX11-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR6]](s64)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_p1_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p5) :: (load (p1), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
+    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[OR6]](s64)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
     %0:_(p5) = COPY $vgpr0
     %1:_(p1) = G_LOAD %0 :: (load (p1), align 1, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -6494,29 +6664,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR]](s32)
-    ; GFX9-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 2, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](p3)
     ;
     ; GFX10-LABEL: name: test_load_private_p3_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR]](s32)
-    ; GFX10-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 2, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](p3)
     ;
     ; GFX11-LABEL: name: test_load_private_p3_align2
     ; GFX11: liveins: $vgpr0
@@ -6564,15 +6720,29 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](p3)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_p3_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](p3)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
     %0:_(p5) = COPY $vgpr0
     %1:_(p3) = G_LOAD %0 :: (load (p3), align 2, addrspace 5)
     $vgpr0 = COPY %1
@@ -6660,49 +6830,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
-    ; GFX9-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 1, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](p3)
     ;
     ; GFX10-LABEL: name: test_load_private_p3_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
-    ; GFX10-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 1, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](p3)
     ;
     ; GFX11-LABEL: name: test_load_private_p3_align1
     ; GFX11: liveins: $vgpr0
@@ -6770,15 +6906,49 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](p3)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_p3_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](p3)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
     %0:_(p5) = COPY $vgpr0
     %1:_(p3) = G_LOAD %0 :: (load (p3), align 1, addrspace 5)
     $vgpr0 = COPY %1
@@ -6923,29 +7093,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR]](s32)
-    ; GFX9-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 2, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX10-LABEL: name: test_load_private_p5_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR]](s32)
-    ; GFX10-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 2, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX11-LABEL: name: test_load_private_p5_align2
     ; GFX11: liveins: $vgpr0
@@ -6993,15 +7149,29 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](p5)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_p5_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](p5)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
     %0:_(p5) = COPY $vgpr0
     %1:_(p5) = G_LOAD %0 :: (load (p5), align 2, addrspace 5)
     $vgpr0 = COPY %1
@@ -7089,49 +7259,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR2]](s32)
-    ; GFX9-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 1, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX10-LABEL: name: test_load_private_p5_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR2]](s32)
-    ; GFX10-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 1, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](p5)
     ;
     ; GFX11-LABEL: name: test_load_private_p5_align1
     ; GFX11: liveins: $vgpr0
@@ -7199,15 +7335,49 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](p5)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_p5_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(p5) = G_LOAD [[COPY]](p5) :: (load (p5), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](p5)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[INTTOPTR]](p5)
     %0:_(p5) = COPY $vgpr0
     %1:_(p5) = G_LOAD %0 :: (load (p5), align 1, addrspace 5)
     $vgpr0 = COPY %1
@@ -7357,30 +7527,20 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[LSHR]](s32)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LSHR]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v2s8_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[LSHR]](s32)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LSHR]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v2s8_align1
@@ -7437,20 +7597,30 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; UNALIGNED_GFX11-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LSHR]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[LSHR]](s32)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v2s8_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; UNALIGNED_GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LSHR]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[LSHR]](s32)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x s8>) = G_LOAD %0 :: (load (<2 x s8>), align 1, addrspace 5)
@@ -7938,81 +8108,71 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
-    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C1]](s32)
-    ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C3]](s32)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C2]](s32)
+    ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
     ; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; GFX9-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
+    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
     ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C4]]
-    ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C5]](s16)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
+    ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL1]]
     ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
+    ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C3]]
     ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
-    ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; GFX9-NEXT: $vgpr0 = COPY [[OR4]](s32)
+    ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
+    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
+    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
+    ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
+    ; GFX9-NEXT: $vgpr0 = COPY [[OR3]](s32)
     ;
     ; GFX10-LABEL: name: test_load_private_v3s8_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
-    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C1]](s32)
-    ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C3]](s32)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C2]](s32)
+    ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
     ; GFX10-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; GFX10-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
+    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX10-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
     ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C4]]
-    ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C5]](s16)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
+    ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
+    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
+    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL1]]
     ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; GFX10-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
+    ; GFX10-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C3]]
     ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
-    ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; GFX10-NEXT: $vgpr0 = COPY [[OR4]](s32)
+    ; GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
+    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
+    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
+    ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
+    ; GFX10-NEXT: $vgpr0 = COPY [[OR3]](s32)
     ;
     ; GFX11-LABEL: name: test_load_private_v3s8_align1
     ; GFX11: liveins: $vgpr0
@@ -8168,71 +8328,81 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
     ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; UNALIGNED_GFX11-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C2]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C3]](s32)
     ; UNALIGNED_GFX11-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; UNALIGNED_GFX11-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; UNALIGNED_GFX11-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
     ; UNALIGNED_GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
-    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
-    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL1]]
+    ; UNALIGNED_GFX11-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C4]]
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C5]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
     ; UNALIGNED_GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C3]]
+    ; UNALIGNED_GFX11-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
     ; UNALIGNED_GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
-    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
-    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
-    ; UNALIGNED_GFX11-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
-    ; UNALIGNED_GFX11-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[OR3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[OR4]](s32)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v3s8_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
     ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; UNALIGNED_GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C2]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C3]](s32)
     ; UNALIGNED_GFX12-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
-    ; UNALIGNED_GFX12-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; UNALIGNED_GFX12-NEXT: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
     ; UNALIGNED_GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C3]]
-    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C4]](s16)
-    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL1]]
+    ; UNALIGNED_GFX12-NEXT: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C4]]
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C5]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL2]]
     ; UNALIGNED_GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C3]]
+    ; UNALIGNED_GFX12-NEXT: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
     ; UNALIGNED_GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
-    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
-    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
-    ; UNALIGNED_GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
-    ; UNALIGNED_GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[OR3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[OR4]](s32)
     %0:_(p5) = COPY $vgpr0
     %1:_(<3 x s8>) = G_LOAD %0 :: (load (<3 x s8>), align 1, addrspace 5)
     %2:_(s24) = G_BITCAST %1
@@ -8658,136 +8828,34 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v16s8_align16
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX10-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v16s8_align16
@@ -8944,15 +9012,137 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v16s8_align16
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<16 x s8>) = G_LOAD %0 :: (load (<16 x s8>), align 1, addrspace 5)
     %2:_(<4 x s32>) = G_BITCAST %1
@@ -9107,27 +9297,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX10-LABEL: name: test_load_private_v2s16_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX10-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX11-LABEL: name: test_load_private_v2s16_align2
     ; GFX11: liveins: $vgpr0
@@ -9173,15 +9351,27 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v2s16_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x s16>) = G_LOAD %0 :: (load (<2 x s16>), align 2, addrspace 5)
     $vgpr0 = COPY %1
@@ -9278,47 +9468,15 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
+    ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX10-LABEL: name: test_load_private_v2s16_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX10-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
+    ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     ;
     ; GFX11-LABEL: name: test_load_private_v2s16_align1
     ; GFX11: liveins: $vgpr0
@@ -9384,15 +9542,47 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v2s16_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x s16>) = G_LOAD %0 :: (load (<2 x s16>), align 1, addrspace 5)
     $vgpr0 = COPY %1
@@ -9824,27 +10014,26 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
+    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX9-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
-    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
-    ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
-    ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
-    ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC3]](s16)
     ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
     ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -9853,27 +10042,26 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
+    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX10-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
-    ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
-    ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
-    ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
-    ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC3]](s16)
     ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
     ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -10215,41 +10403,26 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
+    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX9-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
-    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
-    ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
-    ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
-    ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX9-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC3]](s16)
     ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
     ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -10258,41 +10431,26 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[LOAD]](<2 x s16>)
+    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; GFX10-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
-    ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
-    ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
-    ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
-    ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC3]](s16)
     ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
     ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -10445,22 +10603,36 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
     ; UNALIGNED_GFX11-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; UNALIGNED_GFX11-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
     ; UNALIGNED_GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; UNALIGNED_GFX11-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
     ; UNALIGNED_GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; UNALIGNED_GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
     ; UNALIGNED_GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
@@ -10474,22 +10646,36 @@ body: |
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
     ; UNALIGNED_GFX12-NEXT: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
     ; UNALIGNED_GFX12-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
     ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
     ; UNALIGNED_GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; UNALIGNED_GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C4]](s32)
     ; UNALIGNED_GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
     ; UNALIGNED_GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
     ; UNALIGNED_GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
@@ -10827,44 +11013,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[PTR_ADD]](p5) :: (load (<2 x s16>) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[LOAD]](<2 x s16>), [[LOAD1]](<2 x s16>)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ;
     ; GFX10-LABEL: name: test_load_private_v4s16_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[PTR_ADD]](p5) :: (load (<2 x s16>) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[LOAD]](<2 x s16>), [[LOAD1]](<2 x s16>)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ;
     ; GFX11-LABEL: name: test_load_private_v4s16_align2
@@ -10929,15 +11093,47 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p5) :: (load (<4 x s16>), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v4s16_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p5) :: (load (<4 x s16>), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<4 x s16>) = G_LOAD %0 :: (load (<4 x s16>), align 2, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -11091,80 +11287,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR3]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[PTR_ADD]](p5) :: (load (<2 x s16>) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[LOAD]](<2 x s16>), [[LOAD1]](<2 x s16>)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ;
     ; GFX10-LABEL: name: test_load_private_v4s16_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load (<2 x s16>), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR3]](s32)
-    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-    ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[PTR_ADD]](p5) :: (load (<2 x s16>) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[LOAD]](<2 x s16>), [[LOAD1]](<2 x s16>)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ;
     ; GFX11-LABEL: name: test_load_private_v4s16_align1
@@ -11265,15 +11403,83 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p5) :: (load (<4 x s16>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; UNALIGNED_GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v4s16_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p5) :: (load (<4 x s16>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; UNALIGNED_GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s16>), [[BUILD_VECTOR1]](<2 x s16>)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<4 x s16>) = G_LOAD %0 :: (load (<4 x s16>), align 1, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -11582,42 +11788,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v2s32_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v2s32_align2
@@ -11680,15 +11866,43 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v2s32_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x s32>) = G_LOAD %0 :: (load (<2 x s32>), align 2, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -11821,78 +12035,22 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v2s32_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v2s32_align1
@@ -11991,15 +12149,79 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v2s32_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x s32>) = G_LOAD %0 :: (load (<2 x s32>), align 1, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
@@ -12174,106 +12396,28 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v3s32_align16
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v3s32_align16
@@ -12400,15 +12544,107 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v3s32_align16
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<3 x s32>) = G_LOAD %0 :: (load (<3 x s32>), align 1, addrspace 5)
     $vgpr0_vgpr1_vgpr2 = COPY %1
@@ -12764,136 +13000,34 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v4s32_align16
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX10-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v4s32_align16
@@ -13050,15 +13184,137 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v4s32_align16
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<4 x s32>) = G_LOAD %0 :: (load (<4 x s32>), align 1, addrspace 5)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -13493,70 +13749,34 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 2, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v4s32_align2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 2, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v4s32_align2
@@ -13647,15 +13867,71 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v4s32_align2
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<4 x s32>) = G_LOAD %0 :: (load (<4 x s32>), align 2, addrspace 5)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -13875,136 +14151,34 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX10-LABEL: name: test_load_private_v4s32_align1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX10-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; GFX11-LABEL: name: test_load_private_v4s32_align1
@@ -14161,15 +14335,137 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v4s32_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p5) :: (load (<4 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<4 x s32>) = G_LOAD %0 :: (load (<4 x s32>), align 1, addrspace 5)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -15262,68 +15558,17 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
     ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
     ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ;
@@ -15331,68 +15576,17 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
+    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
     ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX10-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
     ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
     ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ;
@@ -15552,15 +15746,155 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p5) :: (load (<2 x s64>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
+    ; UNALIGNED_GFX11-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
+    ; UNALIGNED_GFX11-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v2s64_align16
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p5) :: (load (<2 x s64>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[ZEXT]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[OR7]]
+    ; UNALIGNED_GFX12-NEXT: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[OR9]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[OR11]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[OR10]]
+    ; UNALIGNED_GFX12-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[OR12]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[COPY1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[ZEXT1]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR6]](s64), [[OR13]](s64)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x s64>) = G_LOAD %0 :: (load (<2 x s64>), align 1, addrspace 5)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -18178,98 +18512,23 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
-    ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX9-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX9-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p5) :: (load (s8) from unknown-address + 16, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p5) :: (load (s8) from unknown-address + 17, addrspace 5)
-    ; GFX9-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[ZEXTLOAD12]]
-    ; GFX9-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p5) :: (load (s8) from unknown-address + 18, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD17]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p5) :: (load (s8) from unknown-address + 19, addrspace 5)
-    ; GFX9-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
-    ; GFX9-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[OR13]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
-    ; GFX9-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C1]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p5) :: (load (s8) from unknown-address + 20, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p5) :: (load (s8) from unknown-address + 21, addrspace 5)
-    ; GFX9-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD15]]
-    ; GFX9-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p5) :: (load (s8) from unknown-address + 22, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD21]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p5) :: (load (s8) from unknown-address + 23, addrspace 5)
-    ; GFX9-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
-    ; GFX9-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[OR16]], [[C3]](s32)
-    ; GFX9-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
-    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR11]](s32), [[OR14]](s32), [[OR17]](s32)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32)
+    ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 1, addrspace 5)
+    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, align 1, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD3]](s32), [[LOAD4]](s32), [[LOAD5]](s32)
     ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
@@ -18280,98 +18539,23 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
     ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
-    ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
-    ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
-    ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
-    ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
-    ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
-    ; GFX10-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
-    ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
-    ; GFX10-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p5) :: (load (s8) from unknown-address + 16, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p5) :: (load (s8) from unknown-address + 17, addrspace 5)
-    ; GFX10-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[ZEXTLOAD12]]
-    ; GFX10-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p5) :: (load (s8) from unknown-address + 18, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD17]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p5) :: (load (s8) from unknown-address + 19, addrspace 5)
-    ; GFX10-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
-    ; GFX10-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[OR13]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
-    ; GFX10-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C1]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p5) :: (load (s8) from unknown-address + 20, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p5) :: (load (s8) from unknown-address + 21, addrspace 5)
-    ; GFX10-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD15]]
-    ; GFX10-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p5) :: (load (s8) from unknown-address + 22, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD21]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p5) :: (load (s8) from unknown-address + 23, addrspace 5)
-    ; GFX10-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
-    ; GFX10-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[OR16]], [[C3]](s32)
-    ; GFX10-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
-    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR11]](s32), [[OR14]](s32), [[OR17]](s32)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
+    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32)
+    ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 1, addrspace 5)
+    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, align 1, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD3]](s32), [[LOAD4]](s32), [[LOAD5]](s32)
     ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
@@ -18616,12 +18800,99 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
-    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD]](p5) :: (load (<3 x s32>) from unknown-address + 12, align 1, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD1]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p5) :: (load (s8) from unknown-address + 16, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p5) :: (load (s8) from unknown-address + 17, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[ZEXTLOAD12]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p5) :: (load (s8) from unknown-address + 18, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD17]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p5) :: (load (s8) from unknown-address + 19, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[OR13]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p5) :: (load (s8) from unknown-address + 20, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p5) :: (load (s8) from unknown-address + 21, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD15]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p5) :: (load (s8) from unknown-address + 22, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD21]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p5) :: (load (s8) from unknown-address + 23, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
+    ; UNALIGNED_GFX11-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[OR16]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR11]](s32), [[OR14]](s32), [[OR17]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; UNALIGNED_GFX11-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; UNALIGNED_GFX11-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
@@ -18631,12 +18902,99 @@ body: |
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
-    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s8), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD]](p5) :: (load (<3 x s32>) from unknown-address + 12, align 1, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD1]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p5) :: (load (s8) from unknown-address + 1, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s8) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s8) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD8]](p5) :: (load (s8) from unknown-address + 9, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD7]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s8) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s8) from unknown-address + 11, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR7]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD12]](p5) :: (load (s8) from unknown-address + 13, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD10]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[SHL9]], [[ZEXTLOAD9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD13]](p5) :: (load (s8) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD13]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p5) :: (load (s8) from unknown-address + 15, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD15]](p5) :: (load (s8) from unknown-address + 16, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD16:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD16]](p5) :: (load (s8) from unknown-address + 17, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD13]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[SHL12]], [[ZEXTLOAD12]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD17:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD15]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD14:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD17]](p5) :: (load (s8) from unknown-address + 18, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD18:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD17]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD18]](p5) :: (load (s8) from unknown-address + 19, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[OR13]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD11]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD19]](p5) :: (load (s8) from unknown-address + 20, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD16:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD20]](p5) :: (load (s8) from unknown-address + 21, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD16]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[SHL15]], [[ZEXTLOAD15]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD21:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD19]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD17:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD21]](p5) :: (load (s8) from unknown-address + 22, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD22:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD21]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD22]](p5) :: (load (s8) from unknown-address + 23, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
+    ; UNALIGNED_GFX12-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[OR16]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR11]](s32), [[OR14]](s32), [[OR17]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; UNALIGNED_GFX12-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; UNALIGNED_GFX12-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
@@ -18818,49 +19176,23 @@ body: |
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
-    ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
-    ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C2]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s16) from unknown-address + 16, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p5) :: (load (s16) from unknown-address + 18, addrspace 5)
-    ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD4]]
-    ; GFX9-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C3]](s32)
-    ; GFX9-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s16) from unknown-address + 20, addrspace 5)
-    ; GFX9-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s16) from unknown-address + 22, addrspace 5)
-    ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[ZEXTLOAD5]]
-    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR3]](s32), [[OR4]](s32), [[OR5]](s32)
+    ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 2, addrspace 5)
+    ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32)
+    ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 2, addrspace 5)
+    ; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C1]](s32)
+    ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, align 2, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD3]](s32), [[LOAD4]](s32), [[LOAD5]](s32)
     ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
@@ -18871,49 +19203,23 @@ body: |
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
-    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+    ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
-    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
-    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
-    ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
-    ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
-    ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
+    ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 2, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32), [[LOAD2]](s32)
     ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
-    ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
-    ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C2]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s16) from unknown-address + 16, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p5) :: (load (s16) from unknown-address + 18, addrspace 5)
-    ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD4]]
-    ; GFX10-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C3]](s32)
-    ; GFX10-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s16) from unknown-address + 20, addrspace 5)
-    ; GFX10-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s16) from unknown-address + 22, addrspace 5)
-    ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
-    ; GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[ZEXTLOAD5]]
-    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR3]](s32), [[OR4]](s32), [[OR5]](s32)
+    ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 2, addrspace 5)
+    ; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32)
+    ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 2, addrspace 5)
+    ; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD2]], [[C1]](s32)
+    ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, align 2, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LOAD3]](s32), [[LOAD4]](s32), [[LOAD5]](s32)
     ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
@@ -19060,12 +19366,50 @@ body: |
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
-    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; UNALIGNED_GFX11-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD]](p5) :: (load (<3 x s32>) from unknown-address + 12, align 2, addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD1]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX11-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX11-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX11-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; UNALIGNED_GFX11-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C2]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s16) from unknown-address + 16, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p5) :: (load (s16) from unknown-address + 18, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD4]]
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C3]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s16) from unknown-address + 20, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s16) from unknown-address + 22, addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR3]](s32), [[OR4]](s32), [[OR5]](s32)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; UNALIGNED_GFX11-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; UNALIGNED_GFX11-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
     ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
@@ -19075,12 +19419,50 @@ body: |
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p5) :: (load (<3 x s32>), align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD]](<3 x s32>)
-    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load (s16), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; UNALIGNED_GFX12-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[PTR_ADD]](p5) :: (load (<3 x s32>) from unknown-address + 12, align 2, addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[LOAD1]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; UNALIGNED_GFX12-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+    ; UNALIGNED_GFX12-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+    ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s16) from unknown-address + 8, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s16) from unknown-address + 10, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR]](<3 x s32>)
+    ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p5) :: (load (s16) from unknown-address + 12, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s16) from unknown-address + 14, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C2]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s16) from unknown-address + 16, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p5) :: (load (s16) from unknown-address + 18, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD4]]
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD5]], [[C3]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD9]](p5) :: (load (s16) from unknown-address + 20, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p5) = G_PTR_ADD [[PTR_ADD9]], [[C]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p5) :: (load (s16) from unknown-address + 22, addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LOAD5]], [[C1]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[ZEXTLOAD5]]
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR3]](s32), [[OR4]](s32), [[OR5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s96) = G_BITCAST [[BUILD_VECTOR1]](<3 x s32>)
     ; UNALIGNED_GFX12-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY [[BITCAST]](s96)
     ; UNALIGNED_GFX12-NEXT: [[COPY2:%[0-9]+]]:_(s96) = COPY [[BITCAST1]](s96)
     ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
index 96cab200b61cdb1a7c7c78a62e0f3e936bf65d2c..2edcf23df411df37a5cccfb6ed910e29da2e3a2b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -global-isel -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -global-isel -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX10 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -global-isel -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX11 %s
 
 declare i32 @llvm.amdgcn.ballot.i32(i1)
 declare i32 @llvm.ctpop.i32(i32)
@@ -33,7 +33,8 @@ define amdgpu_cs i32 @non_compare(i32 %x) {
 ; CHECK-LABEL: non_compare:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; CHECK-NEXT:    s_and_b32 s0, vcc_lo, exec_lo
 ; CHECK-NEXT:    ; return to shader part epilog
   %trunc = trunc i32 %x to i1
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %trunc)
@@ -89,7 +90,8 @@ define amdgpu_cs i32 @branch_divergent_ballot_ne_zero_non_compare(i32 %v) {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; CHECK-NEXT:    s_cmp_eq_u32 vcc_lo, 0
+; CHECK-NEXT:    s_and_b32 s0, vcc_lo, exec_lo
+; CHECK-NEXT:    s_cmp_eq_u32 s0, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB7_2
 ; CHECK-NEXT:  ; %bb.1: ; %true
 ; CHECK-NEXT:    s_mov_b32 s0, 42
@@ -137,7 +139,8 @@ define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_non_compare(i32 %v) {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; CHECK-NEXT:    s_cmp_lg_u32 vcc_lo, 0
+; CHECK-NEXT:    s_and_b32 s0, vcc_lo, exec_lo
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB9_2
 ; CHECK-NEXT:  ; %bb.1: ; %false
 ; CHECK-NEXT:    s_mov_b32 s0, 33
@@ -419,3 +422,80 @@ true:
 false:
   ret i32 33
 }
+
+; Input that is not constant or direct result of a compare.
+; Tests setting 0 to inactive lanes.
+define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid, i32 %cond) {
+; GFX10-LABEL: non_cst_non_compare_input:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-NEXT:  ; %bb.1: ; %B
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 2, v2
+; GFX10-NEXT:    s_andn2_b32 s0, s0, exec_lo
+; GFX10-NEXT:    ; implicit-def: $vgpr2
+; GFX10-NEXT:    s_and_b32 s2, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:  ; %bb.2: ; %Flow
+; GFX10-NEXT:    s_andn2_saveexec_b32 s1, s1
+; GFX10-NEXT:  ; %bb.3: ; %A
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 1, v2
+; GFX10-NEXT:    s_andn2_b32 s0, s0, exec_lo
+; GFX10-NEXT:    s_and_b32 s2, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:  ; %bb.4: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v2, s0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: non_cst_non_compare_input:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_mov_b32 s1, exec_lo
+; GFX11-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v3
+; GFX11-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX11-NEXT:  ; %bb.1: ; %B
+; GFX11-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 2, v2
+; GFX11-NEXT:    s_and_not1_b32 s0, s0, exec_lo
+; GFX11-NEXT:    ; implicit-def: $vgpr2
+; GFX11-NEXT:    s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-NEXT:    s_or_b32 s0, s0, s2
+; GFX11-NEXT:  ; %bb.2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s1, s1
+; GFX11-NEXT:  ; %bb.3: ; %A
+; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, 1, v2
+; GFX11-NEXT:    s_and_not1_b32 s0, s0, exec_lo
+; GFX11-NEXT:    s_and_b32 s2, exec_lo, vcc_lo
+; GFX11-NEXT:    s_or_b32 s0, s0, s2
+; GFX11-NEXT:  ; %bb.4: ; %exit
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT:    s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+entry:
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %A, label %B
+
+A:
+  %val_A = icmp uge i32 %tid, 1
+  br label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 2
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %phi)
+  store i32 %ballot, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
index a18f843440445c033e1fddf73e473cd587ef1045..0bbb40b8db43ab5446457984248c602362436560 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
@@ -34,7 +34,8 @@ define amdgpu_cs i64 @non_compare(i32 %x) {
 ; CHECK-LABEL: non_compare:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v0
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    s_and_b64 s[0:1], vcc, exec
 ; CHECK-NEXT:    ; return to shader part epilog
   %trunc = trunc i32 %x to i1
   %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %trunc)
@@ -92,7 +93,8 @@ define amdgpu_cs i32 @branch_divergent_ballot_ne_zero_non_compare(i32 %v) {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; CHECK-NEXT:    s_cmp_eq_u64 vcc, 0
+; CHECK-NEXT:    s_and_b64 s[0:1], vcc, exec
+; CHECK-NEXT:    s_cmp_eq_u64 s[0:1], 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB7_2
 ; CHECK-NEXT:  ; %bb.1: ; %true
 ; CHECK-NEXT:    s_mov_b32 s0, 42
@@ -140,7 +142,8 @@ define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_non_compare(i32 %v) {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; CHECK-NEXT:    s_cmp_lg_u64 vcc, 0
+; CHECK-NEXT:    s_and_b64 s[0:1], vcc, exec
+; CHECK-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB9_2
 ; CHECK-NEXT:  ; %bb.1: ; %false
 ; CHECK-NEXT:    s_mov_b32 s0, 33
@@ -422,3 +425,52 @@ true:
 false:
   ret i32 33
 }
+
+; Input that is not constant or direct result of a compare.
+; Tests setting 0 to inactive lanes.
+define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid, i32 %cond) {
+; CHECK-LABEL: non_cst_non_compare_input:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_and_b32 s0, 1, s0
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, s0
+; CHECK-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; CHECK-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; CHECK-NEXT:  ; %bb.1: ; %B
+; CHECK-NEXT:    v_cmp_gt_u32_e32 vcc, 2, v2
+; CHECK-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT:    s_and_b64 s[4:5], exec, vcc
+; CHECK-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr2
+; CHECK-NEXT:  ; %bb.2: ; %Flow
+; CHECK-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; CHECK-NEXT:  ; %bb.3: ; %A
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 1, v2
+; CHECK-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT:    s_and_b64 s[4:5], exec, vcc
+; CHECK-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; CHECK-NEXT:  ; %bb.4: ; %exit
+; CHECK-NEXT:    s_or_b64 exec, exec, s[2:3]
+; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
+; CHECK-NEXT:    s_endpgm
+entry:
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %A, label %B
+
+A:
+  %val_A = icmp uge i32 %tid, 1
+  br label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 2
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %phi)
+  store i64 %ballot, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 81abe91b283f96d9ca87660bd9035cdec260cb1f..0b5706aa45b6930d4360b2a12bf520b63e003cd3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -1184,73 +1184,74 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
 ; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
-; GISEL-NEXT:    v_trunc_f32_e32 v7, v5
-; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v7
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], s6, v6, 0
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[5:6]
-; GISEL-NEXT:    v_mul_lo_u32 v5, v7, v4
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9]
-; GISEL-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v4, v7, v4
-; GISEL-NEXT:    v_mul_lo_u32 v10, v6, v8
-; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v8
-; GISEL-NEXT:    v_mul_hi_u32 v12, v6, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v6, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v5, 0
+; GISEL-NEXT:    v_mov_b32_e32 v4, v9
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v7, v[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v4, v7, v8
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v8
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s7, v5, v[9:10]
 ; GISEL-NEXT:    v_mul_hi_u32 v8, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
 ; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v10, v4
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v12
 ; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v5, v4
 ; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v11, 0
-; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, v7, v6, vcc
 ; GISEL-NEXT:    v_mov_b32_e32 v4, v9
-; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v5, v[4:5]
-; GISEL-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v12, v[4:5]
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
 ; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s7, v11, v[9:10]
-; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
-; GISEL-NEXT:    v_xor_b32_e32 v10, v0, v4
-; GISEL-NEXT:    v_mul_lo_u32 v0, v5, v8
-; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v9
-; GISEL-NEXT:    v_xor_b32_e32 v13, v1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v10, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v0, v12, v8
+; GISEL-NEXT:    v_mul_lo_u32 v4, v11, v9
+; GISEL-NEXT:    v_xor_b32_e32 v13, v1, v6
 ; GISEL-NEXT:    v_mul_hi_u32 v1, v11, v8
-; GISEL-NEXT:    v_mul_hi_u32 v8, v5, v8
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v8, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v1, v5, v9
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT:    v_mul_hi_u32 v12, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v1, v12, v9
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GISEL-NEXT:    v_mul_hi_u32 v4, v11, v9
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v8
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_mul_hi_u32 v8, v12, v9
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v9, v1
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v5, v1, vcc
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v12, v1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v8, v13, v0
 ; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v1
 ; GISEL-NEXT:    v_mul_hi_u32 v11, v10, v0
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x12d8fb
+; GISEL-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
 ; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
@@ -1265,40 +1266,39 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
 ; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v0, v8
 ; GISEL-NEXT:    v_mul_hi_u32 v12, v13, v1
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v11, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v11, 0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
 ; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v8
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v5, v12, v[1:2]
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v12, v[1:2]
 ; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], 0, v11, v[8:9]
-; GISEL-NEXT:    s_sub_u32 s6, 0, 0x12d8fb
-; GISEL-NEXT:    s_subb_u32 s7, 0, 0
 ; GISEL-NEXT:    v_subb_u32_e64 v1, s[4:5], v13, v8, vcc
 ; GISEL-NEXT:    v_sub_i32_e64 v8, s[4:5], v13, v8
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
 ; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
 ; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v8, vcc
-; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v5
+; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, v0, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v10, -1, v9, s[4:5]
 ; GISEL-NEXT:    v_subbrev_u32_e32 v9, vcc, 0, v1, vcc
+; GISEL-NEXT:    s_sub_u32 s6, 0, 0x12d8fb
 ; GISEL-NEXT:    v_add_i32_e32 v13, vcc, 1, v11
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v6, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0
 ; GISEL-NEXT:    v_addc_u32_e32 v14, vcc, 0, v12, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
 ; GISEL-NEXT:    v_cndmask_b32_e32 v15, -1, v8, vcc
 ; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[1:2]
+; GISEL-NEXT:    s_subb_u32 s7, 0, 0
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 1, v13
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9]
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v5, v[8:9]
 ; GISEL-NEXT:    v_addc_u32_e32 v16, vcc, 0, v14, vcc
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v15
 ; GISEL-NEXT:    v_cndmask_b32_e32 v9, v13, v1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v1, v7, v0
-; GISEL-NEXT:    v_mul_lo_u32 v13, v6, v8
-; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v0
+; GISEL-NEXT:    v_mul_lo_u32 v13, v5, v8
+; GISEL-NEXT:    v_mul_hi_u32 v15, v5, v0
 ; GISEL-NEXT:    v_cndmask_b32_e32 v14, v14, v16, vcc
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v7, v0
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v13
@@ -1307,7 +1307,7 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v15, v7, v8
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
-; GISEL-NEXT:    v_mul_hi_u32 v13, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v5, v8
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v15, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
@@ -1318,95 +1318,93 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v6, v0
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v0
 ; GISEL-NEXT:    v_addc_u32_e32 v13, vcc, v7, v1, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
-; GISEL-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], s6, v13, v[1:2]
-; GISEL-NEXT:    v_xor_b32_e32 v1, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v7, v11, v9, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v10, v7, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], s6, v13, v[1:2]
+; GISEL-NEXT:    v_cndmask_b32_e32 v9, v12, v14, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v1, v9, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], s7, v5, v[7:8]
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], s7, v8, v[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v10, v12, v14, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
 ; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v9, vcc
-; GISEL-NEXT:    v_xor_b32_e32 v11, v2, v9
+; GISEL-NEXT:    v_xor_b32_e32 v8, v2, v9
 ; GISEL-NEXT:    v_mul_lo_u32 v2, v13, v0
-; GISEL-NEXT:    v_mul_lo_u32 v7, v8, v6
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v7
 ; GISEL-NEXT:    v_xor_b32_e32 v12, v3, v9
-; GISEL-NEXT:    v_mul_hi_u32 v3, v8, v0
+; GISEL-NEXT:    v_mul_hi_u32 v3, v5, v0
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v3, v13, v6
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v8, v6
+; GISEL-NEXT:    v_mul_lo_u32 v3, v13, v7
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v11, v2
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v7
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT:    v_mul_hi_u32 v6, v13, v6
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v11
+; GISEL-NEXT:    v_mul_hi_u32 v7, v13, v7
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v5, v0
 ; GISEL-NEXT:    v_addc_u32_e32 v2, vcc, v13, v2, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v3, v12, v0
-; GISEL-NEXT:    v_mul_lo_u32 v6, v11, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v11, v0
-; GISEL-NEXT:    v_mul_hi_u32 v0, v12, v0
-; GISEL-NEXT:    v_xor_b32_e32 v8, v10, v4
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v12, v3
+; GISEL-NEXT:    v_mul_lo_u32 v7, v8, v2
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v10, v6
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v6, v8, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v6, v12, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v12, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v8, v2
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v7, v12, v2
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GISEL-NEXT:    v_mul_hi_u32 v6, v11, v2
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v0, v3
-; GISEL-NEXT:    v_mul_hi_u32 v7, v12, v2
-; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v10, 0
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v7, v0
-; GISEL-NEXT:    v_mov_b32_e32 v0, v3
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v13, v[0:1]
-; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v8, v4, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], 0, v10, v[6:7]
-; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v11, v2
-; GISEL-NEXT:    v_subb_u32_e64 v4, s[4:5], v12, v3, vcc
-; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v12, v3
-; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v5
-; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v3, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, v12, v2
+; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v7, 0
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v5
+; GISEL-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v4, v10, v[3:4]
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT:    v_subb_u32_e64 v3, s[4:5], v12, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v5, s[4:5], v12, v5
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v4
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v4
-; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, -1, v6, s[4:5]
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v10
-; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, 0, v13, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v5
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v3, -1, v6, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, 1, v7
+; GISEL-NEXT:    v_addc_u32_e32 v8, vcc, 0, v10, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
 ; GISEL-NEXT:    v_cndmask_b32_e32 v2, -1, v2, vcc
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v6
-; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, 0, v8, vcc
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v7, v5, vcc
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v10, v2, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v6, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v5, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v10, v4, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v9
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v9
 ; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index cfac0c2fa56aafb6c68b1bc6309a07043dc6dc4d..3ed864d463ee9c2ee8dd555033073b3de6eb9aad 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -1112,73 +1112,74 @@ define <2 x i64> @v_srem_v2i64_pow2k_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
 ; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
-; GISEL-NEXT:    v_trunc_f32_e32 v7, v5
-; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v7
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], s6, v6, 0
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[5:6]
-; GISEL-NEXT:    v_mul_lo_u32 v5, v7, v4
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9]
-; GISEL-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v4, v7, v4
-; GISEL-NEXT:    v_mul_lo_u32 v10, v6, v8
-; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v8
-; GISEL-NEXT:    v_mul_hi_u32 v12, v6, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v6, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v5, 0
+; GISEL-NEXT:    v_mov_b32_e32 v4, v9
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v7, v[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v4, v7, v8
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v8
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s7, v5, v[9:10]
 ; GISEL-NEXT:    v_mul_hi_u32 v8, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
 ; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v10, v4
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v12
 ; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v5, v4
 ; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v11, 0
-; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, v7, v6, vcc
 ; GISEL-NEXT:    v_mov_b32_e32 v4, v9
-; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v5, v[4:5]
-; GISEL-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v12, v[4:5]
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
 ; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s7, v11, v[9:10]
-; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
-; GISEL-NEXT:    v_xor_b32_e32 v10, v0, v4
-; GISEL-NEXT:    v_mul_lo_u32 v0, v5, v8
-; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v9
-; GISEL-NEXT:    v_xor_b32_e32 v13, v1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v10, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v0, v12, v8
+; GISEL-NEXT:    v_mul_lo_u32 v4, v11, v9
+; GISEL-NEXT:    v_xor_b32_e32 v13, v1, v6
 ; GISEL-NEXT:    v_mul_hi_u32 v1, v11, v8
-; GISEL-NEXT:    v_mul_hi_u32 v8, v5, v8
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v8, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v1, v5, v9
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT:    v_mul_hi_u32 v12, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v1, v12, v9
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GISEL-NEXT:    v_mul_hi_u32 v4, v11, v9
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v8
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_mul_hi_u32 v8, v12, v9
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v9, v1
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v5, v1, vcc
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v12, v1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v8, v13, v0
 ; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v1
 ; GISEL-NEXT:    v_mul_hi_u32 v11, v10, v0
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x1000
+; GISEL-NEXT:    v_mov_b32_e32 v4, 0x1000
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
 ; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
@@ -1191,40 +1192,39 @@ define <2 x i64> @v_srem_v2i64_pow2k_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v9
 ; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v0, v8
-; GISEL-NEXT:    v_mul_hi_u32 v12, v13, v1
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v11, 0
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v13, v1
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v5, v8, v[1:2]
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v8, v[1:2]
 ; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, v10, v0
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], 0, v11, v[8:9]
-; GISEL-NEXT:    s_sub_u32 s6, 0, 0x1000
-; GISEL-NEXT:    s_subb_u32 s7, 0, 0
 ; GISEL-NEXT:    v_subb_u32_e64 v11, s[4:5], v13, v8, vcc
 ; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v13, v8
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v4
 ; GISEL-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, s[4:5]
 ; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, v10, v5
+; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, v10, v4
+; GISEL-NEXT:    s_sub_u32 s6, 0, 0x1000
 ; GISEL-NEXT:    v_cndmask_b32_e64 v12, -1, v1, s[4:5]
 ; GISEL-NEXT:    v_subbrev_u32_e32 v14, vcc, 0, v0, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v6, 0
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v13, v5
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v13, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v14
 ; GISEL-NEXT:    v_cndmask_b32_e32 v15, -1, v8, vcc
 ; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[1:2]
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v13, v5
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9]
+; GISEL-NEXT:    s_subb_u32 s7, 0, 0
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v13, v4
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v5, v[8:9]
 ; GISEL-NEXT:    v_subbrev_u32_e32 v16, vcc, 0, v14, vcc
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v15
 ; GISEL-NEXT:    v_cndmask_b32_e32 v9, v13, v1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v1, v7, v0
-; GISEL-NEXT:    v_mul_lo_u32 v13, v6, v8
-; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v0
+; GISEL-NEXT:    v_mul_lo_u32 v13, v5, v8
+; GISEL-NEXT:    v_mul_hi_u32 v15, v5, v0
 ; GISEL-NEXT:    v_cndmask_b32_e32 v14, v14, v16, vcc
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v7, v0
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v13
@@ -1233,7 +1233,7 @@ define <2 x i64> @v_srem_v2i64_pow2k_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v15, v7, v8
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
-; GISEL-NEXT:    v_mul_hi_u32 v13, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v5, v8
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v15, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
@@ -1244,93 +1244,91 @@ define <2 x i64> @v_srem_v2i64_pow2k_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v6, v0
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v0
 ; GISEL-NEXT:    v_addc_u32_e32 v13, vcc, v7, v1, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
-; GISEL-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], s6, v13, v[1:2]
-; GISEL-NEXT:    v_xor_b32_e32 v1, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v7, v10, v9, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v10, v7, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], s6, v13, v[1:2]
+; GISEL-NEXT:    v_cndmask_b32_e32 v9, v11, v14, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v1, v9, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], s7, v5, v[7:8]
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], s7, v8, v[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
 ; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v9, vcc
-; GISEL-NEXT:    v_xor_b32_e32 v11, v2, v9
+; GISEL-NEXT:    v_xor_b32_e32 v8, v2, v9
 ; GISEL-NEXT:    v_mul_lo_u32 v2, v13, v0
-; GISEL-NEXT:    v_mul_lo_u32 v7, v8, v6
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v7
 ; GISEL-NEXT:    v_xor_b32_e32 v12, v3, v9
-; GISEL-NEXT:    v_mul_hi_u32 v3, v8, v0
+; GISEL-NEXT:    v_mul_hi_u32 v3, v5, v0
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v3, v13, v6
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v8, v6
+; GISEL-NEXT:    v_mul_lo_u32 v3, v13, v7
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v11, v2
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v7
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT:    v_mul_hi_u32 v6, v13, v6
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v11
+; GISEL-NEXT:    v_mul_hi_u32 v7, v13, v7
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v5, v0
 ; GISEL-NEXT:    v_addc_u32_e32 v2, vcc, v13, v2, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v3, v12, v0
-; GISEL-NEXT:    v_mul_lo_u32 v6, v11, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v11, v0
-; GISEL-NEXT:    v_mul_hi_u32 v0, v12, v0
-; GISEL-NEXT:    v_xor_b32_e32 v8, v10, v4
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v12, v3
+; GISEL-NEXT:    v_mul_lo_u32 v7, v8, v2
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v10, v6
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v6, v8, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v6, v12, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v12, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v8, v2
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v7, v12, v2
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GISEL-NEXT:    v_mul_hi_u32 v6, v11, v2
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v0, v3
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GISEL-NEXT:    v_mul_hi_u32 v7, v12, v2
-; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v10, 0
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v0
-; GISEL-NEXT:    v_mov_b32_e32 v0, v3
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v6, v[0:1]
-; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v8, v4, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], 0, v10, v[6:7]
-; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v11, v2
-; GISEL-NEXT:    v_subb_u32_e64 v4, s[4:5], v12, v3, vcc
-; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v12, v3
-; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v2, v5
-; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v5
+; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v4, v5, v[3:4]
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT:    v_subb_u32_e64 v3, s[4:5], v12, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v5, s[4:5], v12, v5
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v2, v4
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v5
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v4
 ; GISEL-NEXT:    v_cndmask_b32_e32 v8, -1, v8, vcc
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v7, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v4
-; GISEL-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
+; GISEL-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v5, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, -1, v6, s[4:5]
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v10, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v10, vcc
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v9
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v9
 ; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
@@ -1707,73 +1705,74 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
 ; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
-; GISEL-NEXT:    v_trunc_f32_e32 v7, v5
-; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v7
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v4
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], s6, v6, 0
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[5:6]
-; GISEL-NEXT:    v_mul_lo_u32 v5, v7, v4
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9]
-; GISEL-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v4, v7, v4
-; GISEL-NEXT:    v_mul_lo_u32 v10, v6, v8
-; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v8
-; GISEL-NEXT:    v_mul_hi_u32 v12, v6, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v6, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v5, 0
+; GISEL-NEXT:    v_mov_b32_e32 v4, v9
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v7, v[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v4, v7, v8
+; GISEL-NEXT:    v_mul_hi_u32 v6, v5, v8
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s7, v5, v[9:10]
 ; GISEL-NEXT:    v_mul_hi_u32 v8, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v10, v5, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, v7, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
 ; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v10, v4
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v12
 ; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v5, v4
 ; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v11, 0
-; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, v7, v6, vcc
 ; GISEL-NEXT:    v_mov_b32_e32 v4, v9
-; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v5, v[4:5]
-; GISEL-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s6, v12, v[4:5]
+; GISEL-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
 ; GISEL-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], s7, v11, v[9:10]
-; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
-; GISEL-NEXT:    v_xor_b32_e32 v10, v0, v4
-; GISEL-NEXT:    v_mul_lo_u32 v0, v5, v8
-; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v9
-; GISEL-NEXT:    v_xor_b32_e32 v13, v1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v10, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v0, v12, v8
+; GISEL-NEXT:    v_mul_lo_u32 v4, v11, v9
+; GISEL-NEXT:    v_xor_b32_e32 v13, v1, v6
 ; GISEL-NEXT:    v_mul_hi_u32 v1, v11, v8
-; GISEL-NEXT:    v_mul_hi_u32 v8, v5, v8
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v8, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v1, v5, v9
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT:    v_mul_hi_u32 v12, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v1, v12, v9
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GISEL-NEXT:    v_mul_hi_u32 v4, v11, v9
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v8
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
-; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT:    v_mul_hi_u32 v8, v12, v9
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v9, v1
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v5, v1, vcc
+; GISEL-NEXT:    v_addc_u32_e32 v1, vcc, v12, v1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v8, v13, v0
 ; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v1
 ; GISEL-NEXT:    v_mul_hi_u32 v11, v10, v0
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x12d8fb
+; GISEL-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
 ; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
@@ -1786,40 +1785,39 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v9
 ; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v0, v8
-; GISEL-NEXT:    v_mul_hi_u32 v12, v13, v1
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v11, 0
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v13, v1
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v5, v8, v[1:2]
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v8, v[1:2]
 ; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, v10, v0
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], 0, v11, v[8:9]
-; GISEL-NEXT:    s_sub_u32 s6, 0, 0x12d8fb
-; GISEL-NEXT:    s_subb_u32 s7, 0, 0
 ; GISEL-NEXT:    v_subb_u32_e64 v11, s[4:5], v13, v8, vcc
 ; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v13, v8
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v4
 ; GISEL-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, -1, s[4:5]
 ; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, v10, v5
+; GISEL-NEXT:    v_sub_i32_e32 v13, vcc, v10, v4
+; GISEL-NEXT:    s_sub_u32 s6, 0, 0x12d8fb
 ; GISEL-NEXT:    v_cndmask_b32_e64 v12, -1, v1, s[4:5]
 ; GISEL-NEXT:    v_subbrev_u32_e32 v14, vcc, 0, v0, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v6, 0
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v13, v5
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v13, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
 ; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v14
 ; GISEL-NEXT:    v_cndmask_b32_e32 v15, -1, v8, vcc
 ; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s6, v7, v[1:2]
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v13, v5
-; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v6, v[8:9]
+; GISEL-NEXT:    s_subb_u32 s7, 0, 0
+; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v13, v4
+; GISEL-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], s7, v5, v[8:9]
 ; GISEL-NEXT:    v_subbrev_u32_e32 v16, vcc, 0, v14, vcc
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v15
 ; GISEL-NEXT:    v_cndmask_b32_e32 v9, v13, v1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v1, v7, v0
-; GISEL-NEXT:    v_mul_lo_u32 v13, v6, v8
-; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v0
+; GISEL-NEXT:    v_mul_lo_u32 v13, v5, v8
+; GISEL-NEXT:    v_mul_hi_u32 v15, v5, v0
 ; GISEL-NEXT:    v_cndmask_b32_e32 v14, v14, v16, vcc
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v7, v0
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v13
@@ -1828,7 +1826,7 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GISEL-NEXT:    v_mul_lo_u32 v15, v7, v8
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
-; GISEL-NEXT:    v_mul_hi_u32 v13, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v5, v8
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v15, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
@@ -1839,93 +1837,91 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
 ; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v8, v1
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v6, v0
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v0
 ; GISEL-NEXT:    v_addc_u32_e32 v13, vcc, v7, v1, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0
+; GISEL-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
-; GISEL-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], s6, v13, v[1:2]
-; GISEL-NEXT:    v_xor_b32_e32 v1, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v7, v10, v9, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v10, v7, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], s6, v13, v[1:2]
+; GISEL-NEXT:    v_cndmask_b32_e32 v9, v11, v14, vcc
+; GISEL-NEXT:    v_xor_b32_e32 v1, v9, v6
+; GISEL-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], s7, v5, v[7:8]
 ; GISEL-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], s7, v8, v[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e32 v10, v11, v14, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
 ; GISEL-NEXT:    v_addc_u32_e32 v3, vcc, v3, v9, vcc
-; GISEL-NEXT:    v_xor_b32_e32 v11, v2, v9
+; GISEL-NEXT:    v_xor_b32_e32 v8, v2, v9
 ; GISEL-NEXT:    v_mul_lo_u32 v2, v13, v0
-; GISEL-NEXT:    v_mul_lo_u32 v7, v8, v6
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v7
 ; GISEL-NEXT:    v_xor_b32_e32 v12, v3, v9
-; GISEL-NEXT:    v_mul_hi_u32 v3, v8, v0
+; GISEL-NEXT:    v_mul_hi_u32 v3, v5, v0
 ; GISEL-NEXT:    v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v3, v13, v6
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v8, v6
+; GISEL-NEXT:    v_mul_lo_u32 v3, v13, v7
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v11, v2
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v7
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT:    v_mul_hi_u32 v6, v13, v6
+; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v11
+; GISEL-NEXT:    v_mul_hi_u32 v7, v13, v7
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v5, v0
 ; GISEL-NEXT:    v_addc_u32_e32 v2, vcc, v13, v2, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v3, v12, v0
-; GISEL-NEXT:    v_mul_lo_u32 v6, v11, v2
-; GISEL-NEXT:    v_mul_hi_u32 v7, v11, v0
-; GISEL-NEXT:    v_mul_hi_u32 v0, v12, v0
-; GISEL-NEXT:    v_xor_b32_e32 v8, v10, v4
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v5, v12, v3
+; GISEL-NEXT:    v_mul_lo_u32 v7, v8, v2
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v10, v6
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v6, v8, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v6, v12, v2
+; GISEL-NEXT:    v_mul_hi_u32 v3, v12, v3
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_mul_hi_u32 v7, v8, v2
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v7, v12, v2
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GISEL-NEXT:    v_mul_hi_u32 v6, v11, v2
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
 ; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v0, v3
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GISEL-NEXT:    v_mul_hi_u32 v7, v12, v2
-; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v10, 0
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v7, v0
-; GISEL-NEXT:    v_mov_b32_e32 v0, v3
-; GISEL-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v6, v[0:1]
-; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v1, v4
-; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v8, v4, vcc
-; GISEL-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], 0, v10, v[6:7]
-; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v11, v2
-; GISEL-NEXT:    v_subb_u32_e64 v4, s[4:5], v12, v3, vcc
-; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v12, v3
-; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v2, v5
-; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v5
+; GISEL-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v4, v5, v[3:4]
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v8, v2
+; GISEL-NEXT:    v_subb_u32_e64 v3, s[4:5], v12, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v5, s[4:5], v12, v5
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v7, vcc, v2, v4
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
-; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v5
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v4
 ; GISEL-NEXT:    v_cndmask_b32_e32 v8, -1, v8, vcc
-; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v7, v4
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v4
-; GISEL-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
+; GISEL-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v5, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e64 v6, -1, v6, s[4:5]
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
-; GISEL-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v10, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v10, vcc
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
-; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
 ; GISEL-NEXT:    v_xor_b32_e32 v2, v2, v9
 ; GISEL-NEXT:    v_xor_b32_e32 v3, v3, v9
 ; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v9
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
index c293891140008d09bd0b36cd1c2884d40d03374a..aa38f43368694d0e2af1e4cca9f0cbab85c5b7af 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
@@ -292,7 +292,6 @@ define void @divergent_br_profitable(i32 noundef inreg %value, ptr addrspace(8)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX9-NEXT:  ; %bb.1: ; %if.then
 ; GFX9-NEXT:    s_mov_b32 s11, s18
 ; GFX9-NEXT:    s_mov_b32 s10, s17
@@ -301,7 +300,7 @@ define void @divergent_br_profitable(i32 noundef inreg %value, ptr addrspace(8)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s19
 ; GFX9-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen
-; GFX9-NEXT:  .LBB5_2: ; %if.end
+; GFX9-NEXT:  ; %bb.2: ; %if.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -311,7 +310,6 @@ define void @divergent_br_profitable(i32 noundef inreg %value, ptr addrspace(8)
 ; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0, v0
 ; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX1010-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX1010-NEXT:  ; %bb.1: ; %if.then
 ; GFX1010-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX1010-NEXT:    v_mov_b32_e32 v1, s19
@@ -320,7 +318,7 @@ define void @divergent_br_profitable(i32 noundef inreg %value, ptr addrspace(8)
 ; GFX1010-NEXT:    s_mov_b32 s9, s16
 ; GFX1010-NEXT:    s_mov_b32 s8, s7
 ; GFX1010-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen
-; GFX1010-NEXT:  .LBB5_2: ; %if.end
+; GFX1010-NEXT:  ; %bb.2: ; %if.end
 ; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -331,7 +329,6 @@ define void @divergent_br_profitable(i32 noundef inreg %value, ptr addrspace(8)
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
 ; GFX1030-NEXT:    v_cmpx_lt_i32_e32 0, v0
-; GFX1030-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX1030-NEXT:  ; %bb.1: ; %if.then
 ; GFX1030-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX1030-NEXT:    v_mov_b32_e32 v1, s19
@@ -340,7 +337,7 @@ define void @divergent_br_profitable(i32 noundef inreg %value, ptr addrspace(8)
 ; GFX1030-NEXT:    s_mov_b32 s9, s16
 ; GFX1030-NEXT:    s_mov_b32 s8, s7
 ; GFX1030-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen
-; GFX1030-NEXT:  .LBB5_2: ; %if.end
+; GFX1030-NEXT:  ; %bb.2: ; %if.end
 ; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll b/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll
index 6efc9f2ae77b8243a1778f2debe887575b73e279..d23d7a7c8e0c803493ff932098f466ba2b651ce1 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll
@@ -17,13 +17,12 @@ define amdgpu_ps void @ham(float %arg, float %arg1) #0 {
 ; GCN-NEXT:    v_cmp_lt_f32_e64 s[0:1], 0, v1
 ; GCN-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
 ; GCN-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; GCN-NEXT:    s_cbranch_execz .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %bb4
 ; GCN-NEXT:    v_mov_b32_e32 v0, 4
 ; GCN-NEXT:    s_mov_b32 m0, -1
 ; GCN-NEXT:    ds_write_b32 v0, v0
 ; GCN-NEXT:    ; divergent unreachable
-; GCN-NEXT:  .LBB0_2: ; %UnifiedReturnBlock
+; GCN-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
 ; GCN-NEXT:    s_endpgm
 bb:
   %tmp = fcmp ogt float %arg, 0.000000e+00
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index ea10547da6ab7f1d5f570d4b94aabc14c8e90349..3fc5d0d4b279eb1be93d7e645a2466ea3e016d7a 100644
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -475,8 +475,14 @@ define amdgpu_kernel void @vload2_private(ptr addrspace(1) nocapture readonly %i
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    scratch_store_short off, v0, s0 offset:4
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; FLATSCR-NEXT:    scratch_load_dword v0, off, s0
-; FLATSCR-NEXT:    scratch_load_dword v1, off, s0 offset:2
+; FLATSCR-NEXT:    scratch_load_ushort v0, off, s0 offset:2
+; FLATSCR-NEXT:    scratch_load_ushort v3, off, s0
+; FLATSCR-NEXT:    s_waitcnt vmcnt(1)
+; FLATSCR-NEXT:    v_mov_b32_e32 v1, v0
+; FLATSCR-NEXT:    scratch_load_short_d16_hi v1, off, s0 offset:4
+; FLATSCR-NEXT:    s_mov_b32 s0, 0x5040100
+; FLATSCR-NEXT:    s_waitcnt vmcnt(1)
+; FLATSCR-NEXT:    v_perm_b32 v0, v0, v3, s0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; FLATSCR-NEXT:    s_endpgm
@@ -537,8 +543,13 @@ define amdgpu_kernel void @vload2_private(ptr addrspace(1) nocapture readonly %i
 ; FLATSCR_GFX10-NEXT:    scratch_store_short off, v0, s0 offset:4
 ; FLATSCR_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; FLATSCR_GFX10-NEXT:    s_clause 0x1
-; FLATSCR_GFX10-NEXT:    scratch_load_dword v0, off, s0
-; FLATSCR_GFX10-NEXT:    scratch_load_dword v1, off, s0 offset:2
+; FLATSCR_GFX10-NEXT:    scratch_load_ushort v0, off, s0 offset:2
+; FLATSCR_GFX10-NEXT:    scratch_load_ushort v3, off, s0
+; FLATSCR_GFX10-NEXT:    s_waitcnt vmcnt(1)
+; FLATSCR_GFX10-NEXT:    v_mov_b32_e32 v1, v0
+; FLATSCR_GFX10-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR_GFX10-NEXT:    v_perm_b32 v0, v0, v3, 0x5040100
+; FLATSCR_GFX10-NEXT:    scratch_load_short_d16_hi v1, off, s0 offset:4
 ; FLATSCR_GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR_GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; FLATSCR_GFX10-NEXT:    s_endpgm
@@ -561,8 +572,13 @@ define amdgpu_kernel void @vload2_private(ptr addrspace(1) nocapture readonly %i
 ; GFX11-NEXT:    scratch_store_b16 off, v0, off offset:4 dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    scratch_load_b32 v0, off, off
-; GFX11-NEXT:    scratch_load_b32 v1, off, off offset:2
+; GFX11-NEXT:    scratch_load_u16 v0, off, off offset:2
+; GFX11-NEXT:    scratch_load_u16 v3, off, off
+; GFX11-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NEXT:    v_mov_b32_e32 v1, v0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_perm_b32 v0, v0, v3, 0x5040100
+; GFX11-NEXT:    scratch_load_d16_hi_b16 v1, off, off offset:4
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX11-NEXT:    s_nop 0
diff --git a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
index 45324392aacde2d63f63589d6c556d30bada02de..bafbc9486a1c213e16f01ec470bbf5bb41a9c6b4 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
@@ -1657,7 +1657,7 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "denormal-fp-math-f32"="ieee.ieee" }
+attributes #2 = { nounwind "denormal-fp-math-f32"="ieee,ieee" }
 attributes #3 = { nounwind "denormal-fp-math-f32"="ieee,ieee" "denormal-fp-math"="preserve-sign,preserve-sign" }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/AMDGPU/debug-value2.ll b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
index 1d4c11de4076cba0cd3478b9225c834ebfefabbd..b09d540dd6d7b50718992259c737735c913670ff 100644
--- a/llvm/test/CodeGen/AMDGPU/debug-value2.ll
+++ b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
@@ -12,11 +12,11 @@ define <4 x float> @Scene_transformT(i32 %subshapeIdx, <4 x float> %v, float %ti
 entry:
   ; CHECK: v_mov_b32_e32 v[[COPIED_ARG_PIECE:[0-9]+]], v9
 
-  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr6
+  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr[[COPIED_ARG_PIECE]]
+  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr8
   ; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr7
+  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gScene <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr6
   call void @llvm.dbg.value(metadata ptr addrspace(1) %gScene, metadata !120, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !154
-  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr8
-  ; CHECK: ;DEBUG_VALUE: Scene_transformT:gSceneOffsets <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr[[COPIED_ARG_PIECE]]
   call void @llvm.dbg.value(metadata ptr addrspace(1) %gSceneOffsets, metadata !121, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !155
   %call = tail call ptr addrspace(1) @Scene_getSubShapeData(i32 %subshapeIdx, ptr addrspace(1) %gScene, ptr addrspace(1) %gSceneOffsets)
   %m_linearMotion = getelementptr inbounds %struct.ShapeData, ptr addrspace(1) %call, i64 0, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/else.ll b/llvm/test/CodeGen/AMDGPU/else.ll
index 655c5cd184a1eddab31df1fdb189cd69b70c63bc..d3d4b860f9ac7fa4e2ce967802fd88bd81d47108 100644
--- a/llvm/test/CodeGen/AMDGPU/else.ll
+++ b/llvm/test/CodeGen/AMDGPU/else.ll
@@ -30,7 +30,6 @@ end:
 ; CHECK-NEXT: s_and_b64 exec, exec, [[INIT_EXEC]]
 ; CHECK-NEXT: s_and_b64 [[AND_INIT:s\[[0-9]+:[0-9]+\]]], exec, [[DST]]
 ; CHECK-NEXT: s_xor_b64 exec, exec, [[AND_INIT]]
-; CHECK-NEXT: s_cbranch_execz
 define amdgpu_ps void @else_execfix_leave_wqm(i32 %z, float %v) #0 {
 main_body:
   %cc = icmp sgt i32 %z, 5
diff --git a/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
index d0fd6685df3d739b81316a5a6aa699d8d7458d85..cca70005b4cdc1c05fb0f190d4cd1777c28c9602 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature
 ; RUN: opt -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s
-; REQUIRES: amdgpu-registered-target
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
 target triple = "amdgcn-amd-amdhsa"
 
diff --git a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
index 0ad53083d0ff3f32285c61aad34abe2dc08f6044..12593e3760fd3ed744f4dcdbb6c7d3e7e66049d4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
@@ -123,10 +123,8 @@ define amdgpu_kernel void @zextload_flat_i16(ptr addrspace(1) noalias %out, ptr
 }
 
 ; GCN-LABEL: flat_scratch_unaligned_load:
-; GCN: flat_load_{{ubyte|u8}}
-; GCN: flat_load_{{ubyte|u8}}
-; GCN: flat_load_{{ubyte|u8}}
-; GCN: flat_load_{{ubyte|u8}}
+; GFX9: flat_load_dword
+; GFX10PLUS: flat_load_{{dword|b32}}
 define amdgpu_kernel void @flat_scratch_unaligned_load() {
   %scratch = alloca i32, addrspace(5)
   %fptr = addrspacecast ptr addrspace(5) %scratch to ptr
@@ -136,10 +134,8 @@ define amdgpu_kernel void @flat_scratch_unaligned_load() {
 }
 
 ; GCN-LABEL: flat_scratch_unaligned_store:
-; GCN: flat_store_{{byte|b8}}
-; GCN: flat_store_{{byte|b8}}
-; GCN: flat_store_{{byte|b8}}
-; GCN: flat_store_{{byte|b8}}
+; GFX9: flat_store_dword
+; GFX10PLUS: flat_store_{{dword|b32}}
 define amdgpu_kernel void @flat_scratch_unaligned_store() {
   %scratch = alloca i32, addrspace(5)
   %fptr = addrspacecast ptr addrspace(5) %scratch to ptr
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 667a3f398c08a24f9a10bf789b95818d4b885731..6e8e6c0721789555721fe26b62765ac93482a9fb 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -1578,6 +1578,7 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; SDAG-NEXT:  .LBB6_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[12:13]
+; SDAG-NEXT:    s_cbranch_execz .LBB6_6
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x86, v5
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
@@ -1589,7 +1590,7 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; SDAG-NEXT:    v_mul_i32_i24_e32 v0, v0, v8
 ; SDAG-NEXT:    v_mov_b32_e32 v3, v2
-; SDAG-NEXT:  ; %bb.6: ; %Flow1
+; SDAG-NEXT:  .LBB6_6: ; %Flow1
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SDAG-NEXT:  .LBB6_7: ; %Flow2
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -1929,6 +1930,7 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; SDAG-NEXT:  .LBB7_4: ; %Flow
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[12:13]
+; SDAG-NEXT:    s_cbranch_execz .LBB7_6
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x86, v5
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
@@ -1940,7 +1942,7 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
 ; SDAG-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; SDAG-NEXT:    v_mul_i32_i24_e32 v0, v0, v8
 ; SDAG-NEXT:    v_mov_b32_e32 v3, v2
-; SDAG-NEXT:  ; %bb.6: ; %Flow1
+; SDAG-NEXT:  .LBB7_6: ; %Flow1
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SDAG-NEXT:  .LBB7_7: ; %Flow2
 ; SDAG-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
diff --git a/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll b/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll
index f34f9f38feeb4a961eae74001ddd1b69cc139f0e..2fd501014d467dbfd0a01afa5215dabe4700a693 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-copy-from-loop.ll
@@ -36,11 +36,12 @@ define amdgpu_ps void @i1_copy_from_loop(ptr addrspace(8) inreg %rsrc, i32 %tid)
 ; SI-NEXT:    v_cmp_le_f32_e32 vcc, 0, v1
 ; SI-NEXT:    s_mov_b64 s[8:9], -1
 ; SI-NEXT:    s_and_saveexec_b64 s[12:13], vcc
+; SI-NEXT:    s_cbranch_execz .LBB0_6
 ; SI-NEXT:  ; %bb.5: ; %end.loop
 ; SI-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; SI-NEXT:    s_add_i32 s14, s14, 1
 ; SI-NEXT:    s_xor_b64 s[8:9], exec, -1
-; SI-NEXT:  ; %bb.6: ; %Flow1
+; SI-NEXT:  .LBB0_6: ; %Flow1
 ; SI-NEXT:    ; in Loop: Header=BB0_3 Depth=1
 ; SI-NEXT:    s_or_b64 exec, exec, s[12:13]
 ; SI-NEXT:    s_branch .LBB0_2
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir b/llvm/test/CodeGen/AMDGPU/insert-handle-flat-vmem-ds.mir
similarity index 78%
rename from llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir
rename to llvm/test/CodeGen/AMDGPU/insert-handle-flat-vmem-ds.mir
index da38929fab9907ad7cc7e344ba10d4f464238819..d89f306c96a36a2473bd44e629aa79ae74e3afb2 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-handle-flat-vmem-ds.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=polaris10 -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=1 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=polaris10 -run-pass si-pre-emit-peephole -verify-machineinstrs %s -o - | FileCheck %s
 
 ---
 
@@ -7,9 +7,7 @@ name: skip_execz_flat
 body: |
   ; CHECK-LABEL: name: skip_execz_flat
   ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT:   successors: %bb.1(0x7fffffff)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
@@ -20,7 +18,7 @@ body: |
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
-    successors: %bb.1, %bb.2
+    successors: %bb.1(0x70000000), %bb.2(0x00000001)
     S_CBRANCH_EXECZ   %bb.2, implicit $exec
 
   bb.1:
@@ -38,9 +36,7 @@ name: skip_execz_mubuf
 body: |
   ; CHECK-LABEL: name: skip_execz_mubuf
   ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT:   successors: %bb.1(0x7fffffff)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
@@ -51,7 +47,7 @@ body: |
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
-    successors: %bb.1, %bb.2
+    successors: %bb.1(0x70000000), %bb.2(0x00000001)
     S_CBRANCH_EXECZ  %bb.2, implicit $exec
 
   bb.1:
@@ -69,9 +65,7 @@ name: skip_execz_ds
 body: |
   ; CHECK-LABEL: name: skip_execz_ds
   ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT:   successors: %bb.1(0x7fffffff)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
@@ -82,7 +76,7 @@ body: |
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
-    successors: %bb.1, %bb.2
+    successors: %bb.1(0x70000000), %bb.2(0x00000001)
     S_CBRANCH_EXECZ %bb.2, implicit $exec
 
   bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx10.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx10.mir
index b4ed3cafbacb5f33fcd14cb33e99318948201954..0bf74d96e134e698bf28b8c4a69da1b5ce0561ae 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx10.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx10.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -run-pass si-pre-emit-peephole -verify-machineinstrs  %s -o - | FileCheck %s
 
 ---
 name: skip_waitcnt_vscnt
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
index 2d092974ac566f12c7dd414777368709fd558aae..d88dc204e1336f21d85650aaf8cdb31422eacf1f 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass si-pre-emit-peephole -verify-machineinstrs  %s -o - | FileCheck %s
 
 ---
 name: skip_wait_loadcnt
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir
index 85618539be5ff5d3b6216cd884bab1c2d255ef6d..3281e4ccda7ab3353b7b24aa2f9480cee1dc94e8 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=1 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass si-pre-emit-peephole -verify-machineinstrs  %s -o - | FileCheck %s
 # Make sure mandatory skips are inserted to ensure GWS ops aren't run with exec = 0
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir
index 92719a5ee53aea835ff56b7f60e0498f54033473..372f22019aefbab4acf5f2a8ddb05045810ee7f0 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir
@@ -1,12 +1,34 @@
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=3 %s -o - | FileCheck %s
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -run-pass si-pre-emit-peephole %s -o - | FileCheck %s
 
 ---
 
-# CHECK-LABEL: name: no_count_dbg_value
-# CHECK: $vgpr1 = V_MOV_B32_e32 7, implicit $exec
-# CHECK-NOT: S_CBRANCH_EXECZ
 name: no_count_dbg_value
 body: |
+  ; CHECK-LABEL: name: no_count_dbg_value
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $vgpr1 = V_MOV_B32_e32 7, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   DBG_VALUE
+  ; CHECK-NEXT:   DBG_VALUE
+  ; CHECK-NEXT:   DBG_VALUE
+  ; CHECK-NEXT:   DBG_VALUE
+  ; CHECK-NEXT:   DBG_VALUE
+  ; CHECK-NEXT:   DBG_VALUE
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1, %bb.2
 
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
index c5198cdb421a502be9689f9ba7761c9400f232d5..38d928a006fb206046d93e3293f5048bca57b586 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
@@ -766,6 +766,7 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[12:13]
 ; GISEL-NEXT:  .LBB2_8: ; %Flow2
 ; GISEL-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT:    s_cbranch_execz .LBB2_10
 ; GISEL-NEXT:  ; %bb.9: ; %itofp-sw-bb
 ; GISEL-NEXT:    v_lshlrev_b64 v[9:10], 1, v[0:1]
 ; GISEL-NEXT:    v_lshlrev_b64 v[2:3], 1, v[2:3]
@@ -775,7 +776,7 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v10
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v11
 ; GISEL-NEXT:    v_mov_b32_e32 v3, v12
-; GISEL-NEXT:  ; %bb.10: ; %itofp-sw-epilog
+; GISEL-NEXT:  .LBB2_10: ; %itofp-sw-epilog
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:    v_bfe_u32 v3, v0, 2, 1
 ; GISEL-NEXT:    v_or_b32_e32 v0, v0, v3
@@ -1043,6 +1044,7 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[12:13]
 ; GISEL-NEXT:  .LBB3_8: ; %Flow2
 ; GISEL-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT:    s_cbranch_execz .LBB3_10
 ; GISEL-NEXT:  ; %bb.9: ; %itofp-sw-bb
 ; GISEL-NEXT:    v_lshlrev_b64 v[8:9], 1, v[0:1]
 ; GISEL-NEXT:    v_lshlrev_b64 v[10:11], 1, v[2:3]
@@ -1052,7 +1054,7 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v9
 ; GISEL-NEXT:    v_mov_b32_e32 v2, v10
 ; GISEL-NEXT:    v_mov_b32_e32 v3, v11
-; GISEL-NEXT:  ; %bb.10: ; %itofp-sw-epilog
+; GISEL-NEXT:  .LBB3_10: ; %itofp-sw-epilog
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:    v_bfe_u32 v4, v0, 2, 1
 ; GISEL-NEXT:    v_or_b32_e32 v0, v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/limit-coalesce.mir b/llvm/test/CodeGen/AMDGPU/limit-coalesce.mir
index b9105418a588c32140e0c0be9de84649ad19ff6d..ca774825f4ddefeb6644278fce4651cdb5350fc1 100644
--- a/llvm/test/CodeGen/AMDGPU/limit-coalesce.mir
+++ b/llvm/test/CodeGen/AMDGPU/limit-coalesce.mir
@@ -2,13 +2,13 @@
 
 # Check that coalescer does not create wider register tuple than in source
 
-# CHECK:  - { id: 2, class: vreg_64, preferred-register: '' }
-# CHECK:  - { id: 3, class: vreg_64, preferred-register: '' }
-# CHECK:  - { id: 4, class: vreg_64, preferred-register: '' }
-# CHECK:  - { id: 5, class: vreg_96, preferred-register: '' }
-# CHECK:  - { id: 6, class: vreg_96, preferred-register: '' }
-# CHECK:  - { id: 7, class: vreg_128, preferred-register: '' }
-# CHECK:  - { id: 8, class: vreg_128, preferred-register: '' }
+# CHECK:  - { id: 2, class: vreg_64, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 3, class: vreg_64, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 4, class: vreg_64, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 5, class: vreg_96, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 6, class: vreg_96, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 7, class: vreg_128, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 8, class: vreg_128, preferred-register: '', flags: [  ] }
 # No more registers shall be defined
 # CHECK-NEXT: liveins:
 # CHECK:    FLAT_STORE_DWORDX2 $vgpr0_vgpr1, %4,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
index 047b35b8c0f9d8a48cdd3e31704938bbac832e7f..026a8d7da7080b55d7fa378c7bbe64f15bc37710 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 < %s | FileCheck %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 < %s | FileCheck -check-prefixes=CHECK,GFX10 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 < %s | FileCheck -check-prefixes=CHECK,GFX11 %s
 
 declare i32 @llvm.amdgcn.ballot.i32(i1)
 declare i32 @llvm.ctpop.i32(i32)
@@ -522,3 +522,76 @@ true:
 false:
   ret i32 33
 }
+
+; Input that is not constant or direct result of a compare.
+; Tests setting 0 to inactive lanes.
+define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid, i32 %cond) {
+; GFX10-LABEL: non_cst_non_compare_input:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    ; implicit-def: $sgpr0
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-NEXT:  ; %bb.1: ; %B
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 2, v2
+; GFX10-NEXT:    ; implicit-def: $vgpr2
+; GFX10-NEXT:    s_and_b32 s0, vcc_lo, exec_lo
+; GFX10-NEXT:  ; %bb.2: ; %Flow
+; GFX10-NEXT:    s_andn2_saveexec_b32 s1, s1
+; GFX10-NEXT:  ; %bb.3: ; %A
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX10-NEXT:    s_andn2_b32 s0, s0, exec_lo
+; GFX10-NEXT:    s_and_b32 s2, vcc_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:  ; %bb.4: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v2
+; GFX10-NEXT:    v_mov_b32_e32 v2, s0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: non_cst_non_compare_input:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_mov_b32 s1, exec_lo
+; GFX11-NEXT:    ; implicit-def: $sgpr0
+; GFX11-NEXT:    v_cmpx_ne_u32_e32 0, v3
+; GFX11-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX11-NEXT:  ; %bb.1: ; %B
+; GFX11-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 2, v2
+; GFX11-NEXT:    ; implicit-def: $vgpr2
+; GFX11-NEXT:    s_and_b32 s0, vcc_lo, exec_lo
+; GFX11-NEXT:  ; %bb.2: ; %Flow
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s1, s1
+; GFX11-NEXT:  ; %bb.3: ; %A
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-NEXT:    s_and_not1_b32 s0, s0, exec_lo
+; GFX11-NEXT:    s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-NEXT:    s_or_b32 s0, s0, s2
+; GFX11-NEXT:  ; %bb.4: ; %exit
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11-NEXT:    v_cmp_ne_u32_e64 s0, 0, v2
+; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+entry:
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %A, label %B
+
+A:
+  %val_A = icmp uge i32 %tid, 1
+  br label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 2
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %phi)
+  store i32 %ballot, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
index 61f0f20f05704347024ffac73f6501f90da31995..c7597e98a6d5831ce09efa205c146d2611558dc7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
@@ -511,3 +511,50 @@ true:
 false:
   ret i32 33
 }
+
+; Input that is not constant or direct result of a compare.
+; Tests setting 0 to inactive lanes.
+define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid, i32 %cond) {
+; CHECK-LABEL: non_cst_non_compare_input:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; CHECK-NEXT:    ; implicit-def: $sgpr0_sgpr1
+; CHECK-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; CHECK-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; CHECK-NEXT:  ; %bb.1: ; %B
+; CHECK-NEXT:    v_cmp_gt_u32_e32 vcc, 2, v2
+; CHECK-NEXT:    s_and_b64 s[0:1], vcc, exec
+; CHECK-NEXT:    ; implicit-def: $vgpr2
+; CHECK-NEXT:  ; %bb.2: ; %Flow
+; CHECK-NEXT:    s_andn2_saveexec_b64 s[2:3], s[2:3]
+; CHECK-NEXT:  ; %bb.3: ; %A
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT:    s_and_b64 s[4:5], vcc, exec
+; CHECK-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; CHECK-NEXT:  ; %bb.4: ; %exit
+; CHECK-NEXT:    s_or_b64 exec, exec, s[2:3]
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
+; CHECK-NEXT:    s_endpgm
+entry:
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %A, label %B
+
+A:
+  %val_A = icmp uge i32 %tid, 1
+  br label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 2
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %phi)
+  store i64 %ballot, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
index 5dbfdf24ef36f749bf22d414357d58a3dccae928..fe69dc490624354ec85936c0455f4705783dfc0e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
@@ -40,12 +40,20 @@ define amdgpu_cs i64 @constant_true() {
 ; Test ballot of a non-comparison operation
 
 define amdgpu_cs i64 @non_compare(i32 %x) {
-; CHECK-LABEL: non_compare:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT:    s_mov_b32 s1, 0
-; CHECK-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
-; CHECK-NEXT:    ; return to shader part epilog
+; DAGISEL-LABEL: non_compare:
+; DAGISEL:       ; %bb.0:
+; DAGISEL-NEXT:    v_and_b32_e32 v0, 1, v0
+; DAGISEL-NEXT:    s_mov_b32 s1, 0
+; DAGISEL-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
+; DAGISEL-NEXT:    ; return to shader part epilog
+;
+; GISEL-LABEL: non_compare:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    v_and_b32_e32 v0, 1, v0
+; GISEL-NEXT:    s_mov_b32 s1, 0
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GISEL-NEXT:    s_and_b32 s0, vcc_lo, exec_lo
+; GISEL-NEXT:    ; return to shader part epilog
   %trunc = trunc i32 %x to i1
   %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %trunc)
   ret i64 %ballot
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
index 0ca01784d83383f00066ce282cc6ea03bd416083..1b1c89d9f5ad2f0cd189b9b83b07de55382d3c70 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
@@ -21,6 +21,7 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; GISEL12-NEXT:    s_mov_b32 s5, s6
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL12-NEXT:    s_cbranch_execz .LBB0_2
 ; GISEL12-NEXT:  ; %bb.1: ; %shader
 ; GISEL12-NEXT:    s_or_saveexec_b64 s[10:11], -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
@@ -35,7 +36,8 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; GISEL12-NEXT:    v_add_nc_u32_e32 v10, 42, v13
 ; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GISEL12-NEXT:    v_mov_b32_e32 v12, v1
-; GISEL12-NEXT:  ; %bb.2: ; %tail
+; GISEL12-NEXT:  .LBB0_2: ; %tail
+; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GISEL12-NEXT:    s_mov_b64 exec, s[4:5]
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
@@ -81,6 +83,7 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; GISEL10-NEXT:    s_mov_b32 s4, s5
 ; GISEL10-NEXT:    s_mov_b32 s5, s6
 ; GISEL10-NEXT:    s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL10-NEXT:    s_cbranch_execz .LBB0_2
 ; GISEL10-NEXT:  ; %bb.1: ; %shader
 ; GISEL10-NEXT:    s_or_saveexec_b64 s[10:11], -1
 ; GISEL10-NEXT:    v_cndmask_b32_e64 v0, 0x47, v13, s[10:11]
@@ -91,7 +94,7 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; GISEL10-NEXT:    v_mov_b32_e32 v11, v0
 ; GISEL10-NEXT:    v_add_nc_u32_e32 v10, 42, v13
 ; GISEL10-NEXT:    v_mov_b32_e32 v12, v1
-; GISEL10-NEXT:  ; %bb.2: ; %tail
+; GISEL10-NEXT:  .LBB0_2: ; %tail
 ; GISEL10-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GISEL10-NEXT:    s_mov_b64 exec, s[4:5]
 ; GISEL10-NEXT:    s_setpc_b64 s[8:9]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index 7371d498a70706359f0f9c9fb71158b96ddd6558..6cc2393d598e044be5a55389cb8ab0c47f223d9f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -539,11 +539,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX8DAGISEL-NEXT:    ; implicit-def: $sgpr4
 ; GFX8DAGISEL-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX8DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX8DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX8DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX8DAGISEL-NEXT:    s_load_dword s4, s[2:3], 0x2c
 ; GFX8DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX8DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX8DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX8DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX8DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -611,11 +610,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX9DAGISEL-NEXT:    ; implicit-def: $sgpr4
 ; GFX9DAGISEL-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX9DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX9DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX9DAGISEL-NEXT:    s_load_dword s4, s[2:3], 0x2c
 ; GFX9DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX9DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX9DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX9DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX9DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -681,11 +679,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1064DAGISEL-NEXT:    ; implicit-def: $sgpr4
 ; GFX1064DAGISEL-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1064DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1064DAGISEL-NEXT:    s_load_dword s4, s[2:3], 0x2c
 ; GFX1064DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1064DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1064DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1064DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX1064DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -751,11 +748,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1032DAGISEL-NEXT:    ; implicit-def: $sgpr1
 ; GFX1032DAGISEL-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032DAGISEL-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1032DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1032DAGISEL-NEXT:    s_load_dword s1, s[2:3], 0x2c
 ; GFX1032DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1032DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1032DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1032DAGISEL-NEXT:    s_or_saveexec_b32 s0, s0
 ; GFX1032DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032DAGISEL-NEXT:    v_mov_b32_e32 v1, s1
@@ -823,11 +819,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1164DAGISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164DAGISEL-NEXT:    v_cmpx_lt_u32_e32 15, v0
 ; GFX1164DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1164DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1164DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1164DAGISEL-NEXT:    s_load_b32 s4, s[2:3], 0x2c
 ; GFX1164DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1164DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1164DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1164DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX1164DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -903,11 +898,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1132DAGISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132DAGISEL-NEXT:    v_cmpx_lt_u32_e32 15, v0
 ; GFX1132DAGISEL-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1132DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1132DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1132DAGISEL-NEXT:    s_load_b32 s1, s[2:3], 0x2c
 ; GFX1132DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1132DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1132DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1132DAGISEL-NEXT:    s_or_saveexec_b32 s0, s0
 ; GFX1132DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132DAGISEL-NEXT:    v_mov_b32_e32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 60af21524a04a1276df4fc8d709844dc08fc8667..f7f8536219db6dd4f1ec9185dd60999118f75b07 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -540,11 +540,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX8DAGISEL-NEXT:    ; implicit-def: $sgpr4
 ; GFX8DAGISEL-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX8DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX8DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX8DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX8DAGISEL-NEXT:    s_load_dword s4, s[2:3], 0x2c
 ; GFX8DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX8DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX8DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX8DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX8DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -612,11 +611,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX9DAGISEL-NEXT:    ; implicit-def: $sgpr4
 ; GFX9DAGISEL-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX9DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX9DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX9DAGISEL-NEXT:    s_load_dword s4, s[2:3], 0x2c
 ; GFX9DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX9DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX9DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX9DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX9DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -682,11 +680,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1064DAGISEL-NEXT:    ; implicit-def: $sgpr4
 ; GFX1064DAGISEL-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1064DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1064DAGISEL-NEXT:    s_load_dword s4, s[2:3], 0x2c
 ; GFX1064DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1064DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1064DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1064DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX1064DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -752,11 +749,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1032DAGISEL-NEXT:    ; implicit-def: $sgpr1
 ; GFX1032DAGISEL-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032DAGISEL-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1032DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1032DAGISEL-NEXT:    s_load_dword s1, s[2:3], 0x2c
 ; GFX1032DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1032DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1032DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1032DAGISEL-NEXT:    s_or_saveexec_b32 s0, s0
 ; GFX1032DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032DAGISEL-NEXT:    v_mov_b32_e32 v1, s1
@@ -824,11 +820,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1164DAGISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164DAGISEL-NEXT:    v_cmpx_lt_u32_e32 15, v0
 ; GFX1164DAGISEL-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1164DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1164DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1164DAGISEL-NEXT:    s_load_b32 s4, s[2:3], 0x2c
 ; GFX1164DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1164DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1164DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1164DAGISEL-NEXT:    s_or_saveexec_b64 s[0:1], s[0:1]
 ; GFX1164DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164DAGISEL-NEXT:    v_mov_b32_e32 v1, s4
@@ -904,11 +899,10 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) {
 ; GFX1132DAGISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132DAGISEL-NEXT:    v_cmpx_lt_u32_e32 15, v0
 ; GFX1132DAGISEL-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1132DAGISEL-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX1132DAGISEL-NEXT:  ; %bb.1: ; %else
 ; GFX1132DAGISEL-NEXT:    s_load_b32 s1, s[2:3], 0x2c
 ; GFX1132DAGISEL-NEXT:    ; implicit-def: $vgpr0
-; GFX1132DAGISEL-NEXT:  .LBB4_2: ; %Flow
+; GFX1132DAGISEL-NEXT:  ; %bb.2: ; %Flow
 ; GFX1132DAGISEL-NEXT:    s_or_saveexec_b32 s0, s0
 ; GFX1132DAGISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132DAGISEL-NEXT:    v_mov_b32_e32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 0b3ef62856f540058bf3c5a0920ed5e741e2e9bc..08b089a32d1d75bedc8a6bf39ef3b9f72f89fd5b 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -7957,11 +7957,10 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_xor_b32 s1, exec_lo, s1
-; GFX12-NEXT:    s_cbranch_execz .LBB29_8
 ; GFX12-NEXT:  ; %bb.7:
 ; GFX12-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s0
 ; GFX12-NEXT:    ds_add_rtn_f32 v1, v1, v2
-; GFX12-NEXT:  .LBB29_8:
+; GFX12-NEXT:  ; %bb.8:
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[2:3], 0x0
@@ -8041,11 +8040,10 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX940-NEXT:    ; implicit-def: $vgpr2
 ; GFX940-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX940-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX940-NEXT:    s_cbranch_execz .LBB29_8
 ; GFX940-NEXT:  ; %bb.7:
 ; GFX940-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX940-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX940-NEXT:  .LBB29_8:
+; GFX940-NEXT:  ; %bb.8:
 ; GFX940-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
@@ -8128,11 +8126,10 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    ; implicit-def: $vgpr2
 ; GFX11-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX11-NEXT:    s_cbranch_execz .LBB29_8
 ; GFX11-NEXT:  ; %bb.7:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX11-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX11-NEXT:  .LBB29_8:
+; GFX11-NEXT:  ; %bb.8:
 ; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[2:3], 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
@@ -8207,11 +8204,10 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX10-NEXT:    ; implicit-def: $vgpr2
 ; GFX10-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX10-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX10-NEXT:    s_cbranch_execz .LBB29_8
 ; GFX10-NEXT:  ; %bb.7:
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX10-NEXT:  .LBB29_8:
+; GFX10-NEXT:  ; %bb.8:
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
@@ -8220,7 +8216,6 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-NEXT:    v_add_f32_e32 v0, s2, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
 ; GFX10-NEXT:    s_endpgm
 ;
@@ -8289,11 +8284,10 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2
 ; GFX90A-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX90A-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX90A-NEXT:    s_cbranch_execz .LBB29_8
 ; GFX90A-NEXT:  ; %bb.7:
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX90A-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX90A-NEXT:  .LBB29_8:
+; GFX90A-NEXT:  ; %bb.8:
 ; GFX90A-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
@@ -8370,11 +8364,10 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX908-NEXT:    ; implicit-def: $vgpr2
 ; GFX908-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX908-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX908-NEXT:    s_cbranch_execz .LBB29_8
 ; GFX908-NEXT:  ; %bb.7:
 ; GFX908-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX908-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX908-NEXT:  .LBB29_8:
+; GFX908-NEXT:  ; %bb.8:
 ; GFX908-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
@@ -8452,12 +8445,11 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX8-NEXT:    ; implicit-def: $vgpr2
 ; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX8-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX8-NEXT:    s_cbranch_execz .LBB29_8
 ; GFX8-NEXT:  ; %bb.7:
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX8-NEXT:  .LBB29_8:
+; GFX8-NEXT:  ; %bb.8:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[2:3], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index 1dd18b4228fe5e1c92fedc4338afe4dff169dd50..9d43efbdf07b1f23c0fbc1c54992640c1b846ff4 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -16,47 +16,18 @@ define void @issue63986(i64 %0, i64 %idxprom) {
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    v_mov_b32_e32 v9, s7
 ; CHECK-NEXT:    v_mov_b32_e32 v8, s6
-; CHECK-NEXT:    flat_load_ubyte v10, v[8:9] offset:5
-; CHECK-NEXT:    flat_load_ubyte v11, v[8:9] offset:6
-; CHECK-NEXT:    flat_load_ubyte v12, v[8:9] offset:7
-; CHECK-NEXT:    flat_load_ubyte v13, v[8:9] offset:3
-; CHECK-NEXT:    flat_load_ubyte v14, v[8:9] offset:2
-; CHECK-NEXT:    flat_load_ubyte v15, v[8:9] offset:1
-; CHECK-NEXT:    flat_load_ubyte v16, v[8:9]
-; CHECK-NEXT:    flat_load_ubyte v17, v[8:9] offset:4
-; CHECK-NEXT:    flat_load_ubyte v18, v[8:9] offset:13
-; CHECK-NEXT:    flat_load_ubyte v19, v[8:9] offset:14
-; CHECK-NEXT:    flat_load_ubyte v20, v[8:9] offset:15
-; CHECK-NEXT:    flat_load_ubyte v21, v[8:9] offset:11
-; CHECK-NEXT:    flat_load_ubyte v22, v[8:9] offset:10
-; CHECK-NEXT:    flat_load_ubyte v23, v[8:9] offset:9
-; CHECK-NEXT:    flat_load_ubyte v24, v[8:9] offset:8
-; CHECK-NEXT:    flat_load_ubyte v25, v[8:9] offset:12
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[8:9]
 ; CHECK-NEXT:    s_add_u32 s4, s4, 1
 ; CHECK-NEXT:    s_addc_u32 s5, s5, 0
-; CHECK-NEXT:    v_add_co_u32_e32 v8, vcc, s6, v6
+; CHECK-NEXT:    v_mov_b32_e32 v13, s7
+; CHECK-NEXT:    v_add_co_u32_e32 v12, vcc, s6, v6
 ; CHECK-NEXT:    v_cmp_ge_u64_e64 s[8:9], s[4:5], 2
-; CHECK-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v7, vcc
+; CHECK-NEXT:    v_addc_co_u32_e32 v13, vcc, v13, v7, vcc
 ; CHECK-NEXT:    s_add_u32 s6, s6, 16
 ; CHECK-NEXT:    s_addc_u32 s7, s7, 0
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[8:9]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[8:9], v13 offset:3
-; CHECK-NEXT:    flat_store_byte v[8:9], v14 offset:2
-; CHECK-NEXT:    flat_store_byte v[8:9], v15 offset:1
-; CHECK-NEXT:    flat_store_byte v[8:9], v16
-; CHECK-NEXT:    flat_store_byte v[8:9], v12 offset:7
-; CHECK-NEXT:    flat_store_byte v[8:9], v11 offset:6
-; CHECK-NEXT:    flat_store_byte v[8:9], v10 offset:5
-; CHECK-NEXT:    flat_store_byte v[8:9], v17 offset:4
-; CHECK-NEXT:    flat_store_byte v[8:9], v21 offset:11
-; CHECK-NEXT:    flat_store_byte v[8:9], v22 offset:10
-; CHECK-NEXT:    flat_store_byte v[8:9], v23 offset:9
-; CHECK-NEXT:    flat_store_byte v[8:9], v24 offset:8
-; CHECK-NEXT:    flat_store_byte v[8:9], v20 offset:15
-; CHECK-NEXT:    flat_store_byte v[8:9], v19 offset:14
-; CHECK-NEXT:    flat_store_byte v[8:9], v18 offset:13
-; CHECK-NEXT:    flat_store_byte v[8:9], v25 offset:12
+; CHECK-NEXT:    flat_store_dwordx4 v[12:13], v[8:11]
 ; CHECK-NEXT:    s_cbranch_vccz .LBB0_2
 ; CHECK-NEXT:  ; %bb.3: ; %loop-memcpy-residual-header
 ; CHECK-NEXT:    s_mov_b32 s4, 0
@@ -128,47 +99,18 @@ define void @issue63986(i64 %0, i64 %idxprom) {
 ; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    v_mov_b32_e32 v10, s10
 ; CHECK-NEXT:    v_mov_b32_e32 v11, s11
-; CHECK-NEXT:    flat_load_ubyte v12, v[10:11] offset:5
-; CHECK-NEXT:    flat_load_ubyte v13, v[10:11] offset:6
-; CHECK-NEXT:    flat_load_ubyte v14, v[10:11] offset:7
-; CHECK-NEXT:    flat_load_ubyte v15, v[10:11] offset:3
-; CHECK-NEXT:    flat_load_ubyte v16, v[10:11] offset:2
-; CHECK-NEXT:    flat_load_ubyte v17, v[10:11] offset:1
-; CHECK-NEXT:    flat_load_ubyte v18, v[10:11]
-; CHECK-NEXT:    flat_load_ubyte v19, v[10:11] offset:4
-; CHECK-NEXT:    flat_load_ubyte v20, v[10:11] offset:13
-; CHECK-NEXT:    flat_load_ubyte v21, v[10:11] offset:14
-; CHECK-NEXT:    flat_load_ubyte v22, v[10:11] offset:15
-; CHECK-NEXT:    flat_load_ubyte v23, v[10:11] offset:11
-; CHECK-NEXT:    flat_load_ubyte v24, v[10:11] offset:10
-; CHECK-NEXT:    flat_load_ubyte v25, v[10:11] offset:9
-; CHECK-NEXT:    flat_load_ubyte v26, v[10:11] offset:8
-; CHECK-NEXT:    flat_load_ubyte v27, v[10:11] offset:12
+; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[10:11]
+; CHECK-NEXT:    v_mov_b32_e32 v15, s11
 ; CHECK-NEXT:    s_add_u32 s14, s14, 1
-; CHECK-NEXT:    v_add_co_u32_e32 v10, vcc, s10, v2
-; CHECK-NEXT:    v_addc_co_u32_e32 v11, vcc, v11, v3, vcc
+; CHECK-NEXT:    v_add_co_u32_e32 v14, vcc, s10, v2
+; CHECK-NEXT:    v_addc_co_u32_e32 v15, vcc, v15, v3, vcc
 ; CHECK-NEXT:    s_addc_u32 s15, s15, 0
 ; CHECK-NEXT:    s_add_u32 s10, s10, 16
 ; CHECK-NEXT:    v_cmp_ge_u64_e32 vcc, s[14:15], v[4:5]
 ; CHECK-NEXT:    s_addc_u32 s11, s11, 0
 ; CHECK-NEXT:    s_or_b64 s[12:13], vcc, s[12:13]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[10:11], v15 offset:3
-; CHECK-NEXT:    flat_store_byte v[10:11], v16 offset:2
-; CHECK-NEXT:    flat_store_byte v[10:11], v17 offset:1
-; CHECK-NEXT:    flat_store_byte v[10:11], v18
-; CHECK-NEXT:    flat_store_byte v[10:11], v14 offset:7
-; CHECK-NEXT:    flat_store_byte v[10:11], v13 offset:6
-; CHECK-NEXT:    flat_store_byte v[10:11], v12 offset:5
-; CHECK-NEXT:    flat_store_byte v[10:11], v19 offset:4
-; CHECK-NEXT:    flat_store_byte v[10:11], v23 offset:11
-; CHECK-NEXT:    flat_store_byte v[10:11], v24 offset:10
-; CHECK-NEXT:    flat_store_byte v[10:11], v25 offset:9
-; CHECK-NEXT:    flat_store_byte v[10:11], v26 offset:8
-; CHECK-NEXT:    flat_store_byte v[10:11], v22 offset:15
-; CHECK-NEXT:    flat_store_byte v[10:11], v21 offset:14
-; CHECK-NEXT:    flat_store_byte v[10:11], v20 offset:13
-; CHECK-NEXT:    flat_store_byte v[10:11], v27 offset:12
+; CHECK-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
 ; CHECK-NEXT:    s_andn2_b64 exec, exec, s[12:13]
 ; CHECK-NEXT:    s_cbranch_execnz .LBB0_14
 ; CHECK-NEXT:  .LBB0_15: ; %Flow20
@@ -251,23 +193,11 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
 ; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:  .LBB1_8: ; %post-loop-memcpy-expansion
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, v2
+; CHECK-NEXT:    v_mov_b32_e32 v4, v2
+; CHECK-NEXT:    v_mov_b32_e32 v5, v2
 ; CHECK-NEXT:    s_and_b64 vcc, exec, 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:3
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:2
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:1
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:7
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:6
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:5
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:4
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:11
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:10
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:9
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:8
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:15
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:14
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:13
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:12
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:  .LBB1_9: ; %loop-memcpy-expansion2
 ; CHECK-NEXT:    s_mov_b64 vcc, vcc
 ; CHECK-NEXT:    s_cbranch_vccz .LBB1_9
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
index 0a76e169e9c385293422bdc0fc5fedcffba49527..8c28fac0d839c29747c7c5ff0e4f1257d1fa8d01 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
@@ -10,108 +10,21 @@ define amdgpu_kernel void @memcpy_p0_p0_minsize(ptr %dest, ptr readonly %src) #0
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-NEXT:    v_mov_b32_e32 v1, s3
-; CHECK-NEXT:    flat_load_ubyte v4, v[0:1]
-; CHECK-NEXT:    flat_load_ubyte v5, v[0:1] offset:1
-; CHECK-NEXT:    flat_load_ubyte v6, v[0:1] offset:2
-; CHECK-NEXT:    flat_load_ubyte v7, v[0:1] offset:3
-; CHECK-NEXT:    flat_load_ubyte v8, v[0:1] offset:4
-; CHECK-NEXT:    flat_load_ubyte v9, v[0:1] offset:5
-; CHECK-NEXT:    flat_load_ubyte v10, v[0:1] offset:6
-; CHECK-NEXT:    flat_load_ubyte v11, v[0:1] offset:7
-; CHECK-NEXT:    flat_load_ubyte v12, v[0:1] offset:8
-; CHECK-NEXT:    flat_load_ubyte v13, v[0:1] offset:9
-; CHECK-NEXT:    flat_load_ubyte v14, v[0:1] offset:10
-; CHECK-NEXT:    flat_load_ubyte v15, v[0:1] offset:11
-; CHECK-NEXT:    flat_load_ubyte v16, v[0:1] offset:12
-; CHECK-NEXT:    flat_load_ubyte v17, v[0:1] offset:13
-; CHECK-NEXT:    flat_load_ubyte v18, v[0:1] offset:14
-; CHECK-NEXT:    v_mov_b32_e32 v3, s1
-; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    v_mov_b32_e32 v12, s3
+; CHECK-NEXT:    v_mov_b32_e32 v11, s2
+; CHECK-NEXT:    flat_load_ubyte v13, v[11:12] offset:46
+; CHECK-NEXT:    flat_load_ushort v14, v[11:12] offset:44
+; CHECK-NEXT:    flat_load_dwordx3 v[8:10], v[11:12] offset:32
+; CHECK-NEXT:    flat_load_dwordx4 v[0:3], v[11:12] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[11:12]
+; CHECK-NEXT:    v_mov_b32_e32 v12, s1
+; CHECK-NEXT:    v_mov_b32_e32 v11, s0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[2:3], v4
-; CHECK-NEXT:    flat_store_byte v[2:3], v5 offset:1
-; CHECK-NEXT:    flat_store_byte v[2:3], v6 offset:2
-; CHECK-NEXT:    flat_store_byte v[2:3], v7 offset:3
-; CHECK-NEXT:    flat_store_byte v[2:3], v8 offset:4
-; CHECK-NEXT:    flat_store_byte v[2:3], v9 offset:5
-; CHECK-NEXT:    flat_store_byte v[2:3], v10 offset:6
-; CHECK-NEXT:    flat_store_byte v[2:3], v11 offset:7
-; CHECK-NEXT:    flat_store_byte v[2:3], v12 offset:8
-; CHECK-NEXT:    flat_store_byte v[2:3], v13 offset:9
-; CHECK-NEXT:    flat_store_byte v[2:3], v14 offset:10
-; CHECK-NEXT:    flat_store_byte v[2:3], v15 offset:11
-; CHECK-NEXT:    flat_store_byte v[2:3], v16 offset:12
-; CHECK-NEXT:    flat_store_byte v[2:3], v17 offset:13
-; CHECK-NEXT:    flat_store_byte v[2:3], v18 offset:14
-; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[0:1] offset:29
-; CHECK-NEXT:    flat_load_ubyte v6, v[0:1] offset:28
-; CHECK-NEXT:    flat_load_ubyte v7, v[0:1] offset:27
-; CHECK-NEXT:    flat_load_ubyte v8, v[0:1] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[0:1] offset:25
-; CHECK-NEXT:    flat_load_ubyte v10, v[0:1] offset:24
-; CHECK-NEXT:    flat_load_ubyte v11, v[0:1] offset:23
-; CHECK-NEXT:    flat_load_ubyte v12, v[0:1] offset:22
-; CHECK-NEXT:    flat_load_ubyte v13, v[0:1] offset:21
-; CHECK-NEXT:    flat_load_ubyte v14, v[0:1] offset:20
-; CHECK-NEXT:    flat_load_ubyte v15, v[0:1] offset:19
-; CHECK-NEXT:    flat_load_ubyte v16, v[0:1] offset:18
-; CHECK-NEXT:    flat_load_ubyte v17, v[0:1] offset:17
-; CHECK-NEXT:    flat_load_ubyte v18, v[0:1] offset:16
-; CHECK-NEXT:    flat_load_ubyte v19, v[0:1] offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:30
-; CHECK-NEXT:    flat_store_byte v[2:3], v5 offset:29
-; CHECK-NEXT:    flat_store_byte v[2:3], v6 offset:28
-; CHECK-NEXT:    flat_store_byte v[2:3], v7 offset:27
-; CHECK-NEXT:    flat_store_byte v[2:3], v8 offset:26
-; CHECK-NEXT:    flat_store_byte v[2:3], v9 offset:25
-; CHECK-NEXT:    flat_store_byte v[2:3], v10 offset:24
-; CHECK-NEXT:    flat_store_byte v[2:3], v11 offset:23
-; CHECK-NEXT:    flat_store_byte v[2:3], v12 offset:22
-; CHECK-NEXT:    flat_store_byte v[2:3], v13 offset:21
-; CHECK-NEXT:    flat_store_byte v[2:3], v14 offset:20
-; CHECK-NEXT:    flat_store_byte v[2:3], v15 offset:19
-; CHECK-NEXT:    flat_store_byte v[2:3], v16 offset:18
-; CHECK-NEXT:    flat_store_byte v[2:3], v17 offset:17
-; CHECK-NEXT:    flat_store_byte v[2:3], v18 offset:16
-; CHECK-NEXT:    flat_store_byte v[2:3], v19 offset:15
-; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:46
-; CHECK-NEXT:    flat_load_ubyte v5, v[0:1] offset:45
-; CHECK-NEXT:    flat_load_ubyte v6, v[0:1] offset:44
-; CHECK-NEXT:    flat_load_ubyte v7, v[0:1] offset:43
-; CHECK-NEXT:    flat_load_ubyte v8, v[0:1] offset:42
-; CHECK-NEXT:    flat_load_ubyte v9, v[0:1] offset:41
-; CHECK-NEXT:    flat_load_ubyte v10, v[0:1] offset:40
-; CHECK-NEXT:    flat_load_ubyte v11, v[0:1] offset:39
-; CHECK-NEXT:    flat_load_ubyte v12, v[0:1] offset:38
-; CHECK-NEXT:    flat_load_ubyte v13, v[0:1] offset:37
-; CHECK-NEXT:    flat_load_ubyte v14, v[0:1] offset:36
-; CHECK-NEXT:    flat_load_ubyte v15, v[0:1] offset:35
-; CHECK-NEXT:    flat_load_ubyte v16, v[0:1] offset:34
-; CHECK-NEXT:    flat_load_ubyte v17, v[0:1] offset:33
-; CHECK-NEXT:    flat_load_ubyte v18, v[0:1] offset:32
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_load_ubyte v0, v[0:1] offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:46
-; CHECK-NEXT:    flat_store_byte v[2:3], v5 offset:45
-; CHECK-NEXT:    flat_store_byte v[2:3], v6 offset:44
-; CHECK-NEXT:    flat_store_byte v[2:3], v7 offset:43
-; CHECK-NEXT:    flat_store_byte v[2:3], v8 offset:42
-; CHECK-NEXT:    flat_store_byte v[2:3], v9 offset:41
-; CHECK-NEXT:    flat_store_byte v[2:3], v10 offset:40
-; CHECK-NEXT:    flat_store_byte v[2:3], v11 offset:39
-; CHECK-NEXT:    flat_store_byte v[2:3], v12 offset:38
-; CHECK-NEXT:    flat_store_byte v[2:3], v13 offset:37
-; CHECK-NEXT:    flat_store_byte v[2:3], v14 offset:36
-; CHECK-NEXT:    flat_store_byte v[2:3], v15 offset:35
-; CHECK-NEXT:    flat_store_byte v[2:3], v16 offset:34
-; CHECK-NEXT:    flat_store_byte v[2:3], v17 offset:33
-; CHECK-NEXT:    flat_store_byte v[2:3], v18 offset:32
-; CHECK-NEXT:    flat_store_byte v[2:3], v0 offset:31
+; CHECK-NEXT:    flat_store_byte v[11:12], v13 offset:46
+; CHECK-NEXT:    flat_store_short v[11:12], v14 offset:44
+; CHECK-NEXT:    flat_store_dwordx3 v[11:12], v[8:10] offset:32
+; CHECK-NEXT:    flat_store_dwordx4 v[11:12], v[0:3] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[11:12], v[4:7]
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 47, i1 false)
@@ -185,375 +98,59 @@ define amdgpu_kernel void @memcpy_p5_p4_minsize(ptr addrspace(5) %local, ptr add
 ; CHECK-NEXT:    s_mov_b64 s[16:17], s[0:1]
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x8
 ; CHECK-NEXT:    s_load_dword s2, s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v24, 0
 ; CHECK-NEXT:    s_add_u32 s16, s16, s13
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:15
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:14
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:13
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:12
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:11
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:10
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:9
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:8
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:7
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:6
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:5
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:4
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:3
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:2
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:1
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1]
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:31
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v24, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v24, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v24, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v24, s[0:1] offset:32
 ; CHECK-NEXT:    s_addc_u32 s17, s17, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, s2
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:10
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:23
+; CHECK-NEXT:    v_mov_b32_e32 v25, s2
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:124
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:120
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:116
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:108
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:104
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:100
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1]
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    buffer_store_dword v11, v25, s[16:19], 0 offen offset:92
+; CHECK-NEXT:    buffer_store_dword v10, v25, s[16:19], 0 offen offset:88
+; CHECK-NEXT:    buffer_store_dword v9, v25, s[16:19], 0 offen offset:84
+; CHECK-NEXT:    buffer_store_dword v8, v25, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
+; CHECK-NEXT:    buffer_store_dword v15, v25, s[16:19], 0 offen offset:76
+; CHECK-NEXT:    buffer_store_dword v14, v25, s[16:19], 0 offen offset:72
+; CHECK-NEXT:    buffer_store_dword v13, v25, s[16:19], 0 offen offset:68
+; CHECK-NEXT:    buffer_store_dword v12, v25, s[16:19], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:9
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:7
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:20
+; CHECK-NEXT:    buffer_store_dword v19, v25, s[16:19], 0 offen offset:60
+; CHECK-NEXT:    buffer_store_dword v18, v25, s[16:19], 0 offen offset:56
+; CHECK-NEXT:    buffer_store_dword v17, v25, s[16:19], 0 offen offset:52
+; CHECK-NEXT:    buffer_store_dword v16, v25, s[16:19], 0 offen offset:48
 ; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:6
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:5
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:2
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:47
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:17
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:3
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:16
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:27
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:26
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:25
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:24
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:45
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:44
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:43
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:23
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:36
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:22
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:35
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:21
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:34
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:33
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:19
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:32
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:28
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:29
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:42
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:18
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:63
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:16
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:61
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:27
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:40
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:26
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:39
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:25
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:38
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:37
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:57
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:43
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:56
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:45
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:58
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:49
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:35
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:48
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:46
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:47
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:60
-; CHECK-NEXT:    s_waitcnt vmcnt(33)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:34
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:79
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:41
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:42
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:55
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:33
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:32
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:77
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:61
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:74
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:53
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:39
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:52
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:38
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:51
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:37
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:50
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:57
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:70
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:69
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:58
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:71
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:49
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:48
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:93
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:46
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:59
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:73
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:41
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:54
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:55
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:68
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:74
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:87
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:53
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:66
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:65
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:51
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:64
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:62
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:63
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:76
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:50
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:95
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:77
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:90
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:71
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:83
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:70
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:69
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:59
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:72
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:73
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:85
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:54
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:67
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:81
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:66
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:111
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:65
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:110
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:64
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:109
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:62
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:75
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:76
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:89
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:90
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:103
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:86
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:84
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:82
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:87
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:100
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:67
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:80
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:78
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:94
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:79
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:92
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:95
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:108
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:93
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:106
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:75
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:88
+; CHECK-NEXT:    buffer_store_dword v23, v25, s[16:19], 0 offen offset:44
+; CHECK-NEXT:    buffer_store_dword v22, v25, s[16:19], 0 offen offset:40
+; CHECK-NEXT:    buffer_store_dword v21, v25, s[16:19], 0 offen offset:36
+; CHECK-NEXT:    buffer_store_dword v20, v25, s[16:19], 0 offen offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:89
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:102
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:78
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:91
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:94
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:107
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:105
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:101
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:91
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:104
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:86
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:85
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:83
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:82
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:96
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:97
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:98
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:99
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:120
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:81
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:80
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:111
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:110
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:109
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:121
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:122
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:123
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:124
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:125
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:126
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:107
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:127
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:106
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:105
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:103
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:102
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:101
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:116
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:117
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:119
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:114
-; CHECK-NEXT:    s_waitcnt vmcnt(34)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:118
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:115
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:113
-; CHECK-NEXT:    global_load_ubyte v21, v0, s[0:1] offset:112
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:99
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:98
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:97
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:96
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:127
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:126
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:125
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:124
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:123
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:122
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:121
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:119
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:118
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:117
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:115
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:114
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:113
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v21, v1, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)
@@ -569,363 +166,57 @@ define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %
 ; CHECK-NEXT:    s_add_u32 s16, s16, s13
 ; CHECK-NEXT:    s_addc_u32 s17, s17, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, s0
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:2
+; CHECK-NEXT:    v_mov_b32_e32 v26, s0
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen offset:96
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, s0
-; CHECK-NEXT:    v_mov_b32_e32 v1, s1
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:23
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:22
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:21
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:19
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:18
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:17
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:47
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:31
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:16
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:45
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:23
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:37
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:22
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:21
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:35
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:20
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:34
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:19
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:33
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:18
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:32
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:29
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:30
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:17
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:63
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:28
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:42
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:26
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:25
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:39
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:24
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:38
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:27
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:41
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:45
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:59
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:37
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:51
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:36
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:50
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:35
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:49
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:34
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:48
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:46
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:47
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:61
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:29
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:43
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:44
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:58
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:33
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:79
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:32
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:42
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:40
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:54
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:39
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:53
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:38
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:41
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:55
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:59
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:73
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:51
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:65
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:50
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:64
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:62
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:63
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:77
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:46
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:61
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:75
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:43
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:57
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:58
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:49
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:95
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:48
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:56
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:70
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:54
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:53
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:67
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:52
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:66
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:55
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:69
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:73
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:87
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:65
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:111
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:64
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:110
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:62
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:76
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:77
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:91
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:60
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:74
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:75
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:89
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:57
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:71
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:72
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:86
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:70
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:68
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:83
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:67
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:81
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:66
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:80
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:78
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:79
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:93
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:69
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:82
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:87
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:101
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:76
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:90
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:91
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:105
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:74
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:89
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:103
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:71
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:85
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:86
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:78
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:93
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:107
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:90
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:88
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:102
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:85
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:99
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:94
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:95
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:109
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:92
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:106
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:94
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:84
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:83
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:82
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:81
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:96
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:97
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:98
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:80
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:111
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:110
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:109
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:99
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:121
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:122
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:123
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v8, v26, s[16:19], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v9, v26, s[16:19], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v10, v26, s[16:19], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v11, v26, s[16:19], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v12, v26, s[16:19], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v13, v26, s[16:19], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v14, v26, s[16:19], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v15, v26, s[16:19], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v16, v26, s[16:19], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v17, v26, s[16:19], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v18, v26, s[16:19], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v19, v26, s[16:19], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v23, v26, s[16:19], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v22, v26, s[16:19], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v21, v26, s[16:19], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v20, v26, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v25, s1
+; CHECK-NEXT:    v_mov_b32_e32 v24, s0
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:112
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:76
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:107
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:105
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:104
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:103
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:106
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:102
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:101
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:100
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:126
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:117
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:118
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:119
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:127
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:114
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:115
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:108
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:125
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:113
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[16:19], 0 offen offset:112
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:98
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:97
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:96
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:127
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:126
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:96
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:12
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[20:23] offset:80
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[16:19] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[12:15] offset:32
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[8:11] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:125
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:124
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:123
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:122
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:121
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:120
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:119
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:118
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:117
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:116
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:115
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:114
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:113
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:112
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7]
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
@@ -972,279 +263,27 @@ define amdgpu_kernel void @memcpy_p0_p3_minsize(ptr %generic) #0 {
 ; CHECK-LABEL: memcpy_p0_p3_minsize:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:112
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:113
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:114
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:115
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:116
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:117
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:118
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:119
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, s0
-; CHECK-NEXT:    v_mov_b32_e32 v1, s1
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:112
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:113
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:114
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:115
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:116
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:117
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:118
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:119
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:120
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:121
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:122
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:123
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:124
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:125
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:126
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:127
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:120
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:121
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:122
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:123
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:124
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:125
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:126
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:127
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:96
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:97
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:98
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:99
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:100
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:101
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:102
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:103
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:96
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:97
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:98
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:99
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:100
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:101
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:102
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:103
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:104
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:105
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:106
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:107
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:108
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:109
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:110
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:111
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:104
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:105
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:106
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:107
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:108
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:109
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:110
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:111
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:80
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:81
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:82
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:83
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:84
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:85
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:86
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:87
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:80
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:81
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:82
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:83
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:84
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:85
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:86
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:87
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:88
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:89
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:90
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:91
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:92
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:93
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:94
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:95
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:88
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:89
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:90
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:91
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:92
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:93
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:94
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:95
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:64
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:65
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:66
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:67
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:68
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:69
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:70
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:71
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:64
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:65
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:66
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:67
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:68
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:69
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:70
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:71
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:72
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:73
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:74
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:75
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:76
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:77
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:78
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:79
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:72
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:73
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:74
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:75
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:76
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:77
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:78
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:79
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:48
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:49
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:50
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:51
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:52
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:53
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:54
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:55
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:48
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:49
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:50
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:51
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:52
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:53
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:54
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:55
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:56
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:57
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:58
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:59
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:60
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:61
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:62
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:63
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:56
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:57
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:58
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:59
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:60
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:61
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:62
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:63
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:32
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:33
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:34
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:35
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:36
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:37
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:38
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:39
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:32
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:33
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:34
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:35
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:36
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:37
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:38
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:39
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:40
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:41
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:42
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:43
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:44
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:45
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:46
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:47
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:40
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:41
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:42
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:43
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:44
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:45
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:46
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:47
-; CHECK-NEXT:    ds_read_u8 v3, v2
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:7
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v17, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v18, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:16
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:17
-; CHECK-NEXT:    ds_read_u8 v21, v2 offset:18
-; CHECK-NEXT:    ds_read_u8 v22, v2 offset:19
-; CHECK-NEXT:    ds_read_u8 v23, v2 offset:20
-; CHECK-NEXT:    ds_read_u8 v24, v2 offset:21
-; CHECK-NEXT:    ds_read_u8 v25, v2 offset:22
-; CHECK-NEXT:    ds_read_u8 v26, v2 offset:23
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:19
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:23
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:24
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:25
-; CHECK-NEXT:    ds_read_u8 v21, v2 offset:26
-; CHECK-NEXT:    ds_read_u8 v22, v2 offset:27
-; CHECK-NEXT:    ds_read_u8 v23, v2 offset:28
-; CHECK-NEXT:    ds_read_u8 v24, v2 offset:29
-; CHECK-NEXT:    ds_read_u8 v25, v2 offset:30
-; CHECK-NEXT:    ds_read_u8 v2, v2 offset:31
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:27
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:28
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:31
-; CHECK-NEXT:    flat_store_byte v[0:1], v3
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:1
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:2
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:3
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:4
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:5
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:6
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:7
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:9
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:10
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:11
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:12
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:13
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:14
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:15
+; CHECK-NEXT:    v_mov_b32_e32 v16, 0
+; CHECK-NEXT:    ds_read2_b64 v[0:3], v16 offset1:1
+; CHECK-NEXT:    ds_read2_b64 v[4:7], v16 offset0:2 offset1:3
+; CHECK-NEXT:    ds_read2_b64 v[8:11], v16 offset0:4 offset1:5
+; CHECK-NEXT:    ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v21, s1
+; CHECK-NEXT:    v_mov_b32_e32 v20, s0
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[0:3]
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[4:7] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[8:11] offset:32
+; CHECK-NEXT:    ds_read2_b64 v[0:3], v16 offset0:8 offset1:9
+; CHECK-NEXT:    ds_read2_b64 v[4:7], v16 offset0:10 offset1:11
+; CHECK-NEXT:    ds_read2_b64 v[8:11], v16 offset0:12 offset1:13
+; CHECK-NEXT:    ds_read2_b64 v[16:19], v16 offset0:14 offset1:15
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[12:15] offset:48
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[0:3] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[4:7] offset:80
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[8:11] offset:96
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[16:19] offset:112
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p3.i64(ptr %generic, ptr addrspace(3) @shared, i64 128, i1 false)
@@ -1256,108 +295,21 @@ define amdgpu_kernel void @memcpy_p0_p0_optsize(ptr %dest, ptr %src) #1 {
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-NEXT:    v_mov_b32_e32 v1, s3
-; CHECK-NEXT:    flat_load_ubyte v4, v[0:1]
-; CHECK-NEXT:    flat_load_ubyte v5, v[0:1] offset:1
-; CHECK-NEXT:    flat_load_ubyte v6, v[0:1] offset:2
-; CHECK-NEXT:    flat_load_ubyte v7, v[0:1] offset:3
-; CHECK-NEXT:    flat_load_ubyte v8, v[0:1] offset:4
-; CHECK-NEXT:    flat_load_ubyte v9, v[0:1] offset:5
-; CHECK-NEXT:    flat_load_ubyte v10, v[0:1] offset:6
-; CHECK-NEXT:    flat_load_ubyte v11, v[0:1] offset:7
-; CHECK-NEXT:    flat_load_ubyte v12, v[0:1] offset:8
-; CHECK-NEXT:    flat_load_ubyte v13, v[0:1] offset:9
-; CHECK-NEXT:    flat_load_ubyte v14, v[0:1] offset:10
-; CHECK-NEXT:    flat_load_ubyte v15, v[0:1] offset:11
-; CHECK-NEXT:    flat_load_ubyte v16, v[0:1] offset:12
-; CHECK-NEXT:    flat_load_ubyte v17, v[0:1] offset:13
-; CHECK-NEXT:    flat_load_ubyte v18, v[0:1] offset:14
-; CHECK-NEXT:    v_mov_b32_e32 v3, s1
-; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    v_mov_b32_e32 v12, s3
+; CHECK-NEXT:    v_mov_b32_e32 v11, s2
+; CHECK-NEXT:    flat_load_ubyte v13, v[11:12] offset:46
+; CHECK-NEXT:    flat_load_ushort v14, v[11:12] offset:44
+; CHECK-NEXT:    flat_load_dwordx3 v[8:10], v[11:12] offset:32
+; CHECK-NEXT:    flat_load_dwordx4 v[0:3], v[11:12] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[11:12]
+; CHECK-NEXT:    v_mov_b32_e32 v12, s1
+; CHECK-NEXT:    v_mov_b32_e32 v11, s0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[2:3], v4
-; CHECK-NEXT:    flat_store_byte v[2:3], v5 offset:1
-; CHECK-NEXT:    flat_store_byte v[2:3], v6 offset:2
-; CHECK-NEXT:    flat_store_byte v[2:3], v7 offset:3
-; CHECK-NEXT:    flat_store_byte v[2:3], v8 offset:4
-; CHECK-NEXT:    flat_store_byte v[2:3], v9 offset:5
-; CHECK-NEXT:    flat_store_byte v[2:3], v10 offset:6
-; CHECK-NEXT:    flat_store_byte v[2:3], v11 offset:7
-; CHECK-NEXT:    flat_store_byte v[2:3], v12 offset:8
-; CHECK-NEXT:    flat_store_byte v[2:3], v13 offset:9
-; CHECK-NEXT:    flat_store_byte v[2:3], v14 offset:10
-; CHECK-NEXT:    flat_store_byte v[2:3], v15 offset:11
-; CHECK-NEXT:    flat_store_byte v[2:3], v16 offset:12
-; CHECK-NEXT:    flat_store_byte v[2:3], v17 offset:13
-; CHECK-NEXT:    flat_store_byte v[2:3], v18 offset:14
-; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[0:1] offset:29
-; CHECK-NEXT:    flat_load_ubyte v6, v[0:1] offset:28
-; CHECK-NEXT:    flat_load_ubyte v7, v[0:1] offset:27
-; CHECK-NEXT:    flat_load_ubyte v8, v[0:1] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[0:1] offset:25
-; CHECK-NEXT:    flat_load_ubyte v10, v[0:1] offset:24
-; CHECK-NEXT:    flat_load_ubyte v11, v[0:1] offset:23
-; CHECK-NEXT:    flat_load_ubyte v12, v[0:1] offset:22
-; CHECK-NEXT:    flat_load_ubyte v13, v[0:1] offset:21
-; CHECK-NEXT:    flat_load_ubyte v14, v[0:1] offset:20
-; CHECK-NEXT:    flat_load_ubyte v15, v[0:1] offset:19
-; CHECK-NEXT:    flat_load_ubyte v16, v[0:1] offset:18
-; CHECK-NEXT:    flat_load_ubyte v17, v[0:1] offset:17
-; CHECK-NEXT:    flat_load_ubyte v18, v[0:1] offset:16
-; CHECK-NEXT:    flat_load_ubyte v19, v[0:1] offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:30
-; CHECK-NEXT:    flat_store_byte v[2:3], v5 offset:29
-; CHECK-NEXT:    flat_store_byte v[2:3], v6 offset:28
-; CHECK-NEXT:    flat_store_byte v[2:3], v7 offset:27
-; CHECK-NEXT:    flat_store_byte v[2:3], v8 offset:26
-; CHECK-NEXT:    flat_store_byte v[2:3], v9 offset:25
-; CHECK-NEXT:    flat_store_byte v[2:3], v10 offset:24
-; CHECK-NEXT:    flat_store_byte v[2:3], v11 offset:23
-; CHECK-NEXT:    flat_store_byte v[2:3], v12 offset:22
-; CHECK-NEXT:    flat_store_byte v[2:3], v13 offset:21
-; CHECK-NEXT:    flat_store_byte v[2:3], v14 offset:20
-; CHECK-NEXT:    flat_store_byte v[2:3], v15 offset:19
-; CHECK-NEXT:    flat_store_byte v[2:3], v16 offset:18
-; CHECK-NEXT:    flat_store_byte v[2:3], v17 offset:17
-; CHECK-NEXT:    flat_store_byte v[2:3], v18 offset:16
-; CHECK-NEXT:    flat_store_byte v[2:3], v19 offset:15
-; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:46
-; CHECK-NEXT:    flat_load_ubyte v5, v[0:1] offset:45
-; CHECK-NEXT:    flat_load_ubyte v6, v[0:1] offset:44
-; CHECK-NEXT:    flat_load_ubyte v7, v[0:1] offset:43
-; CHECK-NEXT:    flat_load_ubyte v8, v[0:1] offset:42
-; CHECK-NEXT:    flat_load_ubyte v9, v[0:1] offset:41
-; CHECK-NEXT:    flat_load_ubyte v10, v[0:1] offset:40
-; CHECK-NEXT:    flat_load_ubyte v11, v[0:1] offset:39
-; CHECK-NEXT:    flat_load_ubyte v12, v[0:1] offset:38
-; CHECK-NEXT:    flat_load_ubyte v13, v[0:1] offset:37
-; CHECK-NEXT:    flat_load_ubyte v14, v[0:1] offset:36
-; CHECK-NEXT:    flat_load_ubyte v15, v[0:1] offset:35
-; CHECK-NEXT:    flat_load_ubyte v16, v[0:1] offset:34
-; CHECK-NEXT:    flat_load_ubyte v17, v[0:1] offset:33
-; CHECK-NEXT:    flat_load_ubyte v18, v[0:1] offset:32
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_load_ubyte v0, v[0:1] offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:46
-; CHECK-NEXT:    flat_store_byte v[2:3], v5 offset:45
-; CHECK-NEXT:    flat_store_byte v[2:3], v6 offset:44
-; CHECK-NEXT:    flat_store_byte v[2:3], v7 offset:43
-; CHECK-NEXT:    flat_store_byte v[2:3], v8 offset:42
-; CHECK-NEXT:    flat_store_byte v[2:3], v9 offset:41
-; CHECK-NEXT:    flat_store_byte v[2:3], v10 offset:40
-; CHECK-NEXT:    flat_store_byte v[2:3], v11 offset:39
-; CHECK-NEXT:    flat_store_byte v[2:3], v12 offset:38
-; CHECK-NEXT:    flat_store_byte v[2:3], v13 offset:37
-; CHECK-NEXT:    flat_store_byte v[2:3], v14 offset:36
-; CHECK-NEXT:    flat_store_byte v[2:3], v15 offset:35
-; CHECK-NEXT:    flat_store_byte v[2:3], v16 offset:34
-; CHECK-NEXT:    flat_store_byte v[2:3], v17 offset:33
-; CHECK-NEXT:    flat_store_byte v[2:3], v18 offset:32
-; CHECK-NEXT:    flat_store_byte v[2:3], v0 offset:31
+; CHECK-NEXT:    flat_store_byte v[11:12], v13 offset:46
+; CHECK-NEXT:    flat_store_short v[11:12], v14 offset:44
+; CHECK-NEXT:    flat_store_dwordx3 v[11:12], v[8:10] offset:32
+; CHECK-NEXT:    flat_store_dwordx4 v[11:12], v[0:3] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[11:12], v[4:7]
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 47, i1 false)
@@ -1431,375 +383,59 @@ define amdgpu_kernel void @memcpy_p5_p4_optsize(ptr addrspace(5) %local, ptr add
 ; CHECK-NEXT:    s_mov_b64 s[16:17], s[0:1]
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x8
 ; CHECK-NEXT:    s_load_dword s2, s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v24, 0
 ; CHECK-NEXT:    s_add_u32 s16, s16, s13
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:15
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:14
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:13
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:12
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:11
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:10
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:9
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:8
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:7
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:6
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:5
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:4
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:3
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:2
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:1
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1]
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:31
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v24, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v24, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v24, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v24, s[0:1] offset:32
 ; CHECK-NEXT:    s_addc_u32 s17, s17, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, s2
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:10
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:23
+; CHECK-NEXT:    v_mov_b32_e32 v25, s2
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:124
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:120
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:116
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(9)
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:108
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:104
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:100
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1]
+; CHECK-NEXT:    s_waitcnt vmcnt(13)
+; CHECK-NEXT:    buffer_store_dword v11, v25, s[16:19], 0 offen offset:92
+; CHECK-NEXT:    buffer_store_dword v10, v25, s[16:19], 0 offen offset:88
+; CHECK-NEXT:    buffer_store_dword v9, v25, s[16:19], 0 offen offset:84
+; CHECK-NEXT:    buffer_store_dword v8, v25, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(16)
+; CHECK-NEXT:    buffer_store_dword v15, v25, s[16:19], 0 offen offset:76
+; CHECK-NEXT:    buffer_store_dword v14, v25, s[16:19], 0 offen offset:72
+; CHECK-NEXT:    buffer_store_dword v13, v25, s[16:19], 0 offen offset:68
+; CHECK-NEXT:    buffer_store_dword v12, v25, s[16:19], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:9
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:7
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:20
+; CHECK-NEXT:    buffer_store_dword v19, v25, s[16:19], 0 offen offset:60
+; CHECK-NEXT:    buffer_store_dword v18, v25, s[16:19], 0 offen offset:56
+; CHECK-NEXT:    buffer_store_dword v17, v25, s[16:19], 0 offen offset:52
+; CHECK-NEXT:    buffer_store_dword v16, v25, s[16:19], 0 offen offset:48
 ; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:6
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:5
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:2
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:47
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:17
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:3
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:16
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:27
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:26
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:25
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:24
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:45
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:44
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:43
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:23
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:36
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:22
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:35
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:21
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:34
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:33
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:19
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:32
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:28
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:29
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:42
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:18
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:63
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:16
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:61
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:27
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:40
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:26
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:39
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:25
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:38
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:37
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:57
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:43
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:56
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:45
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:58
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:49
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:35
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:48
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:46
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:47
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:60
-; CHECK-NEXT:    s_waitcnt vmcnt(33)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:34
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:79
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:41
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:42
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:55
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:33
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:32
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:77
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:61
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:74
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:53
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:39
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:52
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:38
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:51
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:37
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:50
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:57
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:70
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:69
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:58
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:71
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:49
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:48
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:93
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:46
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:59
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:73
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:41
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:54
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:55
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:68
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:74
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:87
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:53
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:66
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:65
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:51
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:64
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:62
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:63
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:76
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:50
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:95
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:77
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:90
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:71
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:83
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:70
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:69
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:59
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:72
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:73
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:85
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:54
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:67
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:81
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:66
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:111
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:65
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:110
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:64
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:109
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:62
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:75
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:76
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:89
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:90
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:103
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:86
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:84
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:82
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:87
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:100
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:67
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:80
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:78
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:94
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:79
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:92
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:95
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:108
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:93
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:106
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:75
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:88
+; CHECK-NEXT:    buffer_store_dword v23, v25, s[16:19], 0 offen offset:44
+; CHECK-NEXT:    buffer_store_dword v22, v25, s[16:19], 0 offen offset:40
+; CHECK-NEXT:    buffer_store_dword v21, v25, s[16:19], 0 offen offset:36
+; CHECK-NEXT:    buffer_store_dword v20, v25, s[16:19], 0 offen offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_dword v3, v25, s[16:19], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v2, v25, s[16:19], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v1, v25, s[16:19], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v0, v25, s[16:19], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:89
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:102
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:78
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:91
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:94
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:107
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:105
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:101
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:91
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:104
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:86
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:85
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:83
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:82
-; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:96
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:97
-; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:98
-; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:99
-; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:120
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:81
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:80
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:111
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:110
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:109
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:121
-; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:122
-; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:123
-; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:124
-; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:125
-; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:126
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:107
-; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:127
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:106
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:105
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:103
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:102
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:101
-; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:116
-; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:117
-; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:119
-; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:114
-; CHECK-NEXT:    s_waitcnt vmcnt(34)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:118
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:115
-; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:113
-; CHECK-NEXT:    global_load_ubyte v21, v0, s[0:1] offset:112
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v18, v1, s[16:19], 0 offen offset:99
-; CHECK-NEXT:    buffer_store_byte v17, v1, s[16:19], 0 offen offset:98
-; CHECK-NEXT:    buffer_store_byte v16, v1, s[16:19], 0 offen offset:97
-; CHECK-NEXT:    buffer_store_byte v15, v1, s[16:19], 0 offen offset:96
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v7, v1, s[16:19], 0 offen offset:127
-; CHECK-NEXT:    buffer_store_byte v6, v1, s[16:19], 0 offen offset:126
-; CHECK-NEXT:    buffer_store_byte v5, v1, s[16:19], 0 offen offset:125
-; CHECK-NEXT:    buffer_store_byte v4, v1, s[16:19], 0 offen offset:124
-; CHECK-NEXT:    buffer_store_byte v3, v1, s[16:19], 0 offen offset:123
-; CHECK-NEXT:    buffer_store_byte v2, v1, s[16:19], 0 offen offset:122
-; CHECK-NEXT:    buffer_store_byte v20, v1, s[16:19], 0 offen offset:121
-; CHECK-NEXT:    buffer_store_byte v19, v1, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v11, v1, s[16:19], 0 offen offset:119
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v10, v1, s[16:19], 0 offen offset:118
-; CHECK-NEXT:    buffer_store_byte v9, v1, s[16:19], 0 offen offset:117
-; CHECK-NEXT:    buffer_store_byte v8, v1, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v13, v1, s[16:19], 0 offen offset:115
-; CHECK-NEXT:    buffer_store_byte v12, v1, s[16:19], 0 offen offset:114
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v1, s[16:19], 0 offen offset:113
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v21, v1, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    buffer_store_dword v7, v25, s[16:19], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v6, v25, s[16:19], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v5, v25, s[16:19], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v4, v25, s[16:19], 0 offen
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)
@@ -1815,363 +451,57 @@ define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %
 ; CHECK-NEXT:    s_add_u32 s16, s16, s13
 ; CHECK-NEXT:    s_addc_u32 s17, s17, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, s0
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:2
+; CHECK-NEXT:    v_mov_b32_e32 v26, s0
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:120
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:108
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:104
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:100
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen offset:96
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, s0
-; CHECK-NEXT:    v_mov_b32_e32 v1, s1
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:23
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:22
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:21
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:20
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:19
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:18
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:17
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:47
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:31
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:16
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:45
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:23
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:37
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:22
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:36
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:21
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:35
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:20
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:34
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:19
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:33
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:18
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:32
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:29
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:30
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:44
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:17
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:63
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:28
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:42
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:26
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:40
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:25
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:39
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:24
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:38
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:27
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:41
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:45
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:59
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:37
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:51
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:36
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:50
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:35
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:49
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:34
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:48
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:46
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:47
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:61
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:29
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:43
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:44
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:58
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:33
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:79
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:32
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:42
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:56
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:40
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:54
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:39
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:53
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:38
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:52
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:41
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:55
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:59
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:73
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:51
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:65
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:50
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:64
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:62
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:63
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:77
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:46
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:60
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:61
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:75
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:43
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:57
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:58
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:72
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:49
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:95
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:48
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:56
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:70
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:54
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:68
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:53
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:67
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:52
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:66
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:55
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:69
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:73
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:87
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:65
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:111
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:64
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:110
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:62
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:76
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:77
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:91
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:60
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:74
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:75
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:89
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:57
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:71
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:72
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:86
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:70
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:84
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:68
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:83
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:67
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:81
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:66
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:80
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:78
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:79
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:93
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:69
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:82
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:87
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:101
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:76
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:90
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:91
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:105
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:74
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:88
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:89
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:103
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:71
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:85
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:86
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:100
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:78
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:92
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:93
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:107
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:90
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:104
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:88
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:102
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:85
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:99
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:94
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:95
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:109
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:92
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:106
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:94
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:108
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:84
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:83
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:82
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:81
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[16:19], 0 offen offset:96
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[16:19], 0 offen offset:97
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[16:19], 0 offen offset:98
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[16:19], 0 offen offset:120
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:80
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:111
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:110
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:109
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:99
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[16:19], 0 offen offset:121
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[16:19], 0 offen offset:122
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[16:19], 0 offen offset:123
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[16:19], 0 offen offset:124
+; CHECK-NEXT:    buffer_load_dword v8, v26, s[16:19], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v9, v26, s[16:19], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v10, v26, s[16:19], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v11, v26, s[16:19], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v12, v26, s[16:19], 0 offen offset:32
+; CHECK-NEXT:    buffer_load_dword v13, v26, s[16:19], 0 offen offset:36
+; CHECK-NEXT:    buffer_load_dword v14, v26, s[16:19], 0 offen offset:40
+; CHECK-NEXT:    buffer_load_dword v15, v26, s[16:19], 0 offen offset:44
+; CHECK-NEXT:    buffer_load_dword v16, v26, s[16:19], 0 offen offset:48
+; CHECK-NEXT:    buffer_load_dword v17, v26, s[16:19], 0 offen offset:52
+; CHECK-NEXT:    buffer_load_dword v18, v26, s[16:19], 0 offen offset:56
+; CHECK-NEXT:    buffer_load_dword v19, v26, s[16:19], 0 offen offset:60
+; CHECK-NEXT:    buffer_load_dword v23, v26, s[16:19], 0 offen offset:92
+; CHECK-NEXT:    buffer_load_dword v22, v26, s[16:19], 0 offen offset:88
+; CHECK-NEXT:    buffer_load_dword v21, v26, s[16:19], 0 offen offset:84
+; CHECK-NEXT:    buffer_load_dword v20, v26, s[16:19], 0 offen offset:80
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v25, s1
+; CHECK-NEXT:    v_mov_b32_e32 v24, s0
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:112
+; CHECK-NEXT:    buffer_load_dword v3, v26, s[16:19], 0 offen offset:76
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:107
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:105
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:104
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:103
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:106
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:102
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:101
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:100
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[16:19], 0 offen offset:126
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[16:19], 0 offen offset:116
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[16:19], 0 offen offset:117
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[16:19], 0 offen offset:118
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[16:19], 0 offen offset:119
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[16:19], 0 offen offset:127
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[16:19], 0 offen offset:114
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[16:19], 0 offen offset:115
+; CHECK-NEXT:    buffer_load_dword v2, v26, s[16:19], 0 offen offset:72
+; CHECK-NEXT:    buffer_load_dword v1, v26, s[16:19], 0 offen offset:68
+; CHECK-NEXT:    buffer_load_dword v0, v26, s[16:19], 0 offen offset:64
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:108
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[16:19], 0 offen offset:125
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[16:19], 0 offen offset:113
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[16:19], 0 offen offset:112
-; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:98
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:97
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:96
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:127
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:126
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7] offset:96
+; CHECK-NEXT:    buffer_load_dword v4, v26, s[16:19], 0 offen
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    buffer_load_dword v5, v26, s[16:19], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v6, v26, s[16:19], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v26, s[16:19], 0 offen offset:12
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[20:23] offset:80
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[0:3] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[16:19] offset:48
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[12:15] offset:32
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[8:11] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:125
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:124
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:123
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:122
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:121
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:120
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:119
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:118
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:117
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:116
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:115
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:114
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:113
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:112
+; CHECK-NEXT:    flat_store_dwordx4 v[24:25], v[4:7]
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
@@ -2218,279 +548,27 @@ define amdgpu_kernel void @memcpy_p0_p3_optsize(ptr %generic) #1 {
 ; CHECK-LABEL: memcpy_p0_p3_optsize:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:112
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:113
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:114
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:115
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:116
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:117
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:118
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:119
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v0, s0
-; CHECK-NEXT:    v_mov_b32_e32 v1, s1
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:112
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:113
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:114
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:115
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:116
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:117
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:118
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:119
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:120
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:121
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:122
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:123
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:124
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:125
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:126
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:127
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:120
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:121
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:122
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:123
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:124
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:125
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:126
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:127
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:96
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:97
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:98
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:99
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:100
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:101
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:102
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:103
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:96
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:97
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:98
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:99
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:100
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:101
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:102
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:103
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:104
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:105
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:106
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:107
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:108
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:109
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:110
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:111
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:104
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:105
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:106
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:107
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:108
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:109
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:110
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:111
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:80
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:81
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:82
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:83
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:84
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:85
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:86
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:87
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:80
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:81
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:82
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:83
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:84
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:85
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:86
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:87
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:88
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:89
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:90
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:91
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:92
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:93
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:94
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:95
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:88
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:89
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:90
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:91
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:92
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:93
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:94
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:95
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:64
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:65
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:66
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:67
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:68
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:69
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:70
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:71
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:64
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:65
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:66
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:67
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:68
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:69
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:70
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:71
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:72
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:73
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:74
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:75
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:76
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:77
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:78
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:79
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:72
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:73
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:74
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:75
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:76
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:77
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:78
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:79
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:48
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:49
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:50
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:51
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:52
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:53
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:54
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:55
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:48
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:49
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:50
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:51
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:52
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:53
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:54
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:55
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:56
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:57
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:58
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:59
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:60
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:61
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:62
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:63
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:56
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:57
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:58
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:59
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:60
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:61
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:62
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:63
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:32
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:33
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:34
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:35
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:36
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:37
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:38
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:39
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:32
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:33
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:34
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:35
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:36
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:37
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:38
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:39
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:40
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:41
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:42
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:43
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:44
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:45
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:46
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:47
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:40
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:41
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:42
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:43
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:44
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:45
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:46
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:47
-; CHECK-NEXT:    ds_read_u8 v3, v2
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:7
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v17, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v18, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:16
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:17
-; CHECK-NEXT:    ds_read_u8 v21, v2 offset:18
-; CHECK-NEXT:    ds_read_u8 v22, v2 offset:19
-; CHECK-NEXT:    ds_read_u8 v23, v2 offset:20
-; CHECK-NEXT:    ds_read_u8 v24, v2 offset:21
-; CHECK-NEXT:    ds_read_u8 v25, v2 offset:22
-; CHECK-NEXT:    ds_read_u8 v26, v2 offset:23
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:19
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:23
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:24
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:25
-; CHECK-NEXT:    ds_read_u8 v21, v2 offset:26
-; CHECK-NEXT:    ds_read_u8 v22, v2 offset:27
-; CHECK-NEXT:    ds_read_u8 v23, v2 offset:28
-; CHECK-NEXT:    ds_read_u8 v24, v2 offset:29
-; CHECK-NEXT:    ds_read_u8 v25, v2 offset:30
-; CHECK-NEXT:    ds_read_u8 v2, v2 offset:31
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:27
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:28
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:31
-; CHECK-NEXT:    flat_store_byte v[0:1], v3
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:1
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:2
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:3
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:4
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:5
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:6
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:7
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:9
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:10
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:11
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:12
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:13
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:14
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:15
+; CHECK-NEXT:    v_mov_b32_e32 v16, 0
+; CHECK-NEXT:    ds_read2_b64 v[0:3], v16 offset1:1
+; CHECK-NEXT:    ds_read2_b64 v[4:7], v16 offset0:2 offset1:3
+; CHECK-NEXT:    ds_read2_b64 v[8:11], v16 offset0:4 offset1:5
+; CHECK-NEXT:    ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v21, s1
+; CHECK-NEXT:    v_mov_b32_e32 v20, s0
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[0:3]
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[4:7] offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[8:11] offset:32
+; CHECK-NEXT:    ds_read2_b64 v[0:3], v16 offset0:8 offset1:9
+; CHECK-NEXT:    ds_read2_b64 v[4:7], v16 offset0:10 offset1:11
+; CHECK-NEXT:    ds_read2_b64 v[8:11], v16 offset0:12 offset1:13
+; CHECK-NEXT:    ds_read2_b64 v[16:19], v16 offset0:14 offset1:15
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[12:15] offset:48
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[0:3] offset:64
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[4:7] offset:80
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[8:11] offset:96
+; CHECK-NEXT:    flat_store_dwordx4 v[20:21], v[16:19] offset:112
 ; CHECK-NEXT:    s_endpgm
 entry:
   tail call void @llvm.memcpy.p0.p3.i64(ptr %generic, ptr addrspace(3) @shared, i64 128, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
index 7575782c1b2acd92b312488b8478e0f1a7a62442..cadc3dadb0a1e9abd0b820a848cd456659a4915d 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
@@ -13,55 +13,9 @@ define void @memcpy_p0_p0_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -73,101 +27,19 @@ define void @memcpy_p0_p0_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xe
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:15
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:30
+; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[6:8], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -179,104 +51,13 @@ define void @memcpy_p0_p0_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:31
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:16
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -288,31 +69,9 @@ define void @memcpy_p0_p0_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -324,55 +83,19 @@ define void @memcpy_p0_p0_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v15, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v16, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v17, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v18, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:30
+; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[6:8], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -384,55 +107,13 @@ define void @memcpy_p0_p0_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v15, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v16, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v17, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v18, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -458,58 +139,13 @@ define void @memcpy_p0_p0_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p0_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:15
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -553,58 +189,13 @@ define void @memcpy_p0_p0_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p0_p0_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(16)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:15
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -634,55 +225,9 @@ define void @memcpy_p0_p1_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p1_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -694,101 +239,19 @@ define void @memcpy_p0_p1_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p1_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xe
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:30
+; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:28
+; CHECK-NEXT:    global_load_dwordx3 v[6:8], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:18
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:17
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:16
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:15
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -800,104 +263,13 @@ define void @memcpy_p0_p1_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p1_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v19, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:31
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:18
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:17
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -909,31 +281,9 @@ define void @memcpy_p0_p1_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p1_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:2
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -945,55 +295,19 @@ define void @memcpy_p0_p1_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p1_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:30
+; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:28
+; CHECK-NEXT:    global_load_dwordx3 v[6:8], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1005,55 +319,13 @@ define void @memcpy_p0_p1_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p1_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1080,35 +352,12 @@ define void @memcpy_p0_p1_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addr
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off
-; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:15
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:15
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v8 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:15
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v9 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:19
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v10 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:23
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v11 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 8, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 24, v11
-; CHECK-NEXT:    v_lshrrev_b32_e32 v9, 8, v11
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:28
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1153,35 +402,12 @@ define void @memcpy_p0_p1_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr a
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off
-; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:15
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:15
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v8 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:15
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v9 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:19
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v10 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:23
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v11 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 8, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 24, v11
-; CHECK-NEXT:    v_lshrrev_b32_e32 v9, 8, v11
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:28
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1211,54 +437,9 @@ define void @memcpy_p0_p3_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:7
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v17, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1270,96 +451,19 @@ define void @memcpy_p0_p3_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:7
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v17, v2
-; CHECK-NEXT:    ds_read_u8 v18, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:16
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:17
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:11
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:9
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:5
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:3
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:30
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:29
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:28
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:27
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:26
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:25
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:24
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:23
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:22
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:21
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:20
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:19
-; CHECK-NEXT:    ds_read_u8 v2, v2 offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:29
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:27
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:25
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:23
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:21
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:19
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:15
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
+; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
+; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1371,100 +475,12 @@ define void @memcpy_p0_p3_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:7
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v17, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v18, v2
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:16
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:17
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:31
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:30
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:29
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:28
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:27
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:26
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:25
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:24
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:23
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:22
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:21
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:20
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:19
-; CHECK-NEXT:    ds_read_u8 v2, v2 offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:31
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
+; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset0:2 offset1:3
+; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1476,30 +492,9 @@ define void @memcpy_p0_p3_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v3, v2 offset:14
-; CHECK-NEXT:    ds_read_u16 v4, v2 offset:12
-; CHECK-NEXT:    ds_read_u16 v5, v2 offset:10
-; CHECK-NEXT:    ds_read_u16 v6, v2 offset:8
-; CHECK-NEXT:    ds_read_u16 v7, v2 offset:6
-; CHECK-NEXT:    ds_read_u16 v8, v2 offset:4
-; CHECK-NEXT:    ds_read_u16 v9, v2 offset:2
-; CHECK-NEXT:    ds_read_u16 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1511,54 +506,19 @@ define void @memcpy_p0_p3_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:30
-; CHECK-NEXT:    ds_read_u16 v4, v2 offset:28
-; CHECK-NEXT:    ds_read_u16 v5, v2 offset:26
-; CHECK-NEXT:    ds_read_u16 v6, v2 offset:24
-; CHECK-NEXT:    ds_read_u16 v7, v2 offset:22
-; CHECK-NEXT:    ds_read_u16 v8, v2 offset:20
-; CHECK-NEXT:    ds_read_u16 v9, v2 offset:18
-; CHECK-NEXT:    ds_read_u16 v10, v2 offset:16
-; CHECK-NEXT:    ds_read_u16 v11, v2 offset:14
-; CHECK-NEXT:    ds_read_u16 v12, v2 offset:12
-; CHECK-NEXT:    ds_read_u16 v13, v2 offset:10
-; CHECK-NEXT:    ds_read_u16 v14, v2 offset:8
-; CHECK-NEXT:    ds_read_u16 v15, v2 offset:6
-; CHECK-NEXT:    ds_read_u16 v16, v2 offset:4
-; CHECK-NEXT:    ds_read_u16 v17, v2 offset:2
-; CHECK-NEXT:    ds_read_u16 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
+; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
+; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1570,54 +530,12 @@ define void @memcpy_p0_p3_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v3, v2 offset:30
-; CHECK-NEXT:    ds_read_u16 v4, v2 offset:28
-; CHECK-NEXT:    ds_read_u16 v5, v2 offset:26
-; CHECK-NEXT:    ds_read_u16 v6, v2 offset:24
-; CHECK-NEXT:    ds_read_u16 v7, v2 offset:22
-; CHECK-NEXT:    ds_read_u16 v8, v2 offset:20
-; CHECK-NEXT:    ds_read_u16 v9, v2 offset:18
-; CHECK-NEXT:    ds_read_u16 v10, v2 offset:16
-; CHECK-NEXT:    ds_read_u16 v11, v2 offset:14
-; CHECK-NEXT:    ds_read_u16 v12, v2 offset:12
-; CHECK-NEXT:    ds_read_u16 v13, v2 offset:10
-; CHECK-NEXT:    ds_read_u16 v14, v2 offset:8
-; CHECK-NEXT:    ds_read_u16 v15, v2 offset:6
-; CHECK-NEXT:    ds_read_u16 v16, v2 offset:4
-; CHECK-NEXT:    ds_read_u16 v17, v2 offset:2
-; CHECK-NEXT:    ds_read_u16 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset0:2 offset1:3
+; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1643,35 +561,12 @@ define void @memcpy_p0_p3_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p3_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset1:1
-; CHECK-NEXT:    ds_read_b128 v[7:10], v2 offset:15
+; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:15
+; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v7 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:15
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v8 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:19
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v9 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:23
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v10 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v9, 8, v10
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:28
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1714,35 +609,12 @@ define void @memcpy_p0_p3_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p0_p3_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_b128 v[3:6], v2
-; CHECK-NEXT:    ds_read_b128 v[7:10], v2 offset:15
+; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:15
+; CHECK-NEXT:    ds_read_b128 v[7:10], v2
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v7 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:15
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v8 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:19
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v9 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:23
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v10 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v9, 8, v10
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:28
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1771,55 +643,12 @@ define void @memcpy_p0_p4_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p4_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1831,100 +660,24 @@ define void @memcpy_p0_p4_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p4_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:1
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:2
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:3
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:4
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:5
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:6
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:7
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:8
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:9
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:10
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:11
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:12
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:13
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:16
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:8
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:16
+; CHECK-NEXT:    global_load_dword v4, v[2:3], off offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dword v[0:1], v4 offset:24
+; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
+; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:15
+; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1936,104 +689,18 @@ define void @memcpy_p0_p4_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p4_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v19, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:31
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:17
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:16
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:8
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:16
+; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:24
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2045,30 +712,12 @@ define void @memcpy_p0_p4_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p4_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v4
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:2
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:4
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:6
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:8
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:10
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:10
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:12
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:12
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2 offset:14
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2080,55 +729,24 @@ define void @memcpy_p0_p4_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p4_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:8
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:16
+; CHECK-NEXT:    global_load_dword v4, v[2:3], off offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dword v[0:1], v4 offset:24
+; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
+; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2140,55 +758,18 @@ define void @memcpy_p0_p4_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p4_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:8
+; CHECK-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[4:5] offset:16
+; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:24
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2219,30 +800,7 @@ define void @memcpy_p0_p4_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addr
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
 ; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v2 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:15
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v3 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:19
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v4 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:23
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v5 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v3
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 8, v3
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 24, v4
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
-; CHECK-NEXT:    v_lshrrev_b32_e32 v9, 24, v5
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:28
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2290,30 +848,7 @@ define void @memcpy_p0_p4_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr a
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
 ; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v2 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:15
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v3 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:19
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v4 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:23
-; CHECK-NEXT:    flat_store_byte_d16_hi v[0:1], v5 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v3
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 8, v3
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 24, v4
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v4
-; CHECK-NEXT:    v_lshrrev_b32_e32 v9, 24, v5
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 8, v5
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:16
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:24
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:28
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2342,55 +877,13 @@ define void @memcpy_p0_p5_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2402,99 +895,23 @@ define void @memcpy_p0_p5_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x11
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17
-; CHECK-NEXT:    s_clause 0xc
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:24
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:23
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:19
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:15
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2506,103 +923,19 @@ define void @memcpy_p0_p5_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x11
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18
-; CHECK-NEXT:    s_clause 0xd
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2614,31 +947,13 @@ define void @memcpy_p0_p5_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:2
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2650,55 +965,23 @@ define void @memcpy_p0_p5_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v12, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v13, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v14, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v15, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v16, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v17, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2710,55 +993,19 @@ define void @memcpy_p0_p5_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v12, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v13, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v14, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v15, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v16, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v17, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2788,53 +1035,19 @@ define void @memcpy_p0_p5_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p0_p5_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    s_clause 0x7
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:15
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:19
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:23
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:27
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2888,53 +1101,19 @@ define void @memcpy_p0_p5_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p0_p5_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    s_clause 0x7
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:15
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:19
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:23
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:27
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2970,41 +1149,8 @@ define void @memcpy_p1_p0_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3]
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 8, v10
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v11, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v12, 8, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 8, v15
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v16, 8, v17
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v18, 8, v2
-; CHECK-NEXT:    v_lshl_or_b32 v2, v6, 16, v5
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 16, v7
-; CHECK-NEXT:    v_lshl_or_b32 v4, v10, 16, v9
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -3016,79 +1162,15 @@ define void @memcpy_p1_p0_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v22, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v23, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v24, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v25, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v26, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v27, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v28, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v29, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v30, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v31, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v32, v[2:3]
-; CHECK-NEXT:    flat_load_ubyte v33, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 8, v9
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v10, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v11, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v13, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(19)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v7, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v15, 8, v16
-; CHECK-NEXT:    v_lshl_or_b32 v7, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v17, 8, v18
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v19, 8, v20
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v21, 8, v22
-; CHECK-NEXT:    v_lshl_or_b32 v9, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v23, 8, v24
-; CHECK-NEXT:    v_lshl_or_b32 v5, v11, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v25, 8, v26
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v27, 8, v28
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v29, 8, v30
-; CHECK-NEXT:    v_lshl_or_b32 v4, v14, 16, v12
+; CHECK-NEXT:    s_clause 0x2
+; CHECK-NEXT:    flat_load_dwordx2 v[6:7], v[2:3] offset:23
+; CHECK-NEXT:    flat_load_dwordx2 v[8:9], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v31, 8, v32
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v33, 8, v2
-; CHECK-NEXT:    v_lshl_or_b32 v3, v17, 16, v16
-; CHECK-NEXT:    v_lshl_or_b32 v2, v2, 16, v18
 ; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
 ; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -3100,79 +1182,13 @@ define void @memcpy_p1_p0_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:31
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v22, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v23, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v24, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v25, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v26, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v27, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v28, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v29, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v30, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v31, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v32, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v33, v[2:3]
-; CHECK-NEXT:    flat_load_ubyte v34, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 8, v10
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
-; CHECK-NEXT:    v_lshl_or_b32 v6, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v11, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v12, 8, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v14, 8, v15
-; CHECK-NEXT:    v_lshl_or_b32 v5, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v16, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v18, 8, v19
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v20, 8, v21
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v22, 8, v23
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v24, 8, v25
-; CHECK-NEXT:    v_lshl_or_b32 v9, v12, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v26, 8, v27
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v28, 8, v29
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v30, 8, v31
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v32, 8, v33
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v19, v34, 8, v2
-; CHECK-NEXT:    v_lshl_or_b32 v2, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v16
-; CHECK-NEXT:    v_lshl_or_b32 v6, v19, 16, v18
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -3183,23 +1199,8 @@ define void @memcpy_p1_p0_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3]
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v9, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 16, v6
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v7
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -3211,52 +1212,16 @@ define void @memcpy_p1_p0_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ushort v19, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v20, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v21, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v22, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v23, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v8, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v10, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v4, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v8, v8, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v15, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v17, 8, v18
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v19, 16, v20
+; CHECK-NEXT:    s_clause 0x2
+; CHECK-NEXT:    flat_load_dwordx2 v[6:7], v[2:3] offset:23
+; CHECK-NEXT:    flat_load_dwordx2 v[8:9], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v21, 16, v22
-; CHECK-NEXT:    v_lshl_or_b32 v9, v13, 16, v12
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v23, 16, v2
-; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:16
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:23
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -3267,39 +1232,13 @@ define void @memcpy_p1_p0_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v15, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ushort v16, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v17, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v18, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v19, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v4, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 16, v9
-; CHECK-NEXT:    v_lshl_or_b32 v9, v7, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v12, 16, v13
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 16, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v14, 16, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v16, 16, v17
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v18, 16, v19
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -3323,47 +1262,13 @@ define void @memcpy_p1_p0_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p0_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v7, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v9, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v11, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v13, 8, v14
-; CHECK-NEXT:    v_lshl_or_b32 v8, v7, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v15, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v17, 8, v18
-; CHECK-NEXT:    v_lshl_or_b32 v9, v11, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v19, 8, v20
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:15
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 8, v21
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 16, v15
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:15
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 31, i1 false)
@@ -3404,47 +1309,13 @@ define void @memcpy_p1_p0_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p1_p0_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v7, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v9, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v11, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v13, 8, v14
-; CHECK-NEXT:    v_lshl_or_b32 v8, v7, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v15, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v17, 8, v18
-; CHECK-NEXT:    v_lshl_or_b32 v9, v11, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v19, 8, v20
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:15
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 8, v21
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 16, v15
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:15
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 31, i1 false)
@@ -4042,44 +1913,13 @@ define void @memcpy_p1_p5_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v4, 8, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v6, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v14, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v16, 8, v15
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v2, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v5, v9, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v10
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -4090,81 +1930,21 @@ define void @memcpy_p1_p5_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v4, 8, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v6, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v14, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v16, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v18, 8, v17
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v21, 8, v20
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v23, 8, v22
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v19, 8, v25
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v26, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v28, 8, v27
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v30, 8, v29
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v31, 8, v21
-; CHECK-NEXT:    v_lshl_or_b32 v7, v13, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v19, v2, 8, v32
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v5, v9, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v6, v14, 16, v15
-; CHECK-NEXT:    v_lshl_or_b32 v9, v17, 16, v16
-; CHECK-NEXT:    v_lshl_or_b32 v8, v19, 16, v18
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:16
-; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:23
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[7:8], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[9:10], off offset:23
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -4175,81 +1955,19 @@ define void @memcpy_p1_p5_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v4, 8, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v6, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v14, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v16, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v18, 8, v17
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v21, 8, v20
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v23, 8, v22
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v19, 8, v25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v26, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v28, 8, v27
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v30, 8, v29
-; CHECK-NEXT:    v_lshl_or_b32 v7, v13, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v32, 8, v31
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v19, v2, 8, v33
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v5, v9, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v6, v14, 16, v15
-; CHECK-NEXT:    v_lshl_or_b32 v9, v17, 16, v16
-; CHECK-NEXT:    v_lshl_or_b32 v8, v19, 16, v18
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -4260,24 +1978,13 @@ define void @memcpy_p1_p5_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v6, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 16, v7
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v10, 16, v9
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -4288,52 +1995,21 @@ define void @memcpy_p1_p5_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v19, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v20, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v21, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v22, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v6, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v14, 8, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v16, 8, v15
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v18, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v9, v9, 16, v10
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v20, 16, v19
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[7:8], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v22, 16, v21
-; CHECK-NEXT:    v_lshl_or_b32 v8, v12, 16, v11
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[6:7], off offset:16
-; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:23
+; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[9:10], off offset:23
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -4344,41 +2020,19 @@ define void @memcpy_p1_p5_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v12, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v13, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v14, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v15, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v16, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v17, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v18, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v6, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v16, 16, v15
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v18, 16, v17
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -4406,49 +2060,18 @@ define void @memcpy_p1_p5_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p1_p5_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    s_clause 0x7
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v14, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v10, v2, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v16, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v18, 8, v17
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v20, 8, v19
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v22, 8, v21
-; CHECK-NEXT:    v_lshl_or_b32 v9, v9, 16, v8
-; CHECK-NEXT:    v_lshl_or_b32 v8, v12, 16, v11
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4500,49 +2123,18 @@ define void @memcpy_p1_p5_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p1_p5_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    s_clause 0x7
 ; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v14, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v10, v2, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v16, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v18, 8, v17
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v20, 8, v19
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v22, 8, v21
-; CHECK-NEXT:    v_lshl_or_b32 v9, v9, 16, v8
-; CHECK-NEXT:    v_lshl_or_b32 v8, v12, 16, v11
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4577,41 +2169,8 @@ define void @memcpy_p3_p0_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2]
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 8, v9
-; CHECK-NEXT:    v_lshl_or_b32 v3, v5, 8, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v11, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 8, v14
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 8, v16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v17, 8, v1
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
 ; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -4624,80 +2183,16 @@ define void @memcpy_p3_p0_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v21, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v22, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v23, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v24, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v25, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v26, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v27, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v28, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v29, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v30, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v31, v[1:2]
-; CHECK-NEXT:    flat_load_ubyte v32, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 8, v8
-; CHECK-NEXT:    v_lshl_or_b32 v3, v5, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v9, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v10, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v12, 8, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(19)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 8, v15
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v16, 8, v17
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v18, 8, v19
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v20, 8, v21
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v22, 8, v23
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v24, 8, v25
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v26, 8, v27
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v28, 8, v29
+; CHECK-NEXT:    s_clause 0x2
+; CHECK-NEXT:    flat_load_dwordx2 v[5:6], v[1:2] offset:23
+; CHECK-NEXT:    flat_load_dwordx2 v[7:8], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v30, 8, v31
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v32, 8, v1
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v6, v10, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v11
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v16
-; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:23
-; CHECK-NEXT:    ds_write_b64 v0, v[3:4] offset:16
-; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[5:6] offset1:1
+; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)
+; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)
+; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4709,79 +2204,13 @@ define void @memcpy_p3_p0_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:31
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v21, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v22, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v23, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v24, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v25, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v26, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v27, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v28, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v29, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v30, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v31, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v32, v[1:2]
-; CHECK-NEXT:    flat_load_ubyte v33, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 8, v9
-; CHECK-NEXT:    v_lshl_or_b32 v3, v5, 8, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v11, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v13, 8, v14
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v15, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v17, 8, v18
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v19, 8, v20
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v21, 8, v22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v23, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v25, 8, v26
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v27, 8, v28
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v29, 8, v30
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v31, 8, v32
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v33, 8, v1
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v6, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v16
-; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[1:2] offset0:2 offset1:3
-; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[5:6] offset1:1
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4793,23 +2222,8 @@ define void @memcpy_p3_p0_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2]
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v8, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v9, 16, v5
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v10, 16, v6
 ; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -4822,51 +2236,16 @@ define void @memcpy_p3_p0_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v18, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v19, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v20, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v21, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v22, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v7, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v10, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v11, v5, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v13, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v15, 8, v16
-; CHECK-NEXT:    v_lshl_or_b32 v7, v11, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v17, 16, v18
+; CHECK-NEXT:    s_clause 0x2
+; CHECK-NEXT:    flat_load_dwordx2 v[5:6], v[1:2] offset:23
+; CHECK-NEXT:    flat_load_dwordx2 v[7:8], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v19, 16, v20
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v21, 16, v22
-; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:16
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:23
+; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)
+; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)
+; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4878,40 +2257,13 @@ define void @memcpy_p3_p0_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v13, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v14, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ushort v15, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v16, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v18, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v13, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v18
-; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[1:2] offset0:2 offset1:3
-; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[5:6] offset1:1
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4937,47 +2289,13 @@ define void @memcpy_p3_p0_sz31_align_8_8(ptr addrspace(3) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p0_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v8, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v10, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v12, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v7, v6, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v14, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v16, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v18, 8, v19
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2]
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2] offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v5, 8, v20
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 16, v11
-; CHECK-NEXT:    v_lshl_or_b32 v5, v5, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write_b128 v0, v[5:8] offset:15
+; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:1
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5021,47 +2339,13 @@ define void @memcpy_p3_p0_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p3_p0_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v8, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v10, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v12, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v7, v6, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v14, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v16, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v18, 8, v19
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:15
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v5, 8, v20
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 16, v11
-; CHECK-NEXT:    v_lshl_or_b32 v5, v5, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_write_b128 v0, v[1:4]
-; CHECK-NEXT:    ds_write_b128 v0, v[5:8] offset:15
+; CHECK-NEXT:    ds_write_b128 v0, v[3:6] offset:15
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    ds_write_b128 v0, v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5701,44 +2985,13 @@ define void @memcpy_p3_p5_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v3, 8, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v5, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v9, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v13, 8, v12
-; CHECK-NEXT:    v_lshl_or_b32 v2, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v15, 8, v14
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v1, 8, v16
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v5
-; CHECK-NEXT:    v_lshl_or_b32 v3, v10, 16, v9
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5750,81 +3003,21 @@ define void @memcpy_p3_p5_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v3, 8, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v5, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v9, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v13, 8, v12
-; CHECK-NEXT:    v_lshl_or_b32 v2, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v15, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v17, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v20, 8, v19
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v22, 8, v21
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v18, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v25, 8, v23
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v27, 8, v26
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v29, 8, v28
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v30, 8, v20
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v1, 8, v31
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v5
-; CHECK-NEXT:    v_lshl_or_b32 v3, v10, 16, v9
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v8, v16, 16, v15
-; CHECK-NEXT:    v_lshl_or_b32 v7, v18, 16, v17
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:16
-; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:23
+; CHECK-NEXT:    ds_write_b64 v0, v[6:7] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ds_write_b64 v0, v[8:9] offset:23
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5836,81 +3029,19 @@ define void @memcpy_p3_p5_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v32, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v3, 8, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v5, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v9, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v13, 8, v12
-; CHECK-NEXT:    v_lshl_or_b32 v2, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v15, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v17, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v20, 8, v19
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v22, 8, v21
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v18, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v25, 8, v23
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v27, 8, v26
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v29, 8, v28
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v31, 8, v30
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v1, 8, v32
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v5
-; CHECK-NEXT:    v_lshl_or_b32 v3, v10, 16, v9
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v8, v16, 16, v15
-; CHECK-NEXT:    v_lshl_or_b32 v7, v18, 16, v17
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[7:8] offset0:2 offset1:3
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset0:2 offset1:3
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5922,24 +3053,13 @@ define void @memcpy_p3_p5_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5951,52 +3071,21 @@ define void @memcpy_p3_p5_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v18, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v19, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v20, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v21, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v13, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v15, 8, v14
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v17, 8, v16
-; CHECK-NEXT:    v_lshl_or_b32 v8, v8, 16, v9
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v19, 16, v18
+; CHECK-NEXT:    ds_write_b64 v0, v[6:7] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v21, 16, v20
-; CHECK-NEXT:    v_lshl_or_b32 v7, v11, 16, v10
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:16
-; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:23
+; CHECK-NEXT:    ds_write_b64 v0, v[8:9] offset:23
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -6008,41 +3097,19 @@ define void @memcpy_p3_p5_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v11, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v12, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v13, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v14, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v15, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v16, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v17, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v11, 16, v10
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v14
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v16
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[7:8] offset0:2 offset1:3
+; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset0:2 offset1:3
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -6072,50 +3139,19 @@ define void @memcpy_p3_p5_sz31_align_8_8(ptr addrspace(3) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p3_p5_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 8, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v9, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v13, 8, v12
-; CHECK-NEXT:    v_lshl_or_b32 v4, v1, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v15, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v17, 8, v16
-; CHECK-NEXT:    v_lshl_or_b32 v2, v11, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v19, 8, v18
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v21, 8, v20
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v3
-; CHECK-NEXT:    v_lshl_or_b32 v1, v13, 16, v12
+; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[7:8] offset1:1
-; CHECK-NEXT:    ds_write_b128 v0, v[1:4] offset:15
+; CHECK-NEXT:    ds_write_b128 v0, v[2:5] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -6169,49 +3205,18 @@ define void @memcpy_p3_p5_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p3_p5_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    s_clause 0x7
 ; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
 ; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v9, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v13, 8, v12
-; CHECK-NEXT:    v_lshl_or_b32 v9, v1, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v15, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v17, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v19, 8, v18
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v21, 8, v20
-; CHECK-NEXT:    v_lshl_or_b32 v8, v8, 16, v7
-; CHECK-NEXT:    v_lshl_or_b32 v7, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    ds_write_b128 v0, v[2:5]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    ds_write_b128 v0, v[6:9] offset:15
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -6248,55 +3253,12 @@ define void @memcpy_p5_p0_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -6307,101 +3269,24 @@ define void @memcpy_p5_p0_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xe
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:30
+; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[5:7], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:17
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -6412,104 +3297,19 @@ define void @memcpy_p5_p0_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:31
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:17
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -6520,31 +3320,12 @@ define void @memcpy_p5_p0_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -6555,55 +3336,24 @@ define void @memcpy_p5_p0_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v13, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v14, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v15, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v16, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:30
+; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[5:7], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -6614,55 +3364,19 @@ define void @memcpy_p5_p0_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v13, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v14, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v15, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v16, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -6689,61 +3403,19 @@ define void @memcpy_p5_p0_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p0_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:15
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 31, i1 false)
@@ -6793,61 +3465,19 @@ define void @memcpy_p5_p0_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p5_p0_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x10
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:15
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 31, i1 false)
@@ -6881,55 +3511,12 @@ define void @memcpy_p5_p1_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p1_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -6940,207 +3527,47 @@ define void @memcpy_p5_p1_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p1_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:17
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 31, i1 false)
   ret void
-}
-
-define void @memcpy_p5_p1_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {
-; CHECK-LABEL: memcpy_p5_p1_sz32_align_1_1:
-; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:31
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v33, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:18
+}
+
+define void @memcpy_p5_p1_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {
+; CHECK-LABEL: memcpy_p5_p1_sz32_align_1_1:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:17
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -7151,31 +3578,12 @@ define void @memcpy_p5_p1_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p1_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -7186,55 +3594,24 @@ define void @memcpy_p5_p1_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p1_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -7245,55 +3622,19 @@ define void @memcpy_p5_p1_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p1_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -7329,30 +3670,10 @@ define void @memcpy_p5_p1_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addr
 ; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte_d16_hi v7, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte_d16_hi v8, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte_d16_hi v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte_d16_hi v10, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 8, v10
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 31, i1 false)
@@ -7411,30 +3732,10 @@ define void @memcpy_p5_p1_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr a
 ; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte_d16_hi v7, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte_d16_hi v8, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte_d16_hi v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte_d16_hi v10, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 8, v10
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 31, i1 false)
@@ -7468,54 +3769,12 @@ define void @memcpy_p5_p3_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p3_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:15
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:14
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:13
-; CHECK-NEXT:    ds_read_u8 v5, v1 offset:12
-; CHECK-NEXT:    ds_read_u8 v6, v1 offset:11
-; CHECK-NEXT:    ds_read_u8 v7, v1 offset:10
-; CHECK-NEXT:    ds_read_u8 v8, v1 offset:9
-; CHECK-NEXT:    ds_read_u8 v9, v1 offset:8
-; CHECK-NEXT:    ds_read_u8 v10, v1 offset:7
-; CHECK-NEXT:    ds_read_u8 v11, v1 offset:6
-; CHECK-NEXT:    ds_read_u8 v12, v1 offset:5
-; CHECK-NEXT:    ds_read_u8 v13, v1 offset:4
-; CHECK-NEXT:    ds_read_u8 v14, v1 offset:3
-; CHECK-NEXT:    ds_read_u8 v15, v1 offset:2
-; CHECK-NEXT:    ds_read_u8 v16, v1 offset:1
-; CHECK-NEXT:    ds_read_u8 v1, v1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -7526,85 +3785,25 @@ define void @memcpy_p5_p3_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p3_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:1
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:2
-; CHECK-NEXT:    ds_read_u8 v5, v1 offset:3
-; CHECK-NEXT:    ds_read_u8 v6, v1 offset:4
-; CHECK-NEXT:    ds_read_u8 v7, v1 offset:5
-; CHECK-NEXT:    ds_read_u8 v8, v1 offset:6
-; CHECK-NEXT:    ds_read_u8 v9, v1 offset:7
-; CHECK-NEXT:    ds_read_u8 v10, v1 offset:8
-; CHECK-NEXT:    ds_read_u8 v11, v1 offset:9
-; CHECK-NEXT:    ds_read_u8 v12, v1 offset:10
-; CHECK-NEXT:    ds_read_u8 v13, v1 offset:11
-; CHECK-NEXT:    ds_read_u8 v14, v1 offset:12
-; CHECK-NEXT:    ds_read_u8 v15, v1 offset:13
-; CHECK-NEXT:    ds_read_u8 v16, v1 offset:14
-; CHECK-NEXT:    ds_read_u8 v17, v1 offset:15
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:24
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:25
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u8 v18, v1 offset:27
-; CHECK-NEXT:    ds_read_u8 v19, v1 offset:28
-; CHECK-NEXT:    ds_read_u8 v20, v1 offset:29
-; CHECK-NEXT:    ds_read_u8 v21, v1 offset:30
-; CHECK-NEXT:    ds_read_u8 v22, v1 offset:16
-; CHECK-NEXT:    ds_read_u8 v23, v1 offset:17
-; CHECK-NEXT:    ds_read_u8 v24, v1 offset:18
-; CHECK-NEXT:    ds_read_u8 v25, v1 offset:19
-; CHECK-NEXT:    ds_read_u8 v26, v1 offset:20
-; CHECK-NEXT:    ds_read_u8 v27, v1 offset:21
-; CHECK-NEXT:    ds_read_u8 v28, v1 offset:22
-; CHECK-NEXT:    ds_read_u8 v1, v1 offset:23
-; CHECK-NEXT:    s_waitcnt lgkmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt lgkmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt lgkmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt lgkmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    ds_read_b32 v8, v1 offset:24
+; CHECK-NEXT:    ds_read_u16 v9, v1 offset:28
+; CHECK-NEXT:    ds_read_u8 v10, v1 offset:30
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read_b64 v[6:7], v1 offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
+; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -7615,79 +3814,18 @@ define void @memcpy_p5_p3_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p3_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:15
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:14
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:13
-; CHECK-NEXT:    ds_read_u8 v5, v1 offset:12
-; CHECK-NEXT:    ds_read_u8 v6, v1 offset:11
-; CHECK-NEXT:    ds_read_u8 v7, v1 offset:8
-; CHECK-NEXT:    ds_read_u8 v8, v1 offset:9
-; CHECK-NEXT:    ds_read_u8 v9, v1 offset:10
-; CHECK-NEXT:    ds_read_u8 v10, v1
-; CHECK-NEXT:    ds_read_u8 v11, v1 offset:1
-; CHECK-NEXT:    ds_read_u8 v12, v1 offset:2
-; CHECK-NEXT:    ds_read_u8 v13, v1 offset:3
-; CHECK-NEXT:    ds_read_u8 v14, v1 offset:4
-; CHECK-NEXT:    ds_read_u8 v15, v1 offset:5
-; CHECK-NEXT:    ds_read_u8 v16, v1 offset:6
-; CHECK-NEXT:    ds_read_u8 v17, v1 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:24
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:25
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u8 v5, v1 offset:27
-; CHECK-NEXT:    ds_read_u8 v6, v1 offset:28
-; CHECK-NEXT:    ds_read_u8 v18, v1 offset:29
-; CHECK-NEXT:    ds_read_u8 v19, v1 offset:30
-; CHECK-NEXT:    ds_read_u8 v20, v1 offset:31
-; CHECK-NEXT:    ds_read_u8 v21, v1 offset:16
-; CHECK-NEXT:    ds_read_u8 v22, v1 offset:17
-; CHECK-NEXT:    ds_read_u8 v23, v1 offset:18
-; CHECK-NEXT:    ds_read_u8 v24, v1 offset:19
-; CHECK-NEXT:    ds_read_u8 v25, v1 offset:20
-; CHECK-NEXT:    ds_read_u8 v26, v1 offset:21
-; CHECK-NEXT:    ds_read_u8 v27, v1 offset:22
-; CHECK-NEXT:    ds_read_u8 v1, v1 offset:23
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -7698,30 +3836,12 @@ define void @memcpy_p5_p3_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p3_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v2, v1
-; CHECK-NEXT:    ds_read_u16 v3, v1 offset:2
-; CHECK-NEXT:    ds_read_u16 v4, v1 offset:4
-; CHECK-NEXT:    ds_read_u16 v5, v1 offset:6
-; CHECK-NEXT:    ds_read_u16 v6, v1 offset:8
-; CHECK-NEXT:    ds_read_u16 v7, v1 offset:10
-; CHECK-NEXT:    ds_read_u16 v8, v1 offset:12
-; CHECK-NEXT:    ds_read_u16 v1, v1 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -7732,54 +3852,25 @@ define void @memcpy_p5_p3_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p3_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:30
-; CHECK-NEXT:    ds_read_u16 v3, v1 offset:28
-; CHECK-NEXT:    ds_read_u16 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u16 v5, v1 offset:24
-; CHECK-NEXT:    ds_read_u16 v6, v1 offset:22
-; CHECK-NEXT:    ds_read_u16 v7, v1 offset:20
-; CHECK-NEXT:    ds_read_u16 v8, v1 offset:18
-; CHECK-NEXT:    ds_read_u16 v9, v1 offset:16
-; CHECK-NEXT:    ds_read_u16 v10, v1 offset:14
-; CHECK-NEXT:    ds_read_u16 v11, v1 offset:12
-; CHECK-NEXT:    ds_read_u16 v12, v1 offset:10
-; CHECK-NEXT:    ds_read_u16 v13, v1 offset:8
-; CHECK-NEXT:    ds_read_u16 v14, v1 offset:6
-; CHECK-NEXT:    ds_read_u16 v15, v1 offset:4
-; CHECK-NEXT:    ds_read_u16 v16, v1 offset:2
-; CHECK-NEXT:    ds_read_u16 v1, v1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
+; CHECK-NEXT:    ds_read_b32 v8, v1 offset:24
+; CHECK-NEXT:    ds_read_u16 v9, v1 offset:28
+; CHECK-NEXT:    ds_read_u8 v10, v1 offset:30
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read_b64 v[6:7], v1 offset:16
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -7790,54 +3881,18 @@ define void @memcpy_p5_p3_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p3_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v2, v1 offset:30
-; CHECK-NEXT:    ds_read_u16 v3, v1 offset:28
-; CHECK-NEXT:    ds_read_u16 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u16 v5, v1 offset:24
-; CHECK-NEXT:    ds_read_u16 v6, v1 offset:22
-; CHECK-NEXT:    ds_read_u16 v7, v1 offset:20
-; CHECK-NEXT:    ds_read_u16 v8, v1 offset:18
-; CHECK-NEXT:    ds_read_u16 v9, v1 offset:16
-; CHECK-NEXT:    ds_read_u16 v10, v1 offset:14
-; CHECK-NEXT:    ds_read_u16 v11, v1 offset:12
-; CHECK-NEXT:    ds_read_u16 v12, v1 offset:10
-; CHECK-NEXT:    ds_read_u16 v13, v1 offset:8
-; CHECK-NEXT:    ds_read_u16 v14, v1 offset:6
-; CHECK-NEXT:    ds_read_u16 v15, v1 offset:4
-; CHECK-NEXT:    ds_read_u16 v16, v1 offset:2
-; CHECK-NEXT:    ds_read_u16 v1, v1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset0:2 offset1:3
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -7872,30 +3927,10 @@ define void @memcpy_p5_p3_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addr
 ; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte_d16_hi v6, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte_d16_hi v7, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte_d16_hi v8, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte_d16_hi v9, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 24, v6
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v6
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 8, v9
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 31, i1 false)
@@ -7952,30 +3987,10 @@ define void @memcpy_p5_p3_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr a
 ; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte_d16_hi v6, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte_d16_hi v7, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte_d16_hi v8, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte_d16_hi v9, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 24, v6
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v6
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 8, v9
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 31, i1 false)
@@ -8008,55 +4023,12 @@ define void @memcpy_p5_p4_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p4_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -8067,100 +4039,24 @@ define void @memcpy_p5_p4_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p4_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:17
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -8171,103 +4067,19 @@ define void @memcpy_p5_p4_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p4_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:31
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v33, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:17
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -8278,31 +4090,12 @@ define void @memcpy_p5_p4_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p4_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -8313,55 +4106,24 @@ define void @memcpy_p5_p4_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p4_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -8372,55 +4134,19 @@ define void @memcpy_p5_p4_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p4_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -8456,30 +4182,10 @@ define void @memcpy_p5_p4_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addr
 ; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte_d16_hi v7, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte_d16_hi v8, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte_d16_hi v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte_d16_hi v10, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 8, v10
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 31, i1 false)
@@ -8538,30 +4244,10 @@ define void @memcpy_p5_p4_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr a
 ; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte_d16_hi v7, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte_d16_hi v8, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte_d16_hi v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte_d16_hi v10, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 24, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 8, v7
-; CHECK-NEXT:    v_lshrrev_b32_e32 v3, 24, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 8, v8
-; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 24, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 8, v9
-; CHECK-NEXT:    v_lshrrev_b32_e32 v7, 24, v10
-; CHECK-NEXT:    v_lshrrev_b32_e32 v8, 8, v10
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 31, i1 false)
@@ -8595,55 +4281,19 @@ define void @memcpy_p5_p5_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:3
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -8654,99 +4304,34 @@ define void @memcpy_p5_p5_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x11
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_clause 0xc
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:26
+; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:25
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:22
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:21
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -8757,103 +4342,31 @@ define void @memcpy_p5_p5_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x11
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_clause 0xd
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:26
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:25
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:22
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:21
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -8864,31 +4377,19 @@ define void @memcpy_p5_p5_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -8899,55 +4400,34 @@ define void @memcpy_p5_p5_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v11, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v12, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v13, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v14, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v15, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v16, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -8958,55 +4438,31 @@ define void @memcpy_p5_p5_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v11, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v12, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v13, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v14, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v15, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v16, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -9040,67 +4496,31 @@ define void @memcpy_p5_p5_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addr
 ; CHECK-LABEL: memcpy_p5_p5_sz31_align_8_8:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v16, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v17, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v18, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v19, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:29
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:19
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:23
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 31, i1 false)
@@ -9169,67 +4589,31 @@ define void @memcpy_p5_p5_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr a
 ; CHECK-LABEL: memcpy_p5_p5_sz31_align_16_16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x13
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_dword v16, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_dword v17, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_dword v18, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_dword v19, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:19
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:23
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:29
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:15
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:19
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:23
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:27
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:27
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 31, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
index cc5256620bfe088961dc257778fa10938b884c31..4e5688adcd6bbd69cc438cba697a42a4042c9e1f 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
@@ -13,55 +13,9 @@ define void @memmove_p0_p0_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -73,100 +27,19 @@ define void @memmove_p0_p0_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v22, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v23, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v24, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v25, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v26, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v27, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v28, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v29, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v30, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v31, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v32, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v33, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(30) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:30
+; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[6:8], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -178,103 +51,13 @@ define void @memmove_p0_p0_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:31
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v22, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v23, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v24, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v25, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v26, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v27, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v28, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v29, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v30, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v31, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v32, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v33, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v34, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(31) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v34 offset:1
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -286,31 +69,9 @@ define void @memmove_p0_p0_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -322,55 +83,19 @@ define void @memmove_p0_p0_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v15, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v16, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v17, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v18, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:30
+; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[6:8], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -382,55 +107,13 @@ define void @memmove_p0_p0_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v15, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v16, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v17, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v18, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -559,55 +242,9 @@ define void @memmove_p0_p1_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p1_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -619,100 +256,19 @@ define void @memmove_p0_p1_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p1_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ubyte v19, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v20, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v21, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v22, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v23, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v24, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v25, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v26, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v27, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v28, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v29, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v30, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v31, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v32, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v33, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:4
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:30
+; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:28
+; CHECK-NEXT:    global_load_dwordx3 v[6:8], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:3
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:2
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:1
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -724,103 +280,13 @@ define void @memmove_p0_p1_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p1_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:31
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v19, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ubyte v20, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v21, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v22, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v23, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v24, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v25, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v26, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v27, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v28, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v29, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v30, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v31, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v32, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v33, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v34, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v34 offset:1
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -832,31 +298,9 @@ define void @memmove_p0_p1_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p1_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:2
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -868,55 +312,19 @@ define void @memmove_p0_p1_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p1_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:30
+; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:28
+; CHECK-NEXT:    global_load_dwordx3 v[6:8], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -928,55 +336,13 @@ define void @memmove_p0_p1_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p1_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1105,54 +471,9 @@ define void @memmove_p0_p3_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:7
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v17, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1164,72 +485,19 @@ define void @memmove_p0_p3_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:24
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:25
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:26
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:27
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:28
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:29
 ; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:16
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:17
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:18
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:19
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:20
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:21
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:22
-; CHECK-NEXT:    ds_read_u8 v17, v2 offset:23
-; CHECK-NEXT:    ds_read_u8 v18, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v21, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v22, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v23, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v24, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v25, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v26, v2
-; CHECK-NEXT:    ds_read_u8 v27, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v28, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v29, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v30, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v31, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v32, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v2, v2 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
+; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
+; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
 ; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:27
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:23
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:19
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:15
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:14
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:13
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:12
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:11
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:10
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:9
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:7
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:6
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:5
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:4
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:3
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:2
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:1
-; CHECK-NEXT:    flat_store_byte v[0:1], v26
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1241,74 +509,12 @@ define void @memmove_p0_p3_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:24
-; CHECK-NEXT:    ds_read_u8 v4, v2 offset:25
-; CHECK-NEXT:    ds_read_u8 v5, v2 offset:26
-; CHECK-NEXT:    ds_read_u8 v6, v2 offset:27
-; CHECK-NEXT:    ds_read_u8 v7, v2 offset:28
-; CHECK-NEXT:    ds_read_u8 v8, v2 offset:29
-; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
-; CHECK-NEXT:    ds_read_u8 v10, v2 offset:31
-; CHECK-NEXT:    ds_read_u8 v11, v2 offset:16
-; CHECK-NEXT:    ds_read_u8 v12, v2 offset:17
-; CHECK-NEXT:    ds_read_u8 v13, v2 offset:18
-; CHECK-NEXT:    ds_read_u8 v14, v2 offset:19
-; CHECK-NEXT:    ds_read_u8 v15, v2 offset:20
-; CHECK-NEXT:    ds_read_u8 v16, v2 offset:21
-; CHECK-NEXT:    ds_read_u8 v17, v2 offset:22
-; CHECK-NEXT:    ds_read_u8 v18, v2 offset:23
-; CHECK-NEXT:    ds_read_u8 v19, v2 offset:8
-; CHECK-NEXT:    ds_read_u8 v20, v2 offset:9
-; CHECK-NEXT:    ds_read_u8 v21, v2 offset:10
-; CHECK-NEXT:    ds_read_u8 v22, v2 offset:11
-; CHECK-NEXT:    ds_read_u8 v23, v2 offset:12
-; CHECK-NEXT:    ds_read_u8 v24, v2 offset:13
-; CHECK-NEXT:    ds_read_u8 v25, v2 offset:14
-; CHECK-NEXT:    ds_read_u8 v26, v2 offset:15
-; CHECK-NEXT:    ds_read_u8 v27, v2
-; CHECK-NEXT:    ds_read_u8 v28, v2 offset:1
-; CHECK-NEXT:    ds_read_u8 v29, v2 offset:2
-; CHECK-NEXT:    ds_read_u8 v30, v2 offset:3
-; CHECK-NEXT:    ds_read_u8 v31, v2 offset:4
-; CHECK-NEXT:    ds_read_u8 v32, v2 offset:5
-; CHECK-NEXT:    ds_read_u8 v33, v2 offset:6
-; CHECK-NEXT:    ds_read_u8 v2, v2 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:31
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:29
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:27
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:26
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:25
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:23
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:22
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:21
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:20
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:19
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:18
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:17
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:15
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:14
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:13
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:12
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:11
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:10
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:9
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:7
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:6
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:5
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:4
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:3
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:2
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:1
-; CHECK-NEXT:    flat_store_byte v[0:1], v27
+; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset0:2 offset1:3
+; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1320,30 +526,9 @@ define void @memmove_p0_p3_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v3, v2 offset:14
-; CHECK-NEXT:    ds_read_u16 v4, v2 offset:12
-; CHECK-NEXT:    ds_read_u16 v5, v2 offset:10
-; CHECK-NEXT:    ds_read_u16 v6, v2 offset:8
-; CHECK-NEXT:    ds_read_u16 v7, v2 offset:6
-; CHECK-NEXT:    ds_read_u16 v8, v2 offset:4
-; CHECK-NEXT:    ds_read_u16 v9, v2 offset:2
-; CHECK-NEXT:    ds_read_u16 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1355,54 +540,19 @@ define void @memmove_p0_p3_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v3, v2 offset:30
-; CHECK-NEXT:    ds_read_u16 v4, v2 offset:28
-; CHECK-NEXT:    ds_read_u16 v5, v2 offset:26
-; CHECK-NEXT:    ds_read_u16 v6, v2 offset:24
-; CHECK-NEXT:    ds_read_u16 v7, v2 offset:22
-; CHECK-NEXT:    ds_read_u16 v8, v2 offset:20
-; CHECK-NEXT:    ds_read_u16 v9, v2 offset:18
-; CHECK-NEXT:    ds_read_u16 v10, v2 offset:16
-; CHECK-NEXT:    ds_read_u16 v11, v2 offset:14
-; CHECK-NEXT:    ds_read_u16 v12, v2 offset:12
-; CHECK-NEXT:    ds_read_u16 v13, v2 offset:10
-; CHECK-NEXT:    ds_read_u16 v14, v2 offset:8
-; CHECK-NEXT:    ds_read_u16 v15, v2 offset:6
-; CHECK-NEXT:    ds_read_u16 v16, v2 offset:4
-; CHECK-NEXT:    ds_read_u16 v17, v2 offset:2
-; CHECK-NEXT:    ds_read_u16 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:30
+; CHECK-NEXT:    ds_read_b32 v8, v2 offset:24
+; CHECK-NEXT:    ds_read_u16 v10, v2 offset:28
+; CHECK-NEXT:    ds_read_b64 v[6:7], v2 offset:16
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1414,54 +564,12 @@ define void @memmove_p0_p3_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p3_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v3, v2 offset:30
-; CHECK-NEXT:    ds_read_u16 v4, v2 offset:28
-; CHECK-NEXT:    ds_read_u16 v5, v2 offset:26
-; CHECK-NEXT:    ds_read_u16 v6, v2 offset:24
-; CHECK-NEXT:    ds_read_u16 v7, v2 offset:22
-; CHECK-NEXT:    ds_read_u16 v8, v2 offset:20
-; CHECK-NEXT:    ds_read_u16 v9, v2 offset:18
-; CHECK-NEXT:    ds_read_u16 v10, v2 offset:16
-; CHECK-NEXT:    ds_read_u16 v11, v2 offset:14
-; CHECK-NEXT:    ds_read_u16 v12, v2 offset:12
-; CHECK-NEXT:    ds_read_u16 v13, v2 offset:10
-; CHECK-NEXT:    ds_read_u16 v14, v2 offset:8
-; CHECK-NEXT:    ds_read_u16 v15, v2 offset:6
-; CHECK-NEXT:    ds_read_u16 v16, v2 offset:4
-; CHECK-NEXT:    ds_read_u16 v17, v2 offset:2
-; CHECK-NEXT:    ds_read_u16 v2, v2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset0:2 offset1:3
+; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset1:1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1583,55 +691,9 @@ define void @memmove_p0_p4_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p4_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:1
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1643,100 +705,19 @@ define void @memmove_p0_p4_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p4_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ubyte v19, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v20, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v21, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v22, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v23, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v24, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v25, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v26, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v27, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v28, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v29, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v30, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v31, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v32, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v33, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:4
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:30
+; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:28
+; CHECK-NEXT:    global_load_dwordx3 v[6:8], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:3
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:2
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:1
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1748,103 +729,13 @@ define void @memmove_p0_p4_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p4_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:31
-; CHECK-NEXT:    global_load_ubyte v5, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ubyte v6, v[2:3], off offset:29
-; CHECK-NEXT:    global_load_ubyte v7, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ubyte v8, v[2:3], off offset:27
-; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ubyte v10, v[2:3], off offset:25
-; CHECK-NEXT:    global_load_ubyte v11, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ubyte v12, v[2:3], off offset:23
-; CHECK-NEXT:    global_load_ubyte v13, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ubyte v14, v[2:3], off offset:21
-; CHECK-NEXT:    global_load_ubyte v15, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ubyte v16, v[2:3], off offset:19
-; CHECK-NEXT:    global_load_ubyte v17, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ubyte v18, v[2:3], off offset:17
-; CHECK-NEXT:    global_load_ubyte v19, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ubyte v20, v[2:3], off offset:15
-; CHECK-NEXT:    global_load_ubyte v21, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ubyte v22, v[2:3], off offset:13
-; CHECK-NEXT:    global_load_ubyte v23, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ubyte v24, v[2:3], off offset:11
-; CHECK-NEXT:    global_load_ubyte v25, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ubyte v26, v[2:3], off offset:9
-; CHECK-NEXT:    global_load_ubyte v27, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ubyte v28, v[2:3], off offset:7
-; CHECK-NEXT:    global_load_ubyte v29, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ubyte v30, v[2:3], off offset:5
-; CHECK-NEXT:    global_load_ubyte v31, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ubyte v32, v[2:3], off offset:3
-; CHECK-NEXT:    global_load_ubyte v33, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ubyte v34, v[2:3], off offset:1
-; CHECK-NEXT:    global_load_ubyte v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v34 offset:1
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1856,31 +747,9 @@ define void @memmove_p0_p4_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p4_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:2
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1892,55 +761,19 @@ define void @memmove_p0_p4_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p4_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_ubyte v9, v[2:3], off offset:30
+; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:28
+; CHECK-NEXT:    global_load_dwordx3 v[6:8], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[6:8] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -1952,55 +785,13 @@ define void @memmove_p0_p4_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p4_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v4, v[2:3], off offset:30
-; CHECK-NEXT:    global_load_ushort v5, v[2:3], off offset:28
-; CHECK-NEXT:    global_load_ushort v6, v[2:3], off offset:26
-; CHECK-NEXT:    global_load_ushort v7, v[2:3], off offset:24
-; CHECK-NEXT:    global_load_ushort v8, v[2:3], off offset:22
-; CHECK-NEXT:    global_load_ushort v9, v[2:3], off offset:20
-; CHECK-NEXT:    global_load_ushort v10, v[2:3], off offset:18
-; CHECK-NEXT:    global_load_ushort v11, v[2:3], off offset:16
-; CHECK-NEXT:    global_load_ushort v12, v[2:3], off offset:14
-; CHECK-NEXT:    global_load_ushort v13, v[2:3], off offset:12
-; CHECK-NEXT:    global_load_ushort v14, v[2:3], off offset:10
-; CHECK-NEXT:    global_load_ushort v15, v[2:3], off offset:8
-; CHECK-NEXT:    global_load_ushort v16, v[2:3], off offset:6
-; CHECK-NEXT:    global_load_ushort v17, v[2:3], off offset:4
-; CHECK-NEXT:    global_load_ushort v18, v[2:3], off offset:2
-; CHECK-NEXT:    global_load_ushort v2, v[2:3], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v18 offset:2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2129,55 +920,13 @@ define void @memmove_p0_p5_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:1
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2189,100 +938,23 @@ define void @memmove_p0_p5_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:6
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:5
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:1
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2294,103 +966,19 @@ define void @memmove_p0_p5_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_byte v[0:1], v22 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_byte v[0:1], v23 offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_byte v[0:1], v24 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_byte v[0:1], v25 offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_byte v[0:1], v26 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_byte v[0:1], v27 offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_byte v[0:1], v28 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_byte v[0:1], v29 offset:5
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_byte v[0:1], v30 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_byte v[0:1], v31 offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_byte v[0:1], v32 offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_byte v[0:1], v33 offset:1
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_byte v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2402,31 +990,13 @@ define void @memmove_p0_p5_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:2
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2438,55 +1008,23 @@ define void @memmove_p0_p5_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v12, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v13, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v14, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v15, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v16, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v17, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
+; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:30
+; CHECK-NEXT:    flat_store_dwordx3 v[0:1], v[7:9] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2498,55 +1036,19 @@ define void @memmove_p0_p5_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p0_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v12, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v13, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v14, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v15, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v16, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v17, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v2, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    flat_store_short v[0:1], v3 offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    flat_store_short v[0:1], v4 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    flat_store_short v[0:1], v5 offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    flat_store_short v[0:1], v6 offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    flat_store_short v[0:1], v7 offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    flat_store_short v[0:1], v8 offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    flat_store_short v[0:1], v9 offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    flat_store_short v[0:1], v10 offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    flat_store_short v[0:1], v11 offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    flat_store_short v[0:1], v12 offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    flat_store_short v[0:1], v13 offset:10
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    flat_store_short v[0:1], v14 offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    flat_store_short v[0:1], v15 offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    flat_store_short v[0:1], v16 offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    flat_store_short v[0:1], v17 offset:2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_store_short v[0:1], v2
+; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2698,41 +1200,8 @@ define void @memmove_p1_p0_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3]
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 8, v10
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v11, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v12, 8, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 8, v15
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v16, 8, v17
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v18, 8, v2
-; CHECK-NEXT:    v_lshl_or_b32 v2, v6, 16, v5
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 16, v7
-; CHECK-NEXT:    v_lshl_or_b32 v4, v10, 16, v9
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2744,79 +1213,18 @@ define void @memmove_p1_p0_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v22, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v23, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v24, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v25, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v26, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v27, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v28, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v29, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v30, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v31, v[2:3]
-; CHECK-NEXT:    flat_load_ubyte v32, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v33, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(29) lgkmcnt(29)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v4, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v8, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(23) lgkmcnt(23)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v10, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshlrev_b16 v12, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(19)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v14, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(17) lgkmcnt(17)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v16, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v16, v6, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v18, 8, v19
-; CHECK-NEXT:    v_lshl_or_b32 v7, v9, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v20, 8, v21
-; CHECK-NEXT:    v_lshl_or_b32 v8, v14, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v22, 8, v23
-; CHECK-NEXT:    v_lshl_or_b32 v5, v4, 16, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v24, 8, v25
-; CHECK-NEXT:    v_lshl_or_b32 v6, v15, 16, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v20, v26, 8, v27
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v19, v28, 8, v29
-; CHECK-NEXT:    v_lshl_or_b32 v4, v18, 16, v17
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:30
+; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[6:8], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v21, v30, 8, v31
+; CHECK-NEXT:    global_store_byte v[0:1], v9, off offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v22, v32, 8, v33
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_or_b32_e32 v12, v12, v2
-; CHECK-NEXT:    v_lshl_or_b32 v3, v20, 16, v19
-; CHECK-NEXT:    v_lshl_or_b32 v2, v22, 16, v21
-; CHECK-NEXT:    global_store_byte v[0:1], v13, off offset:30
-; CHECK-NEXT:    global_store_short v[0:1], v12, off offset:28
 ; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2828,79 +1236,13 @@ define void @memmove_p1_p0_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    flat_load_ubyte v4, v[2:3] offset:29
-; CHECK-NEXT:    flat_load_ubyte v5, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ubyte v6, v[2:3] offset:31
-; CHECK-NEXT:    flat_load_ubyte v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ubyte v8, v[2:3] offset:25
-; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ubyte v10, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ubyte v11, v[2:3] offset:27
-; CHECK-NEXT:    flat_load_ubyte v12, v[2:3] offset:15
-; CHECK-NEXT:    flat_load_ubyte v13, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ubyte v14, v[2:3] offset:13
-; CHECK-NEXT:    flat_load_ubyte v15, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ubyte v16, v[2:3] offset:23
-; CHECK-NEXT:    flat_load_ubyte v17, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ubyte v18, v[2:3] offset:21
-; CHECK-NEXT:    flat_load_ubyte v19, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:19
-; CHECK-NEXT:    flat_load_ubyte v21, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ubyte v22, v[2:3] offset:17
-; CHECK-NEXT:    flat_load_ubyte v23, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ubyte v24, v[2:3] offset:11
-; CHECK-NEXT:    flat_load_ubyte v25, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ubyte v26, v[2:3] offset:9
-; CHECK-NEXT:    flat_load_ubyte v27, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ubyte v28, v[2:3] offset:7
-; CHECK-NEXT:    flat_load_ubyte v29, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ubyte v30, v[2:3] offset:5
-; CHECK-NEXT:    flat_load_ubyte v31, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ubyte v32, v[2:3] offset:1
-; CHECK-NEXT:    flat_load_ubyte v33, v[2:3]
-; CHECK-NEXT:    flat_load_ubyte v34, v[2:3] offset:3
-; CHECK-NEXT:    flat_load_ubyte v2, v[2:3] offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 8, v10
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 8, v5
-; CHECK-NEXT:    v_lshl_or_b32 v6, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v11, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v12, 8, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v14, 8, v15
-; CHECK-NEXT:    v_lshl_or_b32 v5, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v16, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v18, 8, v19
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v20, 8, v21
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v22, 8, v23
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v24, 8, v25
-; CHECK-NEXT:    v_lshl_or_b32 v9, v12, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v26, 8, v27
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v28, 8, v29
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v30, 8, v31
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v32, 8, v33
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v19, v34, 8, v2
-; CHECK-NEXT:    v_lshl_or_b32 v2, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v16
-; CHECK-NEXT:    v_lshl_or_b32 v6, v19, 16, v18
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p0.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -2911,23 +1253,8 @@ define void @memmove_p1_p0_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3]
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v4, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v9, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 16, v6
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v7
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2939,41 +1266,18 @@ define void @memmove_p1_p0_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v15, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ushort v16, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v17, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v18, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v19, v[2:3]
-; CHECK-NEXT:    flat_load_ubyte v20, v[2:3] offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v4, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v9, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v6, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v12, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v14, 16, v15
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v9, v[2:3] offset:30
+; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[6:8], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v16, 16, v17
+; CHECK-NEXT:    global_store_byte v[0:1], v9, off offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
+; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v18, 16, v19
-; CHECK-NEXT:    global_store_short v[0:1], v11, off offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    global_store_byte v[0:1], v20, off offset:30
 ; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -2985,39 +1289,13 @@ define void @memmove_p1_p0_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v4, v[2:3] offset:30
-; CHECK-NEXT:    flat_load_ushort v5, v[2:3] offset:28
-; CHECK-NEXT:    flat_load_ushort v6, v[2:3] offset:26
-; CHECK-NEXT:    flat_load_ushort v7, v[2:3] offset:14
-; CHECK-NEXT:    flat_load_ushort v8, v[2:3] offset:12
-; CHECK-NEXT:    flat_load_ushort v10, v[2:3] offset:10
-; CHECK-NEXT:    flat_load_ushort v11, v[2:3] offset:8
-; CHECK-NEXT:    flat_load_ushort v9, v[2:3] offset:24
-; CHECK-NEXT:    flat_load_ushort v12, v[2:3] offset:22
-; CHECK-NEXT:    flat_load_ushort v13, v[2:3] offset:20
-; CHECK-NEXT:    flat_load_ushort v14, v[2:3] offset:18
-; CHECK-NEXT:    flat_load_ushort v15, v[2:3] offset:16
-; CHECK-NEXT:    flat_load_ushort v16, v[2:3] offset:6
-; CHECK-NEXT:    flat_load_ushort v17, v[2:3] offset:4
-; CHECK-NEXT:    flat_load_ushort v18, v[2:3] offset:2
-; CHECK-NEXT:    flat_load_ushort v19, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v4, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v6, 16, v9
-; CHECK-NEXT:    v_lshl_or_b32 v9, v7, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v12, 16, v13
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 16, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v14, 16, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v16, 16, v17
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v18, 16, v19
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p0.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -3783,44 +2061,13 @@ define void @memmove_p1_p5_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v4, 8, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v6, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v14, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v16, 8, v15
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v2, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v5, v9, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v10
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -3831,82 +2078,24 @@ define void @memmove_p1_p5_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v4, 8, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v6, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    v_lshlrev_b16 v4, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v9, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v11, 8, v10
-; CHECK-NEXT:    v_lshl_or_b32 v8, v3, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v15, 8, v14
-; CHECK-NEXT:    v_lshl_or_b32 v2, v9, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v17, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v19, 8, v18
-; CHECK-NEXT:    v_lshl_or_b32 v3, v6, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v21, 8, v20
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v23, 8, v22
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v26, 8, v25
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v24, 8, v28
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v29, 8, v27
-; CHECK-NEXT:    v_lshl_or_b32 v5, v11, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v31, 8, v30
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    v_or_b32_e32 v18, v4, v32
-; CHECK-NEXT:    v_lshl_or_b32 v4, v13, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v7, v15, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v6, v16, 16, v17
+; CHECK-NEXT:    global_store_byte v[0:1], v11, off offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    global_store_byte v[0:1], v33, off offset:30
-; CHECK-NEXT:    global_store_short v[0:1], v18, off offset:28
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[7:9], off offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -3917,81 +2106,19 @@ define void @memmove_p1_p5_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v2, v2, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v4, 8, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v6, 8, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v14, 8, v13
-; CHECK-NEXT:    v_lshl_or_b32 v3, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v16, 8, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v18, 8, v17
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v21, 8, v20
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v23, 8, v22
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v19, 8, v25
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v26, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v28, 8, v27
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v30, 8, v29
-; CHECK-NEXT:    v_lshl_or_b32 v7, v13, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v32, 8, v31
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v19, v2, 8, v33
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v5, v9, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v6, v14, 16, v15
-; CHECK-NEXT:    v_lshl_or_b32 v9, v17, 16, v16
-; CHECK-NEXT:    v_lshl_or_b32 v8, v19, 16, v18
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -4002,24 +2129,13 @@ define void @memmove_p1_p5_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v6, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 16, v7
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v10, 16, v9
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -4030,43 +2146,24 @@ define void @memmove_p1_p5_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v12, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v13, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v14, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v15, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v16, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v17, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v18, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v6, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v7
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v15, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v17, 16, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    global_store_short v[0:1], v18, off offset:28
+; CHECK-NEXT:    global_store_short v[0:1], v10, off offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    global_store_byte v[0:1], v11, off offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    global_store_byte v[0:1], v19, off offset:30
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[6:8], off offset:16
+; CHECK-NEXT:    global_store_dwordx3 v[0:1], v[7:9], off offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -4077,41 +2174,19 @@ define void @memmove_p1_p5_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p1_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v4, v2, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v5, v2, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v3, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v6, v2, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v7, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v8, v2, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v9, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v10, v2, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v11, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v12, v2, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v13, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v14, v2, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v15, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v16, v2, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v17, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v18, v2, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v6, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v14, 16, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v16, 16, v15
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v18, 16, v17
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off offset:16
+; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -4258,41 +2333,8 @@ define void @memmove_p3_p0_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2]
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 8, v9
-; CHECK-NEXT:    v_lshl_or_b32 v3, v5, 8, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v11, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 8, v14
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 8, v16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v17, 8, v1
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
 ; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -4305,82 +2347,20 @@ define void @memmove_p3_p0_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v21, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v22, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v23, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v24, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v25, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v26, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v27, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v28, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v29, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v30, v[1:2]
-; CHECK-NEXT:    flat_load_ubyte v31, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v32, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(29) lgkmcnt(29)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v3, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(27) lgkmcnt(27)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v5, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v7, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(23) lgkmcnt(23)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v9, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshlrev_b16 v11, 8, v11
-; CHECK-NEXT:    v_lshl_or_b32 v4, v3, 16, v5
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(19)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v13, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(17) lgkmcnt(17)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v15, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v17, 8, v18
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v19, 8, v20
-; CHECK-NEXT:    v_lshl_or_b32 v13, v13, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v21, 8, v22
-; CHECK-NEXT:    v_lshl_or_b32 v2, v7, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v23, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v25, 8, v26
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v27, 8, v28
-; CHECK-NEXT:    v_lshl_or_b32 v3, v14, 16, v6
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:30
+; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[5:7], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v29, 8, v30
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v31, 8, v32
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_or_b32_e32 v11, v11, v1
-; CHECK-NEXT:    v_lshl_or_b32 v1, v10, 16, v8
-; CHECK-NEXT:    v_lshl_or_b32 v6, v16, 16, v15
-; CHECK-NEXT:    v_lshl_or_b32 v5, v18, 16, v17
-; CHECK-NEXT:    ds_write_b8 v0, v12 offset:30
-; CHECK-NEXT:    ds_write_b32 v0, v13 offset:24
-; CHECK-NEXT:    ds_write_b16 v0, v11 offset:28
-; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:16
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[3:4] offset1:1
+; CHECK-NEXT:    ds_write_b8 v0, v8 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
+; CHECK-NEXT:    ds_write_b16 v0, v9 offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; CHECK-NEXT:    ds_write_b32 v0, v7 offset:24
+; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(4)
+; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4392,79 +2372,13 @@ define void @memmove_p3_p0_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:31
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v21, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v22, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v23, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v24, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v25, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v26, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v27, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v28, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v29, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v30, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v31, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v32, v[1:2]
-; CHECK-NEXT:    flat_load_ubyte v33, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2] offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 8, v9
-; CHECK-NEXT:    v_lshl_or_b32 v3, v5, 8, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v10, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v11, 8, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v13, 8, v14
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v15, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v17, 8, v18
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v19, 8, v20
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v21, 8, v22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v23, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v25, 8, v26
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v27, 8, v28
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v29, 8, v30
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v31, 8, v32
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v33, 8, v1
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v4
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
-; CHECK-NEXT:    v_lshl_or_b32 v6, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v16
-; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[1:2] offset0:2 offset1:3
-; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[5:6] offset1:1
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4476,23 +2390,8 @@ define void @memmove_p3_p0_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2]
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v8, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v9, 16, v5
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v10, 16, v6
 ; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -4505,43 +2404,20 @@ define void @memmove_p3_p0_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v13, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ushort v14, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v15, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v16, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2]
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v3, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v5, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v8, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v10, 16, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v12, 16, v13
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:30
+; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[5:7], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v14, 16, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v16, 16, v17
-; CHECK-NEXT:    ds_write_b16 v0, v7 offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
-; CHECK-NEXT:    ds_write_b8 v0, v18 offset:30
-; CHECK-NEXT:    ds_write_b32 v0, v8 offset:24
-; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:16
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[3:4] offset1:1
+; CHECK-NEXT:    ds_write_b8 v0, v8 offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(3)
+; CHECK-NEXT:    ds_write_b16 v0, v9 offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(3)
+; CHECK-NEXT:    ds_write_b32 v0, v7 offset:24
+; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(4)
+; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -4553,40 +2429,13 @@ define void @memmove_p3_p0_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v13, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v14, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ushort v15, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v16, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v18, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v3, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v6, 16, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v5, v8, 16, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v13, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v18
-; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[1:2] offset0:2 offset1:3
-; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[5:6] offset1:1
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
+; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5398,44 +3247,13 @@ define void @memmove_p3_p5_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v3, 8, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v5, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v9, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v13, 8, v12
-; CHECK-NEXT:    v_lshl_or_b32 v2, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v15, 8, v14
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v1, 8, v16
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v5
-; CHECK-NEXT:    v_lshl_or_b32 v3, v10, 16, v9
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5447,83 +3265,26 @@ define void @memmove_p3_p5_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v32, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v3, 8, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    v_lshlrev_b16 v3, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v8, 8, v7
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v10, 8, v9
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v12, 8, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v14, 8, v13
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v16, 8, v15
-; CHECK-NEXT:    v_lshl_or_b32 v16, v2, 16, v1
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v18, 8, v17
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v20, 8, v19
-; CHECK-NEXT:    v_lshl_or_b32 v1, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v22, 8, v21
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(8)
+; CHECK-NEXT:    ds_write_b32 v0, v8 offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v25, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v23, 8, v27
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v28, 8, v26
-; CHECK-NEXT:    v_lshl_or_b32 v4, v9, 16, v8
+; CHECK-NEXT:    ds_write_b16 v0, v9 offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(6)
+; CHECK-NEXT:    ds_write_b8 v0, v10 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v30, 8, v29
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    v_or_b32_e32 v17, v3, v31
-; CHECK-NEXT:    v_lshl_or_b32 v3, v11, 16, v10
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 16, v12
-; CHECK-NEXT:    v_lshl_or_b32 v5, v14, 16, v15
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    ds_write_b8 v0, v32 offset:30
-; CHECK-NEXT:    ds_write_b32 v0, v16 offset:24
-; CHECK-NEXT:    ds_write_b16 v0, v17 offset:28
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:16
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ds_write_b64 v0, v[6:7] offset:16
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5535,81 +3296,19 @@ define void @memmove_p3_p5_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v32, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v3, 8, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v5, 8, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v7, 8, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v9, 8, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    v_lshl_or_b32 v9, v11, 8, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    v_lshl_or_b32 v10, v13, 8, v12
-; CHECK-NEXT:    v_lshl_or_b32 v2, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v15, 8, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v17, 8, v16
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    v_lshl_or_b32 v12, v20, 8, v19
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    v_lshl_or_b32 v14, v22, 8, v21
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    v_lshl_or_b32 v11, v18, 8, v24
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v13, v25, 8, v23
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v15, v27, 8, v26
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v16, v29, 8, v28
-; CHECK-NEXT:    v_lshl_or_b32 v6, v12, 16, v11
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v17, v31, 8, v30
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v1, 8, v32
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    v_lshl_or_b32 v4, v8, 16, v5
-; CHECK-NEXT:    v_lshl_or_b32 v3, v10, 16, v9
-; CHECK-NEXT:    v_lshl_or_b32 v5, v13, 16, v14
-; CHECK-NEXT:    v_lshl_or_b32 v8, v16, 16, v15
-; CHECK-NEXT:    v_lshl_or_b32 v7, v18, 16, v17
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[7:8] offset0:2 offset1:3
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset0:2 offset1:3
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5621,24 +3320,13 @@ define void @memmove_p3_p5_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v7, 16, v6
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5650,44 +3338,26 @@ define void @memmove_p3_p5_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v11, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v12, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v13, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v14, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v15, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v16, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v18, v3, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v5, 16, v4
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v7, 16, v6
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v9, 16, v8
+; CHECK-NEXT:    ds_write_b32 v0, v8 offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(7)
+; CHECK-NEXT:    ds_write_b16 v0, v9 offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v11, 16, v10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 16, v12
+; CHECK-NEXT:    ds_write_b8 v0, v10 offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v15, 16, v14
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    ds_write_b16 v0, v16 offset:28
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    ds_write_b8 v0, v17 offset:30
-; CHECK-NEXT:    ds_write_b32 v0, v18 offset:24
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:16
+; CHECK-NEXT:    ds_write_b64 v0, v[6:7] offset:16
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5699,41 +3369,19 @@ define void @memmove_p3_p5_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p3_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v11, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v12, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v13, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v14, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v15, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v16, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v17, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    v_lshl_or_b32 v1, v4, 16, v3
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    v_lshl_or_b32 v2, v5, 16, v2
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    v_lshl_or_b32 v3, v7, 16, v6
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    v_lshl_or_b32 v4, v9, 16, v8
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    v_lshl_or_b32 v5, v11, 16, v10
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    v_lshl_or_b32 v6, v13, 16, v12
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    v_lshl_or_b32 v8, v15, 16, v14
+; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_lshl_or_b32 v7, v17, 16, v16
-; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
-; CHECK-NEXT:    ds_write2_b64 v0, v[5:6], v[7:8] offset0:2 offset1:3
+; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset0:2 offset1:3
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -5890,55 +3538,12 @@ define void @memmove_p5_p0_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p0_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -5949,100 +3554,24 @@ define void @memmove_p5_p0_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p0_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v21, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v22, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v23, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v24, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v25, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v26, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v27, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v28, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v29, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v30, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v31, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v32, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(30) lgkmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29) lgkmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28) lgkmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27) lgkmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26) lgkmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23) lgkmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21) lgkmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17) lgkmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:30
+; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[5:7], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:3
+; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -6053,103 +3582,19 @@ define void @memmove_p5_p0_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p0_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:31
-; CHECK-NEXT:    flat_load_ubyte v4, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ubyte v5, v[1:2] offset:29
-; CHECK-NEXT:    flat_load_ubyte v6, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ubyte v7, v[1:2] offset:27
-; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ubyte v9, v[1:2] offset:25
-; CHECK-NEXT:    flat_load_ubyte v10, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ubyte v11, v[1:2] offset:23
-; CHECK-NEXT:    flat_load_ubyte v12, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ubyte v13, v[1:2] offset:21
-; CHECK-NEXT:    flat_load_ubyte v14, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ubyte v15, v[1:2] offset:19
-; CHECK-NEXT:    flat_load_ubyte v16, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ubyte v17, v[1:2] offset:17
-; CHECK-NEXT:    flat_load_ubyte v18, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ubyte v19, v[1:2] offset:15
-; CHECK-NEXT:    flat_load_ubyte v20, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ubyte v21, v[1:2] offset:13
-; CHECK-NEXT:    flat_load_ubyte v22, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ubyte v23, v[1:2] offset:11
-; CHECK-NEXT:    flat_load_ubyte v24, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ubyte v25, v[1:2] offset:9
-; CHECK-NEXT:    flat_load_ubyte v26, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ubyte v27, v[1:2] offset:7
-; CHECK-NEXT:    flat_load_ubyte v28, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ubyte v29, v[1:2] offset:5
-; CHECK-NEXT:    flat_load_ubyte v30, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ubyte v31, v[1:2] offset:3
-; CHECK-NEXT:    flat_load_ubyte v32, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ubyte v33, v[1:2] offset:1
-; CHECK-NEXT:    flat_load_ubyte v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(31) lgkmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30) lgkmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29) lgkmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28) lgkmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27) lgkmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26) lgkmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25) lgkmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24) lgkmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23) lgkmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22) lgkmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21) lgkmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20) lgkmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19) lgkmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18) lgkmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17) lgkmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16) lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -6160,31 +3605,12 @@ define void @memmove_p5_p0_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p0_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -6195,55 +3621,24 @@ define void @memmove_p5_p0_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p0_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ubyte v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v13, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v14, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v15, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v16, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    flat_load_ubyte v8, v[1:2] offset:30
+; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:28
+; CHECK-NEXT:    flat_load_dwordx3 v[5:7], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -6254,55 +3649,19 @@ define void @memmove_p5_p0_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p0_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    flat_load_ushort v3, v[1:2] offset:30
-; CHECK-NEXT:    flat_load_ushort v4, v[1:2] offset:28
-; CHECK-NEXT:    flat_load_ushort v5, v[1:2] offset:26
-; CHECK-NEXT:    flat_load_ushort v6, v[1:2] offset:24
-; CHECK-NEXT:    flat_load_ushort v7, v[1:2] offset:22
-; CHECK-NEXT:    flat_load_ushort v8, v[1:2] offset:20
-; CHECK-NEXT:    flat_load_ushort v9, v[1:2] offset:18
-; CHECK-NEXT:    flat_load_ushort v10, v[1:2] offset:16
-; CHECK-NEXT:    flat_load_ushort v11, v[1:2] offset:14
-; CHECK-NEXT:    flat_load_ushort v12, v[1:2] offset:12
-; CHECK-NEXT:    flat_load_ushort v13, v[1:2] offset:10
-; CHECK-NEXT:    flat_load_ushort v14, v[1:2] offset:8
-; CHECK-NEXT:    flat_load_ushort v15, v[1:2] offset:6
-; CHECK-NEXT:    flat_load_ushort v16, v[1:2] offset:4
-; CHECK-NEXT:    flat_load_ushort v17, v[1:2] offset:2
-; CHECK-NEXT:    flat_load_ushort v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(15) lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14) lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13) lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11) lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10) lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9) lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]
 ; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -6452,55 +3811,12 @@ define void @memmove_p5_p1_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p1_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -6511,100 +3827,24 @@ define void @memmove_p5_p1_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p1_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:3
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -6615,103 +3855,19 @@ define void @memmove_p5_p1_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p1_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:31
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v33, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -6722,31 +3878,12 @@ define void @memmove_p5_p1_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p1_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -6757,55 +3894,24 @@ define void @memmove_p5_p1_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p1_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -6816,55 +3922,19 @@ define void @memmove_p5_p1_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p1_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -7009,54 +4079,12 @@ define void @memmove_p5_p3_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p3_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:15
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:14
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:13
-; CHECK-NEXT:    ds_read_u8 v5, v1 offset:12
-; CHECK-NEXT:    ds_read_u8 v6, v1 offset:11
-; CHECK-NEXT:    ds_read_u8 v7, v1 offset:10
-; CHECK-NEXT:    ds_read_u8 v8, v1 offset:9
-; CHECK-NEXT:    ds_read_u8 v9, v1 offset:8
-; CHECK-NEXT:    ds_read_u8 v10, v1 offset:7
-; CHECK-NEXT:    ds_read_u8 v11, v1 offset:6
-; CHECK-NEXT:    ds_read_u8 v12, v1 offset:5
-; CHECK-NEXT:    ds_read_u8 v13, v1 offset:4
-; CHECK-NEXT:    ds_read_u8 v14, v1 offset:3
-; CHECK-NEXT:    ds_read_u8 v15, v1 offset:2
-; CHECK-NEXT:    ds_read_u8 v16, v1 offset:1
-; CHECK-NEXT:    ds_read_u8 v1, v1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -7067,72 +4095,25 @@ define void @memmove_p5_p3_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p3_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:24
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:25
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u8 v5, v1 offset:27
-; CHECK-NEXT:    ds_read_u8 v6, v1 offset:28
-; CHECK-NEXT:    ds_read_u8 v7, v1 offset:29
-; CHECK-NEXT:    ds_read_u8 v8, v1 offset:30
-; CHECK-NEXT:    ds_read_u8 v9, v1 offset:16
-; CHECK-NEXT:    ds_read_u8 v10, v1 offset:17
-; CHECK-NEXT:    ds_read_u8 v11, v1 offset:18
-; CHECK-NEXT:    ds_read_u8 v12, v1 offset:19
-; CHECK-NEXT:    ds_read_u8 v13, v1 offset:20
-; CHECK-NEXT:    ds_read_u8 v14, v1 offset:21
-; CHECK-NEXT:    ds_read_u8 v15, v1 offset:22
-; CHECK-NEXT:    ds_read_u8 v16, v1 offset:23
-; CHECK-NEXT:    ds_read_u8 v17, v1 offset:8
-; CHECK-NEXT:    ds_read_u8 v18, v1 offset:9
-; CHECK-NEXT:    ds_read_u8 v19, v1 offset:10
-; CHECK-NEXT:    ds_read_u8 v20, v1 offset:11
-; CHECK-NEXT:    ds_read_u8 v21, v1 offset:12
-; CHECK-NEXT:    ds_read_u8 v22, v1 offset:13
-; CHECK-NEXT:    ds_read_u8 v23, v1 offset:14
-; CHECK-NEXT:    ds_read_u8 v24, v1 offset:15
-; CHECK-NEXT:    ds_read_u8 v25, v1
-; CHECK-NEXT:    ds_read_u8 v26, v1 offset:1
-; CHECK-NEXT:    ds_read_u8 v27, v1 offset:2
-; CHECK-NEXT:    ds_read_u8 v28, v1 offset:3
-; CHECK-NEXT:    ds_read_u8 v29, v1 offset:4
-; CHECK-NEXT:    ds_read_u8 v30, v1 offset:5
-; CHECK-NEXT:    ds_read_u8 v31, v1 offset:6
-; CHECK-NEXT:    ds_read_u8 v1, v1 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen
+; CHECK-NEXT:    ds_read_b32 v8, v1 offset:24
+; CHECK-NEXT:    ds_read_u16 v9, v1 offset:28
+; CHECK-NEXT:    ds_read_u8 v10, v1 offset:30
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read_b64 v[6:7], v1 offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
+; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:30
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -7143,74 +4124,18 @@ define void @memmove_p5_p3_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p3_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:24
-; CHECK-NEXT:    ds_read_u8 v3, v1 offset:25
-; CHECK-NEXT:    ds_read_u8 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u8 v5, v1 offset:27
-; CHECK-NEXT:    ds_read_u8 v6, v1 offset:28
-; CHECK-NEXT:    ds_read_u8 v7, v1 offset:29
-; CHECK-NEXT:    ds_read_u8 v8, v1 offset:30
-; CHECK-NEXT:    ds_read_u8 v9, v1 offset:31
-; CHECK-NEXT:    ds_read_u8 v10, v1 offset:16
-; CHECK-NEXT:    ds_read_u8 v11, v1 offset:17
-; CHECK-NEXT:    ds_read_u8 v12, v1 offset:18
-; CHECK-NEXT:    ds_read_u8 v13, v1 offset:19
-; CHECK-NEXT:    ds_read_u8 v14, v1 offset:20
-; CHECK-NEXT:    ds_read_u8 v15, v1 offset:21
-; CHECK-NEXT:    ds_read_u8 v16, v1 offset:22
-; CHECK-NEXT:    ds_read_u8 v17, v1 offset:23
-; CHECK-NEXT:    ds_read_u8 v18, v1 offset:8
-; CHECK-NEXT:    ds_read_u8 v19, v1 offset:9
-; CHECK-NEXT:    ds_read_u8 v20, v1 offset:10
-; CHECK-NEXT:    ds_read_u8 v21, v1 offset:11
-; CHECK-NEXT:    ds_read_u8 v22, v1 offset:12
-; CHECK-NEXT:    ds_read_u8 v23, v1 offset:13
-; CHECK-NEXT:    ds_read_u8 v24, v1 offset:14
-; CHECK-NEXT:    ds_read_u8 v25, v1 offset:15
-; CHECK-NEXT:    ds_read_u8 v26, v1
-; CHECK-NEXT:    ds_read_u8 v27, v1 offset:1
-; CHECK-NEXT:    ds_read_u8 v28, v1 offset:2
-; CHECK-NEXT:    ds_read_u8 v29, v1 offset:3
-; CHECK-NEXT:    ds_read_u8 v30, v1 offset:4
-; CHECK-NEXT:    ds_read_u8 v31, v1 offset:5
-; CHECK-NEXT:    ds_read_u8 v32, v1 offset:6
-; CHECK-NEXT:    ds_read_u8 v1, v1 offset:7
-; CHECK-NEXT:    s_waitcnt lgkmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -7221,30 +4146,12 @@ define void @memmove_p5_p3_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p3_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v2, v1
-; CHECK-NEXT:    ds_read_u16 v3, v1 offset:2
-; CHECK-NEXT:    ds_read_u16 v4, v1 offset:4
-; CHECK-NEXT:    ds_read_u16 v5, v1 offset:6
-; CHECK-NEXT:    ds_read_u16 v6, v1 offset:8
-; CHECK-NEXT:    ds_read_u16 v7, v1 offset:10
-; CHECK-NEXT:    ds_read_u16 v8, v1 offset:12
-; CHECK-NEXT:    ds_read_u16 v1, v1 offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -7255,54 +4162,25 @@ define void @memmove_p5_p3_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p3_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u8 v2, v1 offset:30
-; CHECK-NEXT:    ds_read_u16 v3, v1 offset:28
-; CHECK-NEXT:    ds_read_u16 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u16 v5, v1 offset:24
-; CHECK-NEXT:    ds_read_u16 v6, v1 offset:22
-; CHECK-NEXT:    ds_read_u16 v7, v1 offset:20
-; CHECK-NEXT:    ds_read_u16 v8, v1 offset:18
-; CHECK-NEXT:    ds_read_u16 v9, v1 offset:16
-; CHECK-NEXT:    ds_read_u16 v10, v1 offset:14
-; CHECK-NEXT:    ds_read_u16 v11, v1 offset:12
-; CHECK-NEXT:    ds_read_u16 v12, v1 offset:10
-; CHECK-NEXT:    ds_read_u16 v13, v1 offset:8
-; CHECK-NEXT:    ds_read_u16 v14, v1 offset:6
-; CHECK-NEXT:    ds_read_u16 v15, v1 offset:4
-; CHECK-NEXT:    ds_read_u16 v16, v1 offset:2
-; CHECK-NEXT:    ds_read_u16 v1, v1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
+; CHECK-NEXT:    ds_read_b32 v8, v1 offset:24
+; CHECK-NEXT:    ds_read_u16 v9, v1 offset:28
+; CHECK-NEXT:    ds_read_u8 v10, v1 offset:30
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read_b64 v[6:7], v1 offset:16
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -7313,54 +4191,18 @@ define void @memmove_p5_p3_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p3_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    ds_read_u16 v2, v1 offset:30
-; CHECK-NEXT:    ds_read_u16 v3, v1 offset:28
-; CHECK-NEXT:    ds_read_u16 v4, v1 offset:26
-; CHECK-NEXT:    ds_read_u16 v5, v1 offset:24
-; CHECK-NEXT:    ds_read_u16 v6, v1 offset:22
-; CHECK-NEXT:    ds_read_u16 v7, v1 offset:20
-; CHECK-NEXT:    ds_read_u16 v8, v1 offset:18
-; CHECK-NEXT:    ds_read_u16 v9, v1 offset:16
-; CHECK-NEXT:    ds_read_u16 v10, v1 offset:14
-; CHECK-NEXT:    ds_read_u16 v11, v1 offset:12
-; CHECK-NEXT:    ds_read_u16 v12, v1 offset:10
-; CHECK-NEXT:    ds_read_u16 v13, v1 offset:8
-; CHECK-NEXT:    ds_read_u16 v14, v1 offset:6
-; CHECK-NEXT:    ds_read_u16 v15, v1 offset:4
-; CHECK-NEXT:    ds_read_u16 v16, v1 offset:2
-; CHECK-NEXT:    ds_read_u16 v1, v1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(15)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt lgkmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt lgkmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt lgkmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt lgkmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt lgkmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt lgkmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt lgkmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt lgkmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt lgkmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:1
+; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset0:2 offset1:3
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -7505,55 +4347,12 @@ define void @memmove_p5_p4_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p4_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -7564,100 +4363,24 @@ define void @memmove_p5_p4_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p4_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:3
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -7668,103 +4391,19 @@ define void @memmove_p5_p4_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p4_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:31
-; CHECK-NEXT:    global_load_ubyte v4, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ubyte v5, v[1:2], off offset:29
-; CHECK-NEXT:    global_load_ubyte v6, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ubyte v7, v[1:2], off offset:27
-; CHECK-NEXT:    global_load_ubyte v8, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:25
-; CHECK-NEXT:    global_load_ubyte v10, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ubyte v11, v[1:2], off offset:23
-; CHECK-NEXT:    global_load_ubyte v12, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ubyte v13, v[1:2], off offset:21
-; CHECK-NEXT:    global_load_ubyte v14, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ubyte v15, v[1:2], off offset:19
-; CHECK-NEXT:    global_load_ubyte v16, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ubyte v17, v[1:2], off offset:17
-; CHECK-NEXT:    global_load_ubyte v18, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ubyte v19, v[1:2], off offset:15
-; CHECK-NEXT:    global_load_ubyte v20, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ubyte v21, v[1:2], off offset:13
-; CHECK-NEXT:    global_load_ubyte v22, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ubyte v23, v[1:2], off offset:11
-; CHECK-NEXT:    global_load_ubyte v24, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ubyte v25, v[1:2], off offset:9
-; CHECK-NEXT:    global_load_ubyte v26, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ubyte v27, v[1:2], off offset:7
-; CHECK-NEXT:    global_load_ubyte v28, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ubyte v29, v[1:2], off offset:5
-; CHECK-NEXT:    global_load_ubyte v30, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ubyte v31, v[1:2], off offset:3
-; CHECK-NEXT:    global_load_ubyte v32, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ubyte v33, v[1:2], off offset:1
-; CHECK-NEXT:    global_load_ubyte v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v33, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -7775,31 +4414,12 @@ define void @memmove_p5_p4_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p4_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -7810,55 +4430,24 @@ define void @memmove_p5_p4_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p4_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ubyte v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    global_load_dwordx3 v[5:7], v[1:2], off offset:16
+; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:28
+; CHECK-NEXT:    global_load_ubyte v9, v[1:2], off offset:30
+; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -7869,55 +4458,19 @@ define void @memmove_p5_p4_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p4_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    global_load_ushort v3, v[1:2], off offset:30
-; CHECK-NEXT:    global_load_ushort v4, v[1:2], off offset:28
-; CHECK-NEXT:    global_load_ushort v5, v[1:2], off offset:26
-; CHECK-NEXT:    global_load_ushort v6, v[1:2], off offset:24
-; CHECK-NEXT:    global_load_ushort v7, v[1:2], off offset:22
-; CHECK-NEXT:    global_load_ushort v8, v[1:2], off offset:20
-; CHECK-NEXT:    global_load_ushort v9, v[1:2], off offset:18
-; CHECK-NEXT:    global_load_ushort v10, v[1:2], off offset:16
-; CHECK-NEXT:    global_load_ushort v11, v[1:2], off offset:14
-; CHECK-NEXT:    global_load_ushort v12, v[1:2], off offset:12
-; CHECK-NEXT:    global_load_ushort v13, v[1:2], off offset:10
-; CHECK-NEXT:    global_load_ushort v14, v[1:2], off offset:8
-; CHECK-NEXT:    global_load_ushort v15, v[1:2], off offset:6
-; CHECK-NEXT:    global_load_ushort v16, v[1:2], off offset:4
-; CHECK-NEXT:    global_load_ushort v17, v[1:2], off offset:2
-; CHECK-NEXT:    global_load_ushort v1, v[1:2], off
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x1
+; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off
+; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v17, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 32, i1 false)
@@ -8062,55 +4615,19 @@ define void @memmove_p5_p5_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p5_sz16_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:3
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)
@@ -8121,100 +4638,34 @@ define void @memmove_p5_p5_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p5_sz31_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1e
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:9
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:7
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:5
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:3
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)
@@ -8225,103 +4676,31 @@ define void @memmove_p5_p5_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p5_sz32_align_1_1:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x1f
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ubyte v6, v1, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    buffer_load_ubyte v7, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ubyte v8, v1, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    buffer_load_ubyte v9, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ubyte v10, v1, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    buffer_load_ubyte v11, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ubyte v12, v1, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    buffer_load_ubyte v13, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ubyte v14, v1, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    buffer_load_ubyte v15, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ubyte v16, v1, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    buffer_load_ubyte v17, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ubyte v18, v1, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    buffer_load_ubyte v19, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ubyte v20, v1, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    buffer_load_ubyte v21, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ubyte v22, v1, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    buffer_load_ubyte v23, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ubyte v24, v1, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    buffer_load_ubyte v25, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ubyte v26, v1, s[0:3], 0 offen offset:7
-; CHECK-NEXT:    buffer_load_ubyte v27, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ubyte v28, v1, s[0:3], 0 offen offset:5
-; CHECK-NEXT:    buffer_load_ubyte v29, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ubyte v30, v1, s[0:3], 0 offen offset:3
-; CHECK-NEXT:    buffer_load_ubyte v31, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ubyte v32, v1, s[0:3], 0 offen offset:1
-; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(31)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:31
-; CHECK-NEXT:    s_waitcnt vmcnt(30)
-; CHECK-NEXT:    buffer_store_byte v3, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(29)
-; CHECK-NEXT:    buffer_store_byte v4, v0, s[0:3], 0 offen offset:29
-; CHECK-NEXT:    s_waitcnt vmcnt(28)
-; CHECK-NEXT:    buffer_store_byte v5, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(27)
-; CHECK-NEXT:    buffer_store_byte v6, v0, s[0:3], 0 offen offset:27
-; CHECK-NEXT:    s_waitcnt vmcnt(26)
-; CHECK-NEXT:    buffer_store_byte v7, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(25)
-; CHECK-NEXT:    buffer_store_byte v8, v0, s[0:3], 0 offen offset:25
-; CHECK-NEXT:    s_waitcnt vmcnt(24)
-; CHECK-NEXT:    buffer_store_byte v9, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(23)
-; CHECK-NEXT:    buffer_store_byte v10, v0, s[0:3], 0 offen offset:23
-; CHECK-NEXT:    s_waitcnt vmcnt(22)
-; CHECK-NEXT:    buffer_store_byte v11, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(21)
-; CHECK-NEXT:    buffer_store_byte v12, v0, s[0:3], 0 offen offset:21
-; CHECK-NEXT:    s_waitcnt vmcnt(20)
-; CHECK-NEXT:    buffer_store_byte v13, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(19)
-; CHECK-NEXT:    buffer_store_byte v14, v0, s[0:3], 0 offen offset:19
-; CHECK-NEXT:    s_waitcnt vmcnt(18)
-; CHECK-NEXT:    buffer_store_byte v15, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(17)
-; CHECK-NEXT:    buffer_store_byte v16, v0, s[0:3], 0 offen offset:17
-; CHECK-NEXT:    s_waitcnt vmcnt(16)
-; CHECK-NEXT:    buffer_store_byte v17, v0, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v18, v0, s[0:3], 0 offen offset:15
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_byte v19, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_byte v20, v0, s[0:3], 0 offen offset:13
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_byte v21, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_byte v22, v0, s[0:3], 0 offen offset:11
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_byte v23, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_byte v24, v0, s[0:3], 0 offen offset:9
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_byte v25, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_byte v26, v0, s[0:3], 0 offen offset:7
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_byte v27, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_byte v28, v0, s[0:3], 0 offen offset:5
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_byte v29, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_byte v30, v0, s[0:3], 0 offen offset:3
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_byte v31, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_byte v32, v0, s[0:3], 0 offen offset:1
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)
@@ -8332,31 +4711,19 @@ define void @memmove_p5_p5_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p5_sz16_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0x7
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    s_clause 0x3
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)
@@ -8367,55 +4734,34 @@ define void @memmove_p5_p5_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p5_sz31_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v11, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v12, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v13, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v14, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v15, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v16, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_byte v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
+; CHECK-NEXT:    s_clause 0x8
+; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_ubyte v1, v1, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_byte v1, v0, s[0:3], 0 offen offset:30
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)
@@ -8426,55 +4772,31 @@ define void @memmove_p5_p5_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr add
 ; CHECK-LABEL: memmove_p5_p5_sz32_align_2_2:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_clause 0xf
-; CHECK-NEXT:    buffer_load_ushort v2, v1, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    buffer_load_ushort v3, v1, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    buffer_load_ushort v4, v1, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    buffer_load_ushort v5, v1, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    buffer_load_ushort v6, v1, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    buffer_load_ushort v7, v1, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    buffer_load_ushort v8, v1, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    buffer_load_ushort v9, v1, s[0:3], 0 offen offset:16
-; CHECK-NEXT:    buffer_load_ushort v10, v1, s[0:3], 0 offen offset:14
-; CHECK-NEXT:    buffer_load_ushort v11, v1, s[0:3], 0 offen offset:12
-; CHECK-NEXT:    buffer_load_ushort v12, v1, s[0:3], 0 offen offset:10
-; CHECK-NEXT:    buffer_load_ushort v13, v1, s[0:3], 0 offen offset:8
-; CHECK-NEXT:    buffer_load_ushort v14, v1, s[0:3], 0 offen offset:6
-; CHECK-NEXT:    buffer_load_ushort v15, v1, s[0:3], 0 offen offset:4
-; CHECK-NEXT:    buffer_load_ushort v16, v1, s[0:3], 0 offen offset:2
-; CHECK-NEXT:    buffer_load_ushort v1, v1, s[0:3], 0 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(15)
-; CHECK-NEXT:    buffer_store_short v2, v0, s[0:3], 0 offen offset:30
-; CHECK-NEXT:    s_waitcnt vmcnt(14)
-; CHECK-NEXT:    buffer_store_short v3, v0, s[0:3], 0 offen offset:28
-; CHECK-NEXT:    s_waitcnt vmcnt(13)
-; CHECK-NEXT:    buffer_store_short v4, v0, s[0:3], 0 offen offset:26
-; CHECK-NEXT:    s_waitcnt vmcnt(12)
-; CHECK-NEXT:    buffer_store_short v5, v0, s[0:3], 0 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(11)
-; CHECK-NEXT:    buffer_store_short v6, v0, s[0:3], 0 offen offset:22
-; CHECK-NEXT:    s_waitcnt vmcnt(10)
-; CHECK-NEXT:    buffer_store_short v7, v0, s[0:3], 0 offen offset:20
-; CHECK-NEXT:    s_waitcnt vmcnt(9)
-; CHECK-NEXT:    buffer_store_short v8, v0, s[0:3], 0 offen offset:18
-; CHECK-NEXT:    s_waitcnt vmcnt(8)
-; CHECK-NEXT:    buffer_store_short v9, v0, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    s_clause 0x7
+; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:24
+; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:28
+; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(7)
-; CHECK-NEXT:    buffer_store_short v10, v0, s[0:3], 0 offen offset:14
+; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
 ; CHECK-NEXT:    s_waitcnt vmcnt(6)
-; CHECK-NEXT:    buffer_store_short v11, v0, s[0:3], 0 offen offset:12
+; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:28
 ; CHECK-NEXT:    s_waitcnt vmcnt(5)
-; CHECK-NEXT:    buffer_store_short v12, v0, s[0:3], 0 offen offset:10
+; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:16
 ; CHECK-NEXT:    s_waitcnt vmcnt(4)
-; CHECK-NEXT:    buffer_store_short v13, v0, s[0:3], 0 offen offset:8
+; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:20
 ; CHECK-NEXT:    s_waitcnt vmcnt(3)
-; CHECK-NEXT:    buffer_store_short v14, v0, s[0:3], 0 offen offset:6
+; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:8
 ; CHECK-NEXT:    s_waitcnt vmcnt(2)
-; CHECK-NEXT:    buffer_store_short v15, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:12
 ; CHECK-NEXT:    s_waitcnt vmcnt(1)
-; CHECK-NEXT:    buffer_store_short v16, v0, s[0:3], 0 offen offset:2
+; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_short v1, v0, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
   tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir
index ab6f38c83b94e2687d631e1efb6bbdb79f5fe44e..ee5546befd12d3ce087f22dd595688cd7264ff14 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir
+++ b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir
@@ -1,7 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=si-pre-emit-peephole -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=si-pre-emit-peephole -verify-machineinstrs  %s -o - | FileCheck %s
 # Make sure mandatory skips are not removed around mode defs.
-# FIXME: -amdgpu-skip-threshold seems to be backwards.
 
 ---
 
diff --git a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir
index d154fc04b0cd916c99a4e25e6e005e5a0b462014..20de119471ba39e6f23faec6ce07ede0514b29fa 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir
+++ b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir
@@ -1,7 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=si-pre-emit-peephole -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=si-pre-emit-peephole -verify-machineinstrs  %s -o - | FileCheck %s
 # Make sure mandatory skips are not removed around mode defs.
-# FIXME: -amdgpu-skip-threshold seems to be backwards.
 
 ---
 
diff --git a/llvm/test/CodeGen/AMDGPU/ret_jump.ll b/llvm/test/CodeGen/AMDGPU/ret_jump.ll
index ad38d78ddb2ff1f70e3388ea28bb8c61900335ca..66a55d9eb128c656d30d6450e724bfda81833b57 100644
--- a/llvm/test/CodeGen/AMDGPU/ret_jump.ll
+++ b/llvm/test/CodeGen/AMDGPU/ret_jump.ll
@@ -65,7 +65,6 @@ ret.bb:                                          ; preds = %else, %main_body
 
 ; GCN: .LBB{{[0-9]+_[0-9]+}}: ; %else
 ; GCN: s_and_saveexec_b64 [[SAVE_EXEC:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN-NEXT: s_cbranch_execz .LBB1_{{[0-9]+}}
 
 ; GCN-NEXT:  ; %unreachable.bb
 ; GCN: ds_write_b32
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll b/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll
index 94bc6d46b2395bb2815f1e7b0cd4c1e1a174c454..8ad6a4e534d23233af10ed0f177ae5144f5b56a2 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll
@@ -19,8 +19,8 @@ define void @extracted_values(ptr %ret_struct, ptr addrspace(3) %arg0, ptr addrs
 ; CHECK-NEXT:    v_sub_f16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; CHECK-NEXT:    v_add_f16_e32 v4, v6, v7
 ; CHECK-NEXT:    v_add_f16_e32 v2, v3, v2
-; CHECK-NEXT:    flat_store_short v[0:1], v4
-; CHECK-NEXT:    flat_store_short v[0:1], v2 offset:2
+; CHECK-NEXT:    v_pack_b32_f16 v2, v4, v2
+; CHECK-NEXT:    flat_store_dword v[0:1], v2
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir
index bb0a707a7c90b724d8274ae39d95bb0883159081..94cbe568a6a44725b55088c0fc669400791acb6e 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir
@@ -111,7 +111,9 @@ body:             |
   ; GCN-NEXT:   $sgpr22 = IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr5 = IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr5 = SI_SPILL_S32_TO_VGPR $sgpr22, 0, killed $vgpr5
+  ; GCN-NEXT:   $sgpr0 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GCN-NEXT:   dead $vgpr1 = V_SET_INACTIVE_B32 0, $vgpr0, 0, 0, $sgpr_null, implicit $exec, implicit-def $scc
+  ; GCN-NEXT:   $exec_lo = EXIT_STRICT_WWM $sgpr0
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT: bb.1:
   ; GCN-NEXT:   successors: %bb.2(0x80000000)
@@ -211,7 +213,9 @@ body:             |
 
     $sgpr22 = IMPLICIT_DEF
     SI_SPILL_S32_SAVE $sgpr22, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.0, addrspace 5)
+    $sgpr0 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
     %0:vgpr_32 = V_SET_INACTIVE_B32 0, $vgpr0, 0, 0, $sgpr_null, implicit $exec, implicit-def $scc
+    $exec_lo = EXIT_STRICT_WWM $sgpr0
 
   bb.1:
     KILL implicit-def $vcc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr32_sgpr33_sgpr34_sgpr35_sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit-def $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit-def $sgpr96_sgpr97_sgpr98_sgpr99_sgpr100_sgpr101_sgpr102_sgpr103
diff --git a/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll b/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll
index 1eef7b967f6d998f3a647f3ba5ad8a13854723c4..f232275c998d2363496eb03c8ed0e4f241f20a43 100644
--- a/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll
@@ -122,14 +122,16 @@ define void @my_func(i32 %0) {
 ; GCN-NEXT:    s_cbranch_execz .LBB0_8
 ; GCN-NEXT:  .LBB0_2: ; %Flow11
 ; GCN-NEXT:    s_and_saveexec_b64 s[8:9], s[6:7]
+; GCN-NEXT:    s_cbranch_execz .LBB0_4
 ; GCN-NEXT:  .LBB0_3: ; %do.body
 ; GCN-NEXT:    s_or_b64 s[4:5], s[4:5], exec
 ; GCN-NEXT:  .LBB0_4: ; %Flow17
 ; GCN-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB0_6
 ; GCN-NEXT:  ; %bb.5: ; %UnifiedUnreachableBlock
 ; GCN-NEXT:    ; divergent unreachable
-; GCN-NEXT:  ; %bb.6: ; %UnifiedReturnBlock
+; GCN-NEXT:  .LBB0_6: ; %UnifiedReturnBlock
 ; GCN-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .LBB0_7: ; %Flow
@@ -149,12 +151,14 @@ define void @my_func(i32 %0) {
 ; GCN-NEXT:    s_mov_b64 s[6:7], 0
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB0_12
 ; GCN-NEXT:  ; %bb.11: ; %LeafBlock5
 ; GCN-NEXT:    s_mov_b64 s[6:7], exec
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
 ; GCN-NEXT:    s_and_b64 s[8:9], vcc, exec
-; GCN-NEXT:  ; %bb.12: ; %Flow13
+; GCN-NEXT:  .LBB0_12: ; %Flow13
 ; GCN-NEXT:    s_andn2_saveexec_b64 s[10:11], s[4:5]
+; GCN-NEXT:    s_cbranch_execz .LBB0_14
 ; GCN-NEXT:  ; %bb.13: ; %LeafBlock3
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
@@ -164,7 +168,7 @@ define void @my_func(i32 %0) {
 ; GCN-NEXT:    s_and_b64 s[12:13], vcc, exec
 ; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[4:5]
 ; GCN-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
-; GCN-NEXT:  ; %bb.14: ; %Flow14
+; GCN-NEXT:  .LBB0_14: ; %Flow14
 ; GCN-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-NEXT:    s_and_saveexec_b64 s[10:11], s[8:9]
@@ -173,10 +177,11 @@ define void @my_func(i32 %0) {
 ; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, 1, v0
 ; GCN-NEXT:    s_mov_b64 s[8:9], -1
 ; GCN-NEXT:    s_and_saveexec_b64 s[12:13], vcc
+; GCN-NEXT:    s_cbranch_execz .LBB0_17
 ; GCN-NEXT:  ; %bb.16: ; %do.body.i.i.i.i
 ; GCN-NEXT:    s_mov_b64 s[4:5], exec
 ; GCN-NEXT:    s_xor_b64 s[8:9], exec, -1
-; GCN-NEXT:  ; %bb.17: ; %Flow16
+; GCN-NEXT:  .LBB0_17: ; %Flow16
 ; GCN-NEXT:    s_or_b64 exec, exec, s[12:13]
 ; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GCN-NEXT:    s_andn2_b64 s[6:7], s[6:7], exec
diff --git a/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir b/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir
index 558c9d633127c95ffb48036d18e04e1b56a9dd25..5bc13617bea954546dfa5efb3bd8262411a6fe22 100644
--- a/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir
+++ b/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -run-pass=si-pre-emit-peephole -amdgpu-skip-threshold=1000000 -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -run-pass=si-pre-emit-peephole -o -  %s | FileCheck %s
 
 ---
 name: skip_branch_taildup_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
index 4e734d6e0884bc93b8ca8c3de850f11f33a6baa4..fc33a274d7b11af4823a5b34d428f6b00cea1e64 100644
--- a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
@@ -607,7 +607,14 @@ define amdgpu_kernel void @local_store_align1_v16i8(ptr addrspace(3) %out) #0 {
 ; MUBUF: buffer_load_ubyte
 ; MUBUF: buffer_load_ubyte
 ; MUBUF: buffer_load_ubyte
-; FLATSCR: scratch_load_dwordx2
+; FLATSCR: scratch_load_ubyte
+; FLATSCR: scratch_load_ubyte
+; FLATSCR: scratch_load_ubyte
+; FLATSCR: scratch_load_ubyte
+; FLATSCR: scratch_load_ubyte
+; FLATSCR: scratch_load_ubyte
+; FLATSCR: scratch_load_ubyte
+; FLATSCR: scratch_load_ubyte
 define double @private_load_align1_f64(ptr addrspace(5) %in) {
   %x = load double, ptr addrspace(5) %in, align 1
   ret double %x
@@ -622,7 +629,14 @@ define double @private_load_align1_f64(ptr addrspace(5) %in) {
 ; MUBUF: buffer_store_byte
 ; MUBUF: buffer_store_byte
 ; MUBUF: buffer_store_byte
-; FLATSCR: scratch_store_dwordx2
+; FLATSCR: scratch_store_byte
+; FLATSCR: scratch_store_byte
+; FLATSCR: scratch_store_byte
+; FLATSCR: scratch_store_byte
+; FLATSCR: scratch_store_byte
+; FLATSCR: scratch_store_byte
+; FLATSCR: scratch_store_byte
+; FLATSCR: scratch_store_byte
 define void @private_store_align1_f64(ptr addrspace(5) %out, double %x) #0 {
   store double %x, ptr addrspace(5) %out, align 1
   ret void
@@ -651,7 +665,10 @@ define void @private_store_align4_f64(ptr addrspace(5) %out, double %x) #0 {
 ; MUBUF: buffer_load_ushort
 ; MUBUF: buffer_load_ushort
 ; MUBUF: buffer_load_ushort
-; FLATSCR: scratch_load_dwordx2
+; FLATSCR: scratch_load_ushort
+; FLATSCR: scratch_load_ushort
+; FLATSCR: scratch_load_ushort
+; FLATSCR: scratch_load_ushort
 define double @private_load_align2_f64(ptr addrspace(5) %in) {
   %x = load double, ptr addrspace(5) %in, align 2
   ret double %x
@@ -662,7 +679,10 @@ define double @private_load_align2_f64(ptr addrspace(5) %in) {
 ; MUBUF: buffer_store_short
 ; MUBUF: buffer_store_short
 ; MUBUF: buffer_store_short
-; FLATSCR: scratch_store_dwordx2
+; FLATSCR: scratch_store_short
+; FLATSCR: scratch_store_short
+; FLATSCR: scratch_store_short
+; FLATSCR: scratch_store_short
 define void @private_store_align2_f64(ptr addrspace(5) %out, double %x) #0 {
   store double %x, ptr addrspace(5) %out, align 2
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-debug-non-first-terminators.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-debug-non-first-terminators.mir
new file mode 100644
index 0000000000000000000000000000000000000000..bccf266117a7d24affbcc117eabfc8514b7ba6bc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-debug-non-first-terminators.mir
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass si-insert-waitcnts -amdgpu-waitcnt-forcezero=1 %s -o - | FileCheck %s
+
+---
+name: waitcnt-debug-non-first-terminators
+liveins:
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; CHECK-LABEL: name: waitcnt-debug-non-first-terminators
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit $scc
+  ; CHECK-NEXT:   S_BRANCH %bb.2, implicit $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_WAITCNT 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_WAITCNT 0
+  ; CHECK-NEXT:   S_NOP 0
+  bb.0:
+    S_CBRANCH_SCC1 %bb.1, implicit $scc
+    S_BRANCH %bb.2, implicit $scc
+  bb.1:
+    S_NOP 0
+  bb.2:
+    S_NOP 0
+...
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-const.mir b/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-const.mir
new file mode 100644
index 0000000000000000000000000000000000000000..70d4a527493c3a9e8744e74e386895f195f41444
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-const.mir
@@ -0,0 +1,20 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple arm-- -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            get_const
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: get_const
+    ; CHECK: [[CONSTANT_POOL:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[CONSTANT_POOL]](p0) :: (load (s32) from constant-pool)
+    ; CHECK-NEXT: $r0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+    %0:_(s32) = G_CONSTANT i32 287454020
+    $r0 = COPY %0(s32)
+    MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+...
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir b/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir
index 96461e044813d4936c6296ed1d20d3bd49ecabd4..a35f82722adaf4c5ed4db90dde5aa6ba30f6941f 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir
@@ -1,9 +1,9 @@
 # RUN: llc -O0 -mtriple arm-linux-gnueabihf -mattr=+vfp2 -float-abi=hard -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix HARD
-# RUN: llc -O0 -mtriple arm-linux-gnueabi -mattr=+vfp2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-AEABI
-# RUN: llc -O0 -mtriple arm-linux-gnu -mattr=+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s  -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-DEFAULT
+# RUN: llc -O0 -mtriple arm-linux-gnueabi -mattr=+vfp2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-AEABI -check-prefix SOFT_POOL
+# RUN: llc -O0 -mtriple arm-linux-gnu -mattr=+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s  -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-DEFAULT -check-prefix SOFT_POOL
 # RUN: llc -O0 -mtriple thumb-linux-gnueabihf -mattr=+v6t2,+vfp2 -float-abi=hard -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix HARD
-# RUN: llc -O0 -mtriple thumb-linux-gnueabi -mattr=+v6t2,+vfp2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-AEABI
-# RUN: llc -O0 -mtriple thumb-linux-gnu -mattr=+v6t2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s  -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-DEFAULT
+# RUN: llc -O0 -mtriple thumb-linux-gnueabi -mattr=+v6t2,+vfp2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix SOFT -check-prefix=SOFT-AEABI -check-prefix SOFT_CONST
+# RUN: llc -O0 -mtriple thumb-linux-gnu -mattr=+v6t2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s  -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-DEFAULT -check-prefix SOFT_CONST
 --- |
   define void @test_frem_float() { ret void }
   define void @test_frem_double() { ret void }
@@ -657,10 +657,20 @@ body:             |
   bb.0:
     liveins:
 
+    ; SOFT_POOL: constants:
+    ; SOFT_POOL: id: 0
+    ; SOFT_POOL: value: i32 -1073532109
+    ; SOFT_POOL: id: 1
+    ; SOFT_POOL: value: i32 858993459
+
     ; HARD: [[R:%[0-9]+]]:_(s64) = G_FCONSTANT double -2.4
     ; SOFT-NOT: G_FCONSTANT
-    ; SOFT-DAG: [[HI:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1073532109
-    ; SOFT-DAG: [[LO:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; SOFT_CONST-DAG: [[HI:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1073532109
+    ; SOFT_CONST-DAG: [[LO:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; SOFT_POOL-DAG: [[HIPTR:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0
+    ; SOFT_POOL-DAG: [[HI:%[0-9]+]]:_(s32) = G_LOAD [[HIPTR]]
+    ; SOFT_POOL-DAG: [[LOPTR:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.1
+    ; SOFT_POOL-DAG: [[LO:%[0-9]+]]:_(s32) = G_LOAD [[LOPTR]]
     ; SOFT-NOT: G_FCONSTANT
     %0(s64) = G_FCONSTANT double -2.4
     ; HARD-DAG: G_UNMERGE_VALUES [[R]](s64)
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir b/llvm/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir
index 44e17025032021b7adc48c66b07a655274a62015..87d1785809b9091231c1e5fe092dfb7bcbb6a651 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir
@@ -109,9 +109,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -135,9 +135,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -161,9 +161,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -187,9 +187,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -213,9 +213,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -239,9 +239,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -265,9 +265,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -291,9 +291,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -317,9 +317,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -343,9 +343,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -369,9 +369,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -395,12 +395,12 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
-# CHECK: - { id: 4, class: gprb, preferred-register: '' }
-# CHECK: - { id: 5, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 4, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 5, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -429,11 +429,11 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
-# CHECK: - { id: 4, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 4, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -464,11 +464,11 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
-# CHECK: - { id: 4, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 4, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -497,9 +497,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -522,7 +522,7 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
 body:             |
@@ -538,8 +538,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -557,8 +557,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -576,8 +576,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -595,7 +595,7 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
 body:             |
@@ -611,9 +611,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -635,9 +635,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -659,9 +659,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -683,9 +683,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -707,10 +707,10 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -736,10 +736,10 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -765,10 +765,10 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -794,11 +794,11 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: gprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
-# CHECK: - { id: 4, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 4, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -829,8 +829,8 @@ selected:        false
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
-# CHECK: { id: 0, class: gprb, preferred-register: '' }
-# CHECK: { id: 1, class: gprb, preferred-register: '' }
+# CHECK: { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 # Check that we map the condition of the G_BRCOND into the GPR.
 # For the G_BR, there are no registers to map, but make sure we don't crash.
 body:             |
@@ -864,11 +864,11 @@ registers:
   - { id: 2, class: _ }
   - { id: 3, class: _ }
   - { id: 4, class: _ }
-# CHECK: { id: 0, class: gprb, preferred-register: '' }
-# CHECK: { id: 1, class: gprb, preferred-register: '' }
-# CHECK: { id: 2, class: gprb, preferred-register: '' }
-# CHECK: { id: 3, class: gprb, preferred-register: '' }
-# CHECK: { id: 4, class: gprb, preferred-register: '' }
+# CHECK: { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 2, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 3, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 4, class: gprb, preferred-register: '', flags: [  ] }
 body:             |
   bb.0:
     successors: %bb.1(0x40000000), %bb.2(0x40000000)
@@ -905,11 +905,11 @@ registers:
   - { id: 2, class: _ }
   - { id: 3, class: _ }
   - { id: 4, class: _ }
-# CHECK: { id: 0, class: gprb, preferred-register: '' }
-# CHECK: { id: 1, class: gprb, preferred-register: '' }
-# CHECK: { id: 2, class: fprb, preferred-register: '' }
-# CHECK: { id: 3, class: fprb, preferred-register: '' }
-# CHECK: { id: 4, class: fprb, preferred-register: '' }
+# CHECK: { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 2, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 3, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: { id: 4, class: fprb, preferred-register: '', flags: [  ] }
 body:             |
   bb.0:
     successors: %bb.1(0x40000000), %bb.2(0x40000000)
@@ -939,9 +939,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -965,9 +965,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -991,9 +991,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -1017,9 +1017,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -1043,9 +1043,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -1069,9 +1069,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -1095,9 +1095,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -1121,9 +1121,9 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
@@ -1147,8 +1147,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1169,8 +1169,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1191,10 +1191,10 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
-# CHECK: - { id: 3, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1218,10 +1218,10 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
-# CHECK: - { id: 3, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1245,8 +1245,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1266,8 +1266,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1287,7 +1287,7 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
 body:             |
@@ -1305,7 +1305,7 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
 body:             |
@@ -1323,8 +1323,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1345,8 +1345,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1366,8 +1366,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1388,8 +1388,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: fprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1409,8 +1409,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1431,8 +1431,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1452,8 +1452,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1474,8 +1474,8 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: fprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: fprb, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -1495,11 +1495,11 @@ legalized:       true
 regBankSelected: false
 selected:        false
 # CHECK: registers:
-# CHECK: - { id: 0, class: gprb, preferred-register: '' }
-# CHECK: - { id: 1, class: gprb, preferred-register: '' }
-# CHECK: - { id: 2, class: fprb, preferred-register: '' }
-# CHECK: - { id: 3, class: gprb, preferred-register: '' }
-# CHECK: - { id: 4, class: gprb, preferred-register: '' }
+# CHECK: - { id: 0, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 1, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 2, class: fprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 3, class: gprb, preferred-register: '', flags: [  ] }
+# CHECK: - { id: 4, class: gprb, preferred-register: '', flags: [  ] }
 
 registers:
   - { id: 0, class: _ }
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/select-constpool.mir b/llvm/test/CodeGen/ARM/GlobalISel/select-constpool.mir
new file mode 100644
index 0000000000000000000000000000000000000000..1c16c37e8b5cd5862200fdd54001352473616100
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/GlobalISel/select-constpool.mir
@@ -0,0 +1,25 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple arm-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            get_const
+legalized:       true
+regBankSelected: true
+selected:        false
+tracksRegLiveness: true
+constants:
+  - id:              0
+    value:           i32 287454020
+    alignment:       4
+    isTargetSpecific: false
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: get_const
+    ; CHECK: [[LDRcp:%[0-9]+]]:gpr = LDRcp %const.0, 0, 14 /* CC::al */, $noreg :: (load (s32) from constant-pool)
+    ; CHECK-NEXT: $r0 = COPY [[LDRcp]]
+    ; CHECK-NEXT: MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+    %1:gprb(p0) = G_CONSTANT_POOL %const.0
+    %0:gprb(s32) = G_LOAD %1(p0) :: (load (s32) from constant-pool)
+    $r0 = COPY %0(s32)
+    MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+...
diff --git a/llvm/test/CodeGen/ARM/cmse-cve-2021-35465.ll b/llvm/test/CodeGen/ARM/cmse-cve-2021-35465.ll
index 31a0b9814d7f1f43b70880c34c83b38ce96f26f8..5be0641420fc299868258a52b2857b9cdc31ede7 100644
--- a/llvm/test/CodeGen/ARM/cmse-cve-2021-35465.ll
+++ b/llvm/test/CodeGen/ARM/cmse-cve-2021-35465.ll
@@ -7,6 +7,9 @@
 ; RUN: llc %s -o - -mtriple=thumbv8m.main -mcpu=cortex-m33 -verify-machineinstrs | \
 ; RUN:   FileCheck %s --check-prefix=CHECK-8M-FP-CVE-2021-35465
 ;
+; RUN: llc %s -o - -mtriple=thumbv8m.main -mcpu=star-mc1 -verify-machineinstrs | \
+; RUN:   FileCheck %s --check-prefix=CHECK-8M-FP-CVE-2021-35465
+;
 ; RUN: llc %s -o - -mtriple=thumbv8m.main -mcpu=cortex-m35p -verify-machineinstrs | \
 ; RUN:   FileCheck %s --check-prefix=CHECK-8M-FP-CVE-2021-35465
 ;
@@ -17,6 +20,9 @@
 ; RUN: llc %s -o - -mtriple=thumbv8m.main -mcpu=cortex-m33 -mattr=-fpregs -verify-machineinstrs | \
 ; RUN:   FileCheck %s --check-prefix=CHECK-8M-NOFP-CVE-2021-35465
 ;
+; RUN: llc %s -o - -mtriple=thumbv8m.main -mcpu=star-mc1 -mattr=-fpregs -verify-machineinstrs | \
+; RUN:   FileCheck %s --check-prefix=CHECK-8M-NOFP-CVE-2021-35465
+;
 ; RUN: llc %s -o - -mtriple=thumbv8m.main -mcpu=cortex-m35p -mattr=-fpregs -verify-machineinstrs | \
 ; RUN:   FileCheck %s --check-prefix=CHECK-8M-NOFP-CVE-2021-35465
 ;
diff --git a/llvm/test/CodeGen/ARM/peephole-callee-save-regalloc.mir b/llvm/test/CodeGen/ARM/peephole-callee-save-regalloc.mir
index 8ee4a80067fac4335e88499b121700c27c981a35..c02eba580ca41ae45a0f6134da1e777b55d44f2b 100644
--- a/llvm/test/CodeGen/ARM/peephole-callee-save-regalloc.mir
+++ b/llvm/test/CodeGen/ARM/peephole-callee-save-regalloc.mir
@@ -5,7 +5,7 @@
 # the register class of the consumer.
 #
 # Check that register class for %5 is unchanged as 'tcgpr'
-# CHECK: { id: 5, class: tcgpr, preferred-register: '' }
+# CHECK: { id: 5, class: tcgpr, preferred-register: '', flags: [  ] }
 # CHECK: TCRETURNri killed %5
 
 --- |
diff --git a/llvm/test/CodeGen/MIR/AArch64/register-operand-bank.mir b/llvm/test/CodeGen/MIR/AArch64/register-operand-bank.mir
index 5b5102f9e1968a6829262206bf3b5061a7db7f6d..dd0e8fb5c6000dd606c0e29a095beb9db5a830a7 100644
--- a/llvm/test/CodeGen/MIR/AArch64/register-operand-bank.mir
+++ b/llvm/test/CodeGen/MIR/AArch64/register-operand-bank.mir
@@ -6,8 +6,8 @@
 ---
 # CHECK-LABEL: name: func
 # CHECK: registers:
-# CHECK:   - { id: 0, class: gpr, preferred-register: '' }
-# CHECK:   - { id: 1, class: fpr, preferred-register: '' }
+# CHECK:   - { id: 0, class: gpr, preferred-register: '', flags: [  ] }
+# CHECK:   - { id: 1, class: fpr, preferred-register: '', flags: [  ] }
 name: func
 body: |
   bb.0:
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
index ebbb89b7816c584eddfad7550b37cf17b863d506..51795a4fea515ec79ca272558d0c11bb4c4bcc6e 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
@@ -578,3 +578,18 @@ body:             |
     SI_RETURN
 
 ...
+---
+name: vregs
+# FULL: registers:
+# FULL-NEXT:   - { id: 0, class: vgpr_32, preferred-register: '$vgpr1', flags: [ WWM_REG ] }
+# FULL-NEXT:   - { id: 1, class: sgpr_64, preferred-register: '$sgpr0_sgpr1', flags: [  ] }
+# FULL-NEXT:   - { id: 2, class: sgpr_64, preferred-register: '', flags: [  ] }
+registers:
+  - { id: 0, class: vgpr_32, preferred-register: $vgpr1, flags: [ WWM_REG ]}
+  - { id: 1, class: sgpr_64, preferred-register: $sgpr0_sgpr1 }
+  - { id: 2, class: sgpr_64, flags: [ ] }
+body: |
+  bb.0:
+    %2:sgpr_64 = COPY %1
+    %1:sgpr_64 = COPY %0
+...
diff --git a/llvm/test/CodeGen/MIR/Generic/register-flag-error.mir b/llvm/test/CodeGen/MIR/Generic/register-flag-error.mir
new file mode 100644
index 0000000000000000000000000000000000000000..efbcf23af071ed58ba7b78310f02421eedfab93f
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/Generic/register-flag-error.mir
@@ -0,0 +1,13 @@
+# RUN: not llc -run-pass=none -filetype=null %s 2>&1 | FileCheck %s --check-prefix=ERR
+
+---
+name:  flags
+registers:
+  - { id: 0, class: _, flags: [ 'VFLAG_ERR' ] }
+body: |
+  bb.0:
+    liveins: $w0
+    %0 = G_ADD $w0, $w0
+...
+# ERR: use of undefined register flag
+# ERR: VFLAG_ERR
diff --git a/llvm/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir b/llvm/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir
index 7ed390570adc7c6203916f2817215132c2117f67..03f2ec4d6cd3f7765f287deeb0aa87f7549f5e55 100644
--- a/llvm/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir
+++ b/llvm/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir
@@ -14,8 +14,8 @@ name:            test
 tracksRegLiveness: true
 registers:
   - { id: 0, class: gr32 }
-  # CHECK: - { id: 1, class: gr32, preferred-register: '%0' }
-  # CHECK: - { id: 2, class: gr32, preferred-register: '$edi' }
+  # CHECK: - { id: 1, class: gr32, preferred-register: '%0', flags: [  ] }
+  # CHECK: - { id: 2, class: gr32, preferred-register: '$edi', flags: [  ] }
   - { id: 1, class: gr32, preferred-register: '%0' }
   - { id: 2, class: gr32, preferred-register: '$edi' }
 body: |
diff --git a/llvm/test/CodeGen/MIR/X86/generic-instr-type.mir b/llvm/test/CodeGen/MIR/X86/generic-instr-type.mir
index 710a18ac3aeff4375921416ea15ccfb62c480f69..7514cdab0ab11001086705fe1404522c101c4494 100644
--- a/llvm/test/CodeGen/MIR/X86/generic-instr-type.mir
+++ b/llvm/test/CodeGen/MIR/X86/generic-instr-type.mir
@@ -18,11 +18,11 @@
 ---
 name:            test_vregs
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 4, class: _, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 3, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 4, class: _, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
diff --git a/llvm/test/CodeGen/MIR/X86/register-operand-class.mir b/llvm/test/CodeGen/MIR/X86/register-operand-class.mir
index f62d7294eabc10e5337b2bc5f69c822e58dabc57..521722d9f24c54dccd0483706815e5c501a170f2 100644
--- a/llvm/test/CodeGen/MIR/X86/register-operand-class.mir
+++ b/llvm/test/CodeGen/MIR/X86/register-operand-class.mir
@@ -6,11 +6,11 @@
 ---
 # CHECK-LABEL: name: func
 # CHECK: registers:
-# CHECK:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK:   - { id: 1, class: gr64, preferred-register: '' }
-# CHECK:   - { id: 2, class: gr32, preferred-register: '' }
-# CHECK:   - { id: 3, class: gr16, preferred-register: '' }
-# CHECK:   - { id: 4, class: _, preferred-register: '' }
+# CHECK:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 3, class: gr16, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 4, class: _, preferred-register: '', flags: [   ] }
 name: func
 body: |
   bb.0:
diff --git a/llvm/test/CodeGen/MIR/X86/roundtrip.mir b/llvm/test/CodeGen/MIR/X86/roundtrip.mir
index 46f08ad1a214dab717e206dc3a8d5edaace96961..6124113a0dd88f13017ad3e1e2cebf68114fc467 100644
--- a/llvm/test/CodeGen/MIR/X86/roundtrip.mir
+++ b/llvm/test/CodeGen/MIR/X86/roundtrip.mir
@@ -2,8 +2,8 @@
 ---
 # CHECK-LABEL: name: func0
 # CHECK: registers:
-# CHECK:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK:   - { id: 1, class: gr32, preferred-register: '' }
+# CHECK:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 # CHECK: body: |
 # CHECK:   bb.0:
 # CHECK:     %0:gr32 = MOV32r0 implicit-def $eflags
diff --git a/llvm/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir b/llvm/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir
index 84d298dbd407009ab9560f2850544b6dbc0bf025..aacf66c98cf5d367dad1692f95afe5ed6dee932d 100644
--- a/llvm/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir
+++ b/llvm/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir
@@ -15,9 +15,9 @@
 name:            test
 tracksRegLiveness: true
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gr32, preferred-register: '$esi' }
-# CHECK-NEXT:  - { id: 2, class: gr32, preferred-register: '$edi' }
+# CHECK-NEXT:  - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:  - { id: 1, class: gr32, preferred-register: '$esi', flags: [   ] }
+# CHECK-NEXT:  - { id: 2, class: gr32, preferred-register: '$edi', flags: [   ] }
 registers:
   - { id: 0, class: gr32 }
   - { id: 1, class: gr32, preferred-register: '$esi' }
diff --git a/llvm/test/CodeGen/MIR/X86/virtual-registers.mir b/llvm/test/CodeGen/MIR/X86/virtual-registers.mir
index e317746e08a18e29dd7ea9f77f20ed1052fd180a..819f65638b67de3da9b509d6256e62c517c5dca7 100644
--- a/llvm/test/CodeGen/MIR/X86/virtual-registers.mir
+++ b/llvm/test/CodeGen/MIR/X86/virtual-registers.mir
@@ -33,9 +33,9 @@
 name:            bar
 tracksRegLiveness: true
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gr32 }
   - { id: 1, class: gr32 }
@@ -67,9 +67,9 @@ name:            foo
 tracksRegLiveness: true
 # CHECK: name: foo
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 2, class: gr32 }
   - { id: 0, class: gr32 }
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index 84dde539ce4c470a60ebd35ae4bb7cb7508cd6f8..96a4359d0ec43e7161f31e659fbfd3f39ff4d583 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -101,38 +101,38 @@ define <4 x i8> @test_add(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-LABEL: test_add(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<13>;
-; CHECK-NEXT:    .reg .b32 %r<18>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_add_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_add_param_0];
-; CHECK-NEXT:    bfe.u32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r2, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; CHECK-NEXT:    cvt.u32.u16 %r5, %rs3;
-; CHECK-NEXT:    bfe.u32 %r6, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r2, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs4, %r6;
-; CHECK-NEXT:    bfe.u32 %r7, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; CHECK-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-NEXT:    prmt.b32 %r9, %r8, %r5, 16435;
-; CHECK-NEXT:    bfe.u32 %r10, %r2, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r9, %r8, %r5, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r2, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs7, %r10;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; CHECK-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; CHECK-NEXT:    cvt.u32.u16 %r12, %rs9;
-; CHECK-NEXT:    bfe.u32 %r13, %r2, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs10, %r13;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs11, %r14;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r9, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r2, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs10, %r14;
+; CHECK-NEXT:    bfe.u32 %r15, %r1, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs11, %r15;
 ; CHECK-NEXT:    add.s16 %rs12, %rs11, %rs10;
-; CHECK-NEXT:    cvt.u32.u16 %r15, %rs12;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r12, 13120;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r9, 21520;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs12;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r13, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r17;
 ; CHECK-NEXT:    ret;
   %r = add <4 x i8> %a, %b
@@ -143,29 +143,29 @@ define <4 x i8> @test_add_imm_0(<4 x i8> %a) #0 {
 ; CHECK-LABEL: test_add_imm_0(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<9>;
-; CHECK-NEXT:    .reg .b32 %r<13>;
+; CHECK-NEXT:    .reg .b32 %r<14>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_add_imm_0_param_0];
-; CHECK-NEXT:    bfe.u32 %r2, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r2, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs1, %r2;
-; CHECK-NEXT:    add.s16 %rs2, %rs1, 4;
+; CHECK-NEXT:    add.s16 %rs2, %rs1, 1;
 ; CHECK-NEXT:    cvt.u32.u16 %r3, %rs2;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs3, %r4;
-; CHECK-NEXT:    add.s16 %rs4, %rs3, 3;
+; CHECK-NEXT:    add.s16 %rs4, %rs3, 2;
 ; CHECK-NEXT:    cvt.u32.u16 %r5, %rs4;
-; CHECK-NEXT:    prmt.b32 %r6, %r5, %r3, 16435;
-; CHECK-NEXT:    bfe.u32 %r7, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r6, %r5, %r3, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs5, %r7;
-; CHECK-NEXT:    add.s16 %rs6, %rs5, 2;
+; CHECK-NEXT:    add.s16 %rs6, %rs5, 3;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-NEXT:    bfe.u32 %r9, %r1, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs7, %r9;
-; CHECK-NEXT:    add.s16 %rs8, %rs7, 1;
-; CHECK-NEXT:    cvt.u32.u16 %r10, %rs8;
-; CHECK-NEXT:    prmt.b32 %r11, %r10, %r8, 13120;
-; CHECK-NEXT:    prmt.b32 %r12, %r11, %r6, 21520;
+; CHECK-NEXT:    bfi.b32 %r9, %r8, %r6, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs7, %r10;
+; CHECK-NEXT:    add.s16 %rs8, %rs7, 4;
+; CHECK-NEXT:    cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT:    bfi.b32 %r12, %r11, %r9, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r12;
 ; CHECK-NEXT:    ret;
   %r = add <4 x i8> <i8 1, i8 2, i8 3, i8 4>, %a
@@ -176,29 +176,29 @@ define <4 x i8> @test_add_imm_1(<4 x i8> %a) #0 {
 ; CHECK-LABEL: test_add_imm_1(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<9>;
-; CHECK-NEXT:    .reg .b32 %r<13>;
+; CHECK-NEXT:    .reg .b32 %r<14>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_add_imm_1_param_0];
-; CHECK-NEXT:    bfe.u32 %r2, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r2, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs1, %r2;
-; CHECK-NEXT:    add.s16 %rs2, %rs1, 4;
+; CHECK-NEXT:    add.s16 %rs2, %rs1, 1;
 ; CHECK-NEXT:    cvt.u32.u16 %r3, %rs2;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs3, %r4;
-; CHECK-NEXT:    add.s16 %rs4, %rs3, 3;
+; CHECK-NEXT:    add.s16 %rs4, %rs3, 2;
 ; CHECK-NEXT:    cvt.u32.u16 %r5, %rs4;
-; CHECK-NEXT:    prmt.b32 %r6, %r5, %r3, 16435;
-; CHECK-NEXT:    bfe.u32 %r7, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r6, %r5, %r3, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs5, %r7;
-; CHECK-NEXT:    add.s16 %rs6, %rs5, 2;
+; CHECK-NEXT:    add.s16 %rs6, %rs5, 3;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-NEXT:    bfe.u32 %r9, %r1, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs7, %r9;
-; CHECK-NEXT:    add.s16 %rs8, %rs7, 1;
-; CHECK-NEXT:    cvt.u32.u16 %r10, %rs8;
-; CHECK-NEXT:    prmt.b32 %r11, %r10, %r8, 13120;
-; CHECK-NEXT:    prmt.b32 %r12, %r11, %r6, 21520;
+; CHECK-NEXT:    bfi.b32 %r9, %r8, %r6, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs7, %r10;
+; CHECK-NEXT:    add.s16 %rs8, %rs7, 4;
+; CHECK-NEXT:    cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT:    bfi.b32 %r12, %r11, %r9, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r12;
 ; CHECK-NEXT:    ret;
   %r = add <4 x i8> %a, <i8 1, i8 2, i8 3, i8 4>
@@ -209,38 +209,38 @@ define <4 x i8> @test_sub(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-LABEL: test_sub(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<13>;
-; CHECK-NEXT:    .reg .b32 %r<18>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_sub_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_sub_param_0];
-; CHECK-NEXT:    bfe.u32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r2, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; CHECK-NEXT:    sub.s16 %rs3, %rs2, %rs1;
 ; CHECK-NEXT:    cvt.u32.u16 %r5, %rs3;
-; CHECK-NEXT:    bfe.u32 %r6, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r2, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs4, %r6;
-; CHECK-NEXT:    bfe.u32 %r7, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; CHECK-NEXT:    sub.s16 %rs6, %rs5, %rs4;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-NEXT:    prmt.b32 %r9, %r8, %r5, 16435;
-; CHECK-NEXT:    bfe.u32 %r10, %r2, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r9, %r8, %r5, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r2, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs7, %r10;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; CHECK-NEXT:    sub.s16 %rs9, %rs8, %rs7;
 ; CHECK-NEXT:    cvt.u32.u16 %r12, %rs9;
-; CHECK-NEXT:    bfe.u32 %r13, %r2, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs10, %r13;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs11, %r14;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r9, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r2, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs10, %r14;
+; CHECK-NEXT:    bfe.u32 %r15, %r1, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs11, %r15;
 ; CHECK-NEXT:    sub.s16 %rs12, %rs11, %rs10;
-; CHECK-NEXT:    cvt.u32.u16 %r15, %rs12;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r12, 13120;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r9, 21520;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs12;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r13, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r17;
 ; CHECK-NEXT:    ret;
   %r = sub <4 x i8> %a, %b
@@ -251,38 +251,38 @@ define <4 x i8> @test_smax(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-LABEL: test_smax(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<26>;
+; CHECK-NEXT:    .reg .b32 %r<27>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_smax_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_smax_param_0];
-; CHECK-NEXT:    bfe.s32 %r3, %r2, 0, 8;
-; CHECK-NEXT:    bfe.s32 %r4, %r1, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 24, 8;
 ; CHECK-NEXT:    setp.gt.s32 %p1, %r4, %r3;
-; CHECK-NEXT:    bfe.s32 %r5, %r2, 8, 8;
-; CHECK-NEXT:    bfe.s32 %r6, %r1, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 16, 8;
 ; CHECK-NEXT:    setp.gt.s32 %p2, %r6, %r5;
-; CHECK-NEXT:    bfe.s32 %r7, %r2, 16, 8;
-; CHECK-NEXT:    bfe.s32 %r8, %r1, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r7, %r2, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r8, %r1, 8, 8;
 ; CHECK-NEXT:    setp.gt.s32 %p3, %r8, %r7;
-; CHECK-NEXT:    bfe.s32 %r9, %r2, 24, 8;
-; CHECK-NEXT:    bfe.s32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r9, %r2, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r10, %r1, 0, 8;
 ; CHECK-NEXT:    setp.gt.s32 %p4, %r10, %r9;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r12, %r1, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r13, %r1, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r15, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r13, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r15, %r2, 0, 8;
 ; CHECK-NEXT:    selp.b32 %r16, %r14, %r15, %p4;
-; CHECK-NEXT:    bfe.u32 %r17, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r17, %r2, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r18, %r13, %r17, %p3;
-; CHECK-NEXT:    prmt.b32 %r19, %r18, %r16, 16435;
-; CHECK-NEXT:    bfe.u32 %r20, %r2, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r19, %r18, %r16, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r20, %r2, 16, 8;
 ; CHECK-NEXT:    selp.b32 %r21, %r12, %r20, %p2;
-; CHECK-NEXT:    bfe.u32 %r22, %r2, 0, 8;
-; CHECK-NEXT:    selp.b32 %r23, %r11, %r22, %p1;
-; CHECK-NEXT:    prmt.b32 %r24, %r23, %r21, 13120;
-; CHECK-NEXT:    prmt.b32 %r25, %r24, %r19, 21520;
+; CHECK-NEXT:    bfi.b32 %r22, %r21, %r19, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r23, %r2, 24, 8;
+; CHECK-NEXT:    selp.b32 %r24, %r11, %r23, %p1;
+; CHECK-NEXT:    bfi.b32 %r25, %r24, %r22, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r25;
 ; CHECK-NEXT:    ret;
   %cmp = icmp sgt <4 x i8> %a, %b
@@ -294,30 +294,30 @@ define <4 x i8> @test_umax(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-LABEL: test_umax(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<18>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_umax_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_umax_param_0];
-; CHECK-NEXT:    bfe.u32 %r3, %r2, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 24, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p1, %r4, %r3;
-; CHECK-NEXT:    bfe.u32 %r5, %r2, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r6, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r1, 16, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p2, %r6, %r5;
-; CHECK-NEXT:    bfe.u32 %r7, %r2, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r8, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r8, %r1, 8, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p3, %r8, %r7;
-; CHECK-NEXT:    bfe.u32 %r9, %r2, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r9, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r1, 0, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p4, %r10, %r9;
 ; CHECK-NEXT:    selp.b32 %r11, %r10, %r9, %p4;
 ; CHECK-NEXT:    selp.b32 %r12, %r8, %r7, %p3;
-; CHECK-NEXT:    prmt.b32 %r13, %r12, %r11, 16435;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r11, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r14, %r6, %r5, %p2;
-; CHECK-NEXT:    selp.b32 %r15, %r4, %r3, %p1;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r14, 13120;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r13, 21520;
+; CHECK-NEXT:    bfi.b32 %r15, %r14, %r13, 16, 8;
+; CHECK-NEXT:    selp.b32 %r16, %r4, %r3, %p1;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r15, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r17;
 ; CHECK-NEXT:    ret;
   %cmp = icmp ugt <4 x i8> %a, %b
@@ -329,38 +329,38 @@ define <4 x i8> @test_smin(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-LABEL: test_smin(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<26>;
+; CHECK-NEXT:    .reg .b32 %r<27>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_smin_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_smin_param_0];
-; CHECK-NEXT:    bfe.s32 %r3, %r2, 0, 8;
-; CHECK-NEXT:    bfe.s32 %r4, %r1, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 24, 8;
 ; CHECK-NEXT:    setp.le.s32 %p1, %r4, %r3;
-; CHECK-NEXT:    bfe.s32 %r5, %r2, 8, 8;
-; CHECK-NEXT:    bfe.s32 %r6, %r1, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 16, 8;
 ; CHECK-NEXT:    setp.le.s32 %p2, %r6, %r5;
-; CHECK-NEXT:    bfe.s32 %r7, %r2, 16, 8;
-; CHECK-NEXT:    bfe.s32 %r8, %r1, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r7, %r2, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r8, %r1, 8, 8;
 ; CHECK-NEXT:    setp.le.s32 %p3, %r8, %r7;
-; CHECK-NEXT:    bfe.s32 %r9, %r2, 24, 8;
-; CHECK-NEXT:    bfe.s32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r9, %r2, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r10, %r1, 0, 8;
 ; CHECK-NEXT:    setp.le.s32 %p4, %r10, %r9;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r12, %r1, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r13, %r1, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r15, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r13, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r15, %r2, 0, 8;
 ; CHECK-NEXT:    selp.b32 %r16, %r14, %r15, %p4;
-; CHECK-NEXT:    bfe.u32 %r17, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r17, %r2, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r18, %r13, %r17, %p3;
-; CHECK-NEXT:    prmt.b32 %r19, %r18, %r16, 16435;
-; CHECK-NEXT:    bfe.u32 %r20, %r2, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r19, %r18, %r16, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r20, %r2, 16, 8;
 ; CHECK-NEXT:    selp.b32 %r21, %r12, %r20, %p2;
-; CHECK-NEXT:    bfe.u32 %r22, %r2, 0, 8;
-; CHECK-NEXT:    selp.b32 %r23, %r11, %r22, %p1;
-; CHECK-NEXT:    prmt.b32 %r24, %r23, %r21, 13120;
-; CHECK-NEXT:    prmt.b32 %r25, %r24, %r19, 21520;
+; CHECK-NEXT:    bfi.b32 %r22, %r21, %r19, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r23, %r2, 24, 8;
+; CHECK-NEXT:    selp.b32 %r24, %r11, %r23, %p1;
+; CHECK-NEXT:    bfi.b32 %r25, %r24, %r22, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r25;
 ; CHECK-NEXT:    ret;
   %cmp = icmp sle <4 x i8> %a, %b
@@ -372,30 +372,30 @@ define <4 x i8> @test_umin(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-LABEL: test_umin(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<18>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_umin_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_umin_param_0];
-; CHECK-NEXT:    bfe.u32 %r3, %r2, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 24, 8;
 ; CHECK-NEXT:    setp.ls.u32 %p1, %r4, %r3;
-; CHECK-NEXT:    bfe.u32 %r5, %r2, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r6, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r1, 16, 8;
 ; CHECK-NEXT:    setp.ls.u32 %p2, %r6, %r5;
-; CHECK-NEXT:    bfe.u32 %r7, %r2, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r8, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r8, %r1, 8, 8;
 ; CHECK-NEXT:    setp.ls.u32 %p3, %r8, %r7;
-; CHECK-NEXT:    bfe.u32 %r9, %r2, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r9, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r1, 0, 8;
 ; CHECK-NEXT:    setp.ls.u32 %p4, %r10, %r9;
 ; CHECK-NEXT:    selp.b32 %r11, %r10, %r9, %p4;
 ; CHECK-NEXT:    selp.b32 %r12, %r8, %r7, %p3;
-; CHECK-NEXT:    prmt.b32 %r13, %r12, %r11, 16435;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r11, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r14, %r6, %r5, %p2;
-; CHECK-NEXT:    selp.b32 %r15, %r4, %r3, %p1;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r14, 13120;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r13, 21520;
+; CHECK-NEXT:    bfi.b32 %r15, %r14, %r13, 16, 8;
+; CHECK-NEXT:    selp.b32 %r16, %r4, %r3, %p1;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r15, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r17;
 ; CHECK-NEXT:    ret;
   %cmp = icmp ule <4 x i8> %a, %b
@@ -407,35 +407,35 @@ define <4 x i8> @test_eq(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c) #0 {
 ; CHECK-LABEL: test_eq(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<23>;
+; CHECK-NEXT:    .reg .b32 %r<24>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r3, [test_eq_param_2];
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_eq_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_eq_param_0];
-; CHECK-NEXT:    bfe.u32 %r4, %r2, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r5, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r1, 24, 8;
 ; CHECK-NEXT:    setp.eq.u32 %p1, %r5, %r4;
-; CHECK-NEXT:    bfe.u32 %r6, %r2, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r7, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r1, 16, 8;
 ; CHECK-NEXT:    setp.eq.u32 %p2, %r7, %r6;
-; CHECK-NEXT:    bfe.u32 %r8, %r2, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r9, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r8, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r9, %r1, 8, 8;
 ; CHECK-NEXT:    setp.eq.u32 %p3, %r9, %r8;
-; CHECK-NEXT:    bfe.u32 %r10, %r2, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r1, 0, 8;
 ; CHECK-NEXT:    setp.eq.u32 %p4, %r11, %r10;
-; CHECK-NEXT:    bfe.u32 %r12, %r3, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r3, 0, 8;
 ; CHECK-NEXT:    selp.b32 %r13, %r11, %r12, %p4;
-; CHECK-NEXT:    bfe.u32 %r14, %r3, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r3, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r15, %r9, %r14, %p3;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r13, 16435;
-; CHECK-NEXT:    bfe.u32 %r17, %r3, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r16, %r15, %r13, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r17, %r3, 16, 8;
 ; CHECK-NEXT:    selp.b32 %r18, %r7, %r17, %p2;
-; CHECK-NEXT:    bfe.u32 %r19, %r3, 0, 8;
-; CHECK-NEXT:    selp.b32 %r20, %r5, %r19, %p1;
-; CHECK-NEXT:    prmt.b32 %r21, %r20, %r18, 13120;
-; CHECK-NEXT:    prmt.b32 %r22, %r21, %r16, 21520;
+; CHECK-NEXT:    bfi.b32 %r19, %r18, %r16, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r20, %r3, 24, 8;
+; CHECK-NEXT:    selp.b32 %r21, %r5, %r20, %p1;
+; CHECK-NEXT:    bfi.b32 %r22, %r21, %r19, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r22;
 ; CHECK-NEXT:    ret;
   %cmp = icmp eq <4 x i8> %a, %b
@@ -447,35 +447,35 @@ define <4 x i8> @test_ne(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c) #0 {
 ; CHECK-LABEL: test_ne(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<23>;
+; CHECK-NEXT:    .reg .b32 %r<24>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r3, [test_ne_param_2];
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_ne_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_ne_param_0];
-; CHECK-NEXT:    bfe.u32 %r4, %r2, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r5, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r1, 24, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p1, %r5, %r4;
-; CHECK-NEXT:    bfe.u32 %r6, %r2, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r7, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r1, 16, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p2, %r7, %r6;
-; CHECK-NEXT:    bfe.u32 %r8, %r2, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r9, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r8, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r9, %r1, 8, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p3, %r9, %r8;
-; CHECK-NEXT:    bfe.u32 %r10, %r2, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r1, 0, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p4, %r11, %r10;
-; CHECK-NEXT:    bfe.u32 %r12, %r3, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r3, 0, 8;
 ; CHECK-NEXT:    selp.b32 %r13, %r11, %r12, %p4;
-; CHECK-NEXT:    bfe.u32 %r14, %r3, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r3, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r15, %r9, %r14, %p3;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r13, 16435;
-; CHECK-NEXT:    bfe.u32 %r17, %r3, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r16, %r15, %r13, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r17, %r3, 16, 8;
 ; CHECK-NEXT:    selp.b32 %r18, %r7, %r17, %p2;
-; CHECK-NEXT:    bfe.u32 %r19, %r3, 0, 8;
-; CHECK-NEXT:    selp.b32 %r20, %r5, %r19, %p1;
-; CHECK-NEXT:    prmt.b32 %r21, %r20, %r18, 13120;
-; CHECK-NEXT:    prmt.b32 %r22, %r21, %r16, 21520;
+; CHECK-NEXT:    bfi.b32 %r19, %r18, %r16, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r20, %r3, 24, 8;
+; CHECK-NEXT:    selp.b32 %r21, %r5, %r20, %p1;
+; CHECK-NEXT:    bfi.b32 %r22, %r21, %r19, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r22;
 ; CHECK-NEXT:    ret;
   %cmp = icmp ne <4 x i8> %a, %b
@@ -487,38 +487,38 @@ define <4 x i8> @test_mul(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-LABEL: test_mul(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<13>;
-; CHECK-NEXT:    .reg .b32 %r<18>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_mul_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_mul_param_0];
-; CHECK-NEXT:    bfe.u32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r2, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs1, %r3;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; CHECK-NEXT:    mul.lo.s16 %rs3, %rs2, %rs1;
 ; CHECK-NEXT:    cvt.u32.u16 %r5, %rs3;
-; CHECK-NEXT:    bfe.u32 %r6, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r2, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs4, %r6;
-; CHECK-NEXT:    bfe.u32 %r7, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; CHECK-NEXT:    mul.lo.s16 %rs6, %rs5, %rs4;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-NEXT:    prmt.b32 %r9, %r8, %r5, 16435;
-; CHECK-NEXT:    bfe.u32 %r10, %r2, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r9, %r8, %r5, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r2, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs7, %r10;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; CHECK-NEXT:    mul.lo.s16 %rs9, %rs8, %rs7;
 ; CHECK-NEXT:    cvt.u32.u16 %r12, %rs9;
-; CHECK-NEXT:    bfe.u32 %r13, %r2, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs10, %r13;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 0, 8;
-; CHECK-NEXT:    cvt.u16.u32 %rs11, %r14;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r9, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r2, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs10, %r14;
+; CHECK-NEXT:    bfe.u32 %r15, %r1, 24, 8;
+; CHECK-NEXT:    cvt.u16.u32 %rs11, %r15;
 ; CHECK-NEXT:    mul.lo.s16 %rs12, %rs11, %rs10;
-; CHECK-NEXT:    cvt.u32.u16 %r15, %rs12;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r12, 13120;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r9, 21520;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs12;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r13, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r17;
 ; CHECK-NEXT:    ret;
   %r = mul <4 x i8> %a, %b
@@ -548,13 +548,12 @@ define <4 x i8> @test_or_computed(i8 %a) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u8 %rs1, [test_or_computed_param_0];
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 16435;
-; CHECK-NEXT:    cvt.u32.u16 %r3, %rs1;
-; CHECK-NEXT:    prmt.b32 %r4, %r3, 0, 13120;
-; CHECK-NEXT:    prmt.b32 %r5, %r4, %r2, 21520;
-; CHECK-NEXT:    bfi.b32 %r6, 5, %r5, 8, 8;
-; CHECK-NEXT:    or.b32 %r8, %r6, %r5;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    bfi.b32 %r2, 0, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r3, 0, %r2, 16, 8;
+; CHECK-NEXT:    bfi.b32 %r4, 0, %r3, 24, 8;
+; CHECK-NEXT:    bfi.b32 %r6, 5, %r4, 8, 8;
+; CHECK-NEXT:    or.b32 %r8, %r6, %r4;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r8;
 ; CHECK-NEXT:    ret;
   %ins.0 = insertelement <4 x i8> zeroinitializer, i8 %a, i32 0
@@ -614,13 +613,12 @@ define <4 x i8> @test_xor_computed(i8 %a) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u8 %rs1, [test_xor_computed_param_0];
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 16435;
-; CHECK-NEXT:    cvt.u32.u16 %r3, %rs1;
-; CHECK-NEXT:    prmt.b32 %r4, %r3, 0, 13120;
-; CHECK-NEXT:    prmt.b32 %r5, %r4, %r2, 21520;
-; CHECK-NEXT:    bfi.b32 %r6, 5, %r5, 8, 8;
-; CHECK-NEXT:    xor.b32 %r8, %r6, %r5;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    bfi.b32 %r2, 0, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r3, 0, %r2, 16, 8;
+; CHECK-NEXT:    bfi.b32 %r4, 0, %r3, 24, 8;
+; CHECK-NEXT:    bfi.b32 %r6, 5, %r4, 8, 8;
+; CHECK-NEXT:    xor.b32 %r8, %r6, %r4;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r8;
 ; CHECK-NEXT:    ret;
   %ins.0 = insertelement <4 x i8> zeroinitializer, i8 %a, i32 0
@@ -680,13 +678,12 @@ define <4 x i8> @test_and_computed(i8 %a) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u8 %rs1, [test_and_computed_param_0];
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    prmt.b32 %r2, %r1, 0, 16435;
-; CHECK-NEXT:    cvt.u32.u16 %r3, %rs1;
-; CHECK-NEXT:    prmt.b32 %r4, %r3, 0, 13120;
-; CHECK-NEXT:    prmt.b32 %r5, %r4, %r2, 21520;
-; CHECK-NEXT:    bfi.b32 %r6, 5, %r5, 8, 8;
-; CHECK-NEXT:    and.b32 %r8, %r6, %r5;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    bfi.b32 %r2, 0, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r3, 0, %r2, 16, 8;
+; CHECK-NEXT:    bfi.b32 %r4, 0, %r3, 24, 8;
+; CHECK-NEXT:    bfi.b32 %r6, 5, %r4, 8, 8;
+; CHECK-NEXT:    and.b32 %r8, %r6, %r4;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r8;
 ; CHECK-NEXT:    ret;
   %ins.0 = insertelement <4 x i8> zeroinitializer, i8 %a, i32 0
@@ -929,40 +926,40 @@ define <4 x i8> @test_select_cc(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8>
 ; CHECK-LABEL: test_select_cc(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<28>;
+; CHECK-NEXT:    .reg .b32 %r<29>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u32 %r4, [test_select_cc_param_3];
 ; CHECK-NEXT:    ld.param.u32 %r3, [test_select_cc_param_2];
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_select_cc_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_select_cc_param_0];
-; CHECK-NEXT:    bfe.u32 %r5, %r4, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r6, %r3, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r4, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r3, 24, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p1, %r6, %r5;
-; CHECK-NEXT:    bfe.u32 %r7, %r4, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r8, %r3, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r4, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r8, %r3, 16, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p2, %r8, %r7;
-; CHECK-NEXT:    bfe.u32 %r9, %r4, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r10, %r3, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r9, %r4, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r3, 8, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p3, %r10, %r9;
-; CHECK-NEXT:    bfe.u32 %r11, %r4, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r12, %r3, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r11, %r4, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r3, 0, 8;
 ; CHECK-NEXT:    setp.ne.u32 %p4, %r12, %r11;
-; CHECK-NEXT:    bfe.u32 %r13, %r2, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r13, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r1, 0, 8;
 ; CHECK-NEXT:    selp.b32 %r15, %r14, %r13, %p4;
-; CHECK-NEXT:    bfe.u32 %r16, %r2, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r17, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r16, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r17, %r1, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r18, %r17, %r16, %p3;
-; CHECK-NEXT:    prmt.b32 %r19, %r18, %r15, 16435;
-; CHECK-NEXT:    bfe.u32 %r20, %r2, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r21, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r19, %r18, %r15, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r20, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r21, %r1, 16, 8;
 ; CHECK-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
-; CHECK-NEXT:    bfe.u32 %r23, %r2, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r24, %r1, 0, 8;
-; CHECK-NEXT:    selp.b32 %r25, %r24, %r23, %p1;
-; CHECK-NEXT:    prmt.b32 %r26, %r25, %r22, 13120;
-; CHECK-NEXT:    prmt.b32 %r27, %r26, %r19, 21520;
+; CHECK-NEXT:    bfi.b32 %r23, %r22, %r19, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r24, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r25, %r1, 24, 8;
+; CHECK-NEXT:    selp.b32 %r26, %r25, %r24, %p1;
+; CHECK-NEXT:    bfi.b32 %r27, %r26, %r23, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r27;
 ; CHECK-NEXT:    ret;
   %cc = icmp ne <4 x i8> %c, %d
@@ -1009,32 +1006,32 @@ define <4 x i8> @test_select_cc_i8_i32(<4 x i8> %a, <4 x i8> %b,
 ; CHECK-LABEL: test_select_cc_i8_i32(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<26>;
+; CHECK-NEXT:    .reg .b32 %r<27>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.u32 {%r7, %r8, %r9, %r10}, [test_select_cc_i8_i32_param_3];
 ; CHECK-NEXT:    ld.param.v4.u32 {%r3, %r4, %r5, %r6}, [test_select_cc_i8_i32_param_2];
 ; CHECK-NEXT:    ld.param.u32 %r2, [test_select_cc_i8_i32_param_1];
 ; CHECK-NEXT:    ld.param.u32 %r1, [test_select_cc_i8_i32_param_0];
-; CHECK-NEXT:    setp.ne.s32 %p1, %r3, %r7;
-; CHECK-NEXT:    setp.ne.s32 %p2, %r4, %r8;
-; CHECK-NEXT:    setp.ne.s32 %p3, %r5, %r9;
-; CHECK-NEXT:    setp.ne.s32 %p4, %r6, %r10;
-; CHECK-NEXT:    bfe.u32 %r11, %r2, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r12, %r1, 24, 8;
+; CHECK-NEXT:    setp.ne.s32 %p1, %r6, %r10;
+; CHECK-NEXT:    setp.ne.s32 %p2, %r5, %r9;
+; CHECK-NEXT:    setp.ne.s32 %p3, %r4, %r8;
+; CHECK-NEXT:    setp.ne.s32 %p4, %r3, %r7;
+; CHECK-NEXT:    bfe.u32 %r11, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r1, 0, 8;
 ; CHECK-NEXT:    selp.b32 %r13, %r12, %r11, %p4;
-; CHECK-NEXT:    bfe.u32 %r14, %r2, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r15, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r15, %r1, 8, 8;
 ; CHECK-NEXT:    selp.b32 %r16, %r15, %r14, %p3;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r13, 16435;
-; CHECK-NEXT:    bfe.u32 %r18, %r2, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r19, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r13, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r18, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r19, %r1, 16, 8;
 ; CHECK-NEXT:    selp.b32 %r20, %r19, %r18, %p2;
-; CHECK-NEXT:    bfe.u32 %r21, %r2, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r22, %r1, 0, 8;
-; CHECK-NEXT:    selp.b32 %r23, %r22, %r21, %p1;
-; CHECK-NEXT:    prmt.b32 %r24, %r23, %r20, 13120;
-; CHECK-NEXT:    prmt.b32 %r25, %r24, %r17, 21520;
+; CHECK-NEXT:    bfi.b32 %r21, %r20, %r17, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r22, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r23, %r1, 24, 8;
+; CHECK-NEXT:    selp.b32 %r24, %r23, %r22, %p1;
+; CHECK-NEXT:    bfi.b32 %r25, %r24, %r21, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r25;
 ; CHECK-NEXT:    ret;
                                           <4 x i32> %c, <4 x i32> %d) #0 {
@@ -1047,13 +1044,13 @@ define <4 x i8> @test_select_cc_i8_i32(<4 x i8> %a, <4 x i8> %b,
 define <4 x i8> @test_trunc_2xi32(<4 x i32> %a) #0 {
 ; CHECK-LABEL: test_trunc_2xi32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.u32 {%r1, %r2, %r3, %r4}, [test_trunc_2xi32_param_0];
-; CHECK-NEXT:    prmt.b32 %r5, %r3, %r4, 16435;
-; CHECK-NEXT:    prmt.b32 %r6, %r1, %r2, 13120;
-; CHECK-NEXT:    prmt.b32 %r7, %r6, %r5, 21520;
+; CHECK-NEXT:    bfi.b32 %r5, %r2, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r6, %r3, %r5, 16, 8;
+; CHECK-NEXT:    bfi.b32 %r7, %r4, %r6, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r7;
 ; CHECK-NEXT:    ret;
   %r = trunc <4 x i32> %a to <4 x i8>
@@ -1063,19 +1060,19 @@ define <4 x i8> @test_trunc_2xi32(<4 x i32> %a) #0 {
 define <4 x i8> @test_trunc_2xi64(<4 x i64> %a) #0 {
 ; CHECK-LABEL: test_trunc_2xi64(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd3, %rd4}, [test_trunc_2xi64_param_0+16];
 ; CHECK-NEXT:    ld.param.v2.u64 {%rd1, %rd2}, [test_trunc_2xi64_param_0];
-; CHECK-NEXT:    cvt.u32.u64 %r1, %rd4;
-; CHECK-NEXT:    cvt.u32.u64 %r2, %rd3;
-; CHECK-NEXT:    prmt.b32 %r3, %r2, %r1, 16435;
-; CHECK-NEXT:    cvt.u32.u64 %r4, %rd2;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd1;
-; CHECK-NEXT:    prmt.b32 %r6, %r5, %r4, 13120;
-; CHECK-NEXT:    prmt.b32 %r7, %r6, %r3, 21520;
+; CHECK-NEXT:    cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT:    cvt.u32.u64 %r2, %rd2;
+; CHECK-NEXT:    bfi.b32 %r3, %r2, %r1, 8, 8;
+; CHECK-NEXT:    cvt.u32.u64 %r4, %rd3;
+; CHECK-NEXT:    bfi.b32 %r5, %r4, %r3, 16, 8;
+; CHECK-NEXT:    cvt.u32.u64 %r6, %rd4;
+; CHECK-NEXT:    bfi.b32 %r7, %r6, %r5, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r7;
 ; CHECK-NEXT:    ret;
   %r = trunc <4 x i64> %a to <4 x i8>
@@ -1187,16 +1184,15 @@ define <2 x half> @test_bitcast_4xi8_to_2xhalf(i8 %a) #0 {
 ; CHECK-LABEL: test_bitcast_4xi8_to_2xhalf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.u8 %rs1, [test_bitcast_4xi8_to_2xhalf_param_0];
-; CHECK-NEXT:    mov.b32 %r1, 6;
-; CHECK-NEXT:    prmt.b32 %r2, %r1, 7, 16435;
-; CHECK-NEXT:    cvt.u32.u16 %r3, %rs1;
-; CHECK-NEXT:    prmt.b32 %r4, %r3, 5, 13120;
-; CHECK-NEXT:    prmt.b32 %r5, %r4, %r2, 21520;
-; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r5;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    bfi.b32 %r2, 5, %r1, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r3, 6, %r2, 16, 8;
+; CHECK-NEXT:    bfi.b32 %r4, 7, %r3, 24, 8;
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r4;
 ; CHECK-NEXT:    ret;
   %ins.0 = insertelement <4 x i8> undef, i8 %a, i32 0
   %ins.1 = insertelement <4 x i8> %ins.0, i8 5, i32 1
@@ -1259,27 +1255,27 @@ define <4 x i8> @test_fptosi_4xhalf_to_4xi8(<4 x half> %a) #0 {
 ; CHECK-LABEL: test_fptosi_4xhalf_to_4xi8(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<13>;
-; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-NEXT:    .reg .b32 %r<15>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.u32 {%r3, %r4}, [test_fptosi_4xhalf_to_4xi8_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
 ; CHECK-NEXT:    cvt.rzi.s16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rzi.s16.f16 %rs4, %rs1;
 ; CHECK-NEXT:    mov.b32 %r5, {%rs4, %rs3};
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r5;
-; CHECK-NEXT:    cvt.u32.u16 %r6, %rs6;
-; CHECK-NEXT:    cvt.u32.u16 %r7, %rs5;
-; CHECK-NEXT:    prmt.b32 %r8, %r7, %r6, 16435;
-; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
+; CHECK-NEXT:    cvt.u32.u16 %r6, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r7, %rs6;
+; CHECK-NEXT:    bfi.b32 %r8, %r7, %r6, 8, 8;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; CHECK-NEXT:    cvt.rzi.s16.f16 %rs9, %rs8;
 ; CHECK-NEXT:    cvt.rzi.s16.f16 %rs10, %rs7;
 ; CHECK-NEXT:    mov.b32 %r9, {%rs10, %rs9};
 ; CHECK-NEXT:    mov.b32 {%rs11, %rs12}, %r9;
-; CHECK-NEXT:    cvt.u32.u16 %r10, %rs12;
-; CHECK-NEXT:    cvt.u32.u16 %r11, %rs11;
-; CHECK-NEXT:    prmt.b32 %r12, %r11, %r10, 13120;
-; CHECK-NEXT:    prmt.b32 %r13, %r12, %r8, 21520;
+; CHECK-NEXT:    cvt.u32.u16 %r10, %rs11;
+; CHECK-NEXT:    bfi.b32 %r11, %r10, %r8, 16, 8;
+; CHECK-NEXT:    cvt.u32.u16 %r12, %rs12;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r11, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r13;
 ; CHECK-NEXT:    ret;
   %r = fptosi <4 x half> %a to <4 x i8>
@@ -1290,27 +1286,27 @@ define <4 x i8> @test_fptoui_4xhalf_to_4xi8(<4 x half> %a) #0 {
 ; CHECK-LABEL: test_fptoui_4xhalf_to_4xi8(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<13>;
-; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-NEXT:    .reg .b32 %r<15>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.u32 {%r3, %r4}, [test_fptoui_4xhalf_to_4xi8_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
 ; CHECK-NEXT:    cvt.rzi.u16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rzi.u16.f16 %rs4, %rs1;
 ; CHECK-NEXT:    mov.b32 %r5, {%rs4, %rs3};
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r5;
-; CHECK-NEXT:    cvt.u32.u16 %r6, %rs6;
-; CHECK-NEXT:    cvt.u32.u16 %r7, %rs5;
-; CHECK-NEXT:    prmt.b32 %r8, %r7, %r6, 16435;
-; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
+; CHECK-NEXT:    cvt.u32.u16 %r6, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r7, %rs6;
+; CHECK-NEXT:    bfi.b32 %r8, %r7, %r6, 8, 8;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; CHECK-NEXT:    cvt.rzi.u16.f16 %rs9, %rs8;
 ; CHECK-NEXT:    cvt.rzi.u16.f16 %rs10, %rs7;
 ; CHECK-NEXT:    mov.b32 %r9, {%rs10, %rs9};
 ; CHECK-NEXT:    mov.b32 {%rs11, %rs12}, %r9;
-; CHECK-NEXT:    cvt.u32.u16 %r10, %rs12;
-; CHECK-NEXT:    cvt.u32.u16 %r11, %rs11;
-; CHECK-NEXT:    prmt.b32 %r12, %r11, %r10, 13120;
-; CHECK-NEXT:    prmt.b32 %r13, %r12, %r8, 21520;
+; CHECK-NEXT:    cvt.u32.u16 %r10, %rs11;
+; CHECK-NEXT:    bfi.b32 %r11, %r10, %r8, 16, 8;
+; CHECK-NEXT:    cvt.u32.u16 %r12, %rs12;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r11, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r13;
 ; CHECK-NEXT:    ret;
   %r = fptoui <4 x half> %a to <4 x i8>
@@ -1330,33 +1326,33 @@ define void @test_srem_v4i8(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    ld.param.u64 %rd1, [test_srem_v4i8_param_0];
 ; CHECK-NEXT:    ld.u32 %r1, [%rd1];
 ; CHECK-NEXT:    ld.u32 %r2, [%rd2];
-; CHECK-NEXT:    bfe.s32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r3, %r2, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs1, %r3;
-; CHECK-NEXT:    bfe.s32 %r4, %r1, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs2, %r4;
 ; CHECK-NEXT:    rem.s16 %rs3, %rs2, %rs1;
 ; CHECK-NEXT:    cvt.u32.u16 %r5, %rs3;
-; CHECK-NEXT:    bfe.s32 %r6, %r2, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r6, %r2, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs4, %r6;
-; CHECK-NEXT:    bfe.s32 %r7, %r1, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r7, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs5, %r7;
 ; CHECK-NEXT:    rem.s16 %rs6, %rs5, %rs4;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-NEXT:    prmt.b32 %r9, %r8, %r5, 16435;
-; CHECK-NEXT:    bfe.s32 %r10, %r2, 8, 8;
+; CHECK-NEXT:    bfi.b32 %r9, %r8, %r5, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r10, %r2, 16, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs7, %r10;
-; CHECK-NEXT:    bfe.s32 %r11, %r1, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r11, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs8, %r11;
 ; CHECK-NEXT:    rem.s16 %rs9, %rs8, %rs7;
 ; CHECK-NEXT:    cvt.u32.u16 %r12, %rs9;
-; CHECK-NEXT:    bfe.s32 %r13, %r2, 0, 8;
-; CHECK-NEXT:    cvt.s8.s32 %rs10, %r13;
-; CHECK-NEXT:    bfe.s32 %r14, %r1, 0, 8;
-; CHECK-NEXT:    cvt.s8.s32 %rs11, %r14;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r9, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r14, %r2, 24, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs10, %r14;
+; CHECK-NEXT:    bfe.s32 %r15, %r1, 24, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs11, %r15;
 ; CHECK-NEXT:    rem.s16 %rs12, %rs11, %rs10;
-; CHECK-NEXT:    cvt.u32.u16 %r15, %rs12;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r12, 13120;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r9, 21520;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs12;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r13, 24, 8;
 ; CHECK-NEXT:    st.u32 [%rd3], %r17;
 ; CHECK-NEXT:    ret;
 entry:
@@ -1377,7 +1373,7 @@ define void @test_srem_v3i8(ptr %a, ptr %b, ptr %c) {
 ; CHECK-LABEL: test_srem_v3i8(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<20>;
-; CHECK-NEXT:    .reg .b32 %r<17>;
+; CHECK-NEXT:    .reg .b32 %r<16>;
 ; CHECK-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
@@ -1396,25 +1392,25 @@ define void @test_srem_v3i8(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    or.b16 %rs9, %rs8, %rs6;
 ; CHECK-NEXT:    cvt.u32.u16 %r3, %rs9;
 ; CHECK-NEXT:    ld.s8 %rs10, [%rd2+2];
-; CHECK-NEXT:    bfe.s32 %r5, %r3, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r5, %r3, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs11, %r5;
-; CHECK-NEXT:    bfe.s32 %r6, %r1, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs12, %r6;
 ; CHECK-NEXT:    rem.s16 %rs13, %rs12, %rs11;
 ; CHECK-NEXT:    cvt.u32.u16 %r7, %rs13;
-; CHECK-NEXT:    bfe.s32 %r8, %r3, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r8, %r3, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs14, %r8;
-; CHECK-NEXT:    bfe.s32 %r9, %r1, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r9, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs15, %r9;
 ; CHECK-NEXT:    rem.s16 %rs16, %rs15, %rs14;
 ; CHECK-NEXT:    cvt.u32.u16 %r10, %rs16;
-; CHECK-NEXT:    prmt.b32 %r11, %r10, %r7, 13120;
+; CHECK-NEXT:    bfi.b32 %r11, %r10, %r7, 8, 8;
 ; CHECK-NEXT:    // implicit-def: %r13
-; CHECK-NEXT:    // implicit-def: %r14
-; CHECK-NEXT:    prmt.b32 %r12, %r13, %r14, 16435;
-; CHECK-NEXT:    prmt.b32 %r15, %r11, %r12, 21520;
+; CHECK-NEXT:    bfi.b32 %r12, %r13, %r11, 16, 8;
+; CHECK-NEXT:    // implicit-def: %r15
+; CHECK-NEXT:    bfi.b32 %r14, %r15, %r12, 24, 8;
 ; CHECK-NEXT:    rem.s16 %rs17, %rs5, %rs10;
-; CHECK-NEXT:    { .reg .b16 tmp; mov.b32 {%rs18, tmp}, %r15; }
+; CHECK-NEXT:    cvt.u16.u32 %rs18, %r14;
 ; CHECK-NEXT:    st.u8 [%rd3], %rs18;
 ; CHECK-NEXT:    shr.u16 %rs19, %rs18, 8;
 ; CHECK-NEXT:    st.u8 [%rd3+1], %rs19;
@@ -1441,25 +1437,25 @@ define void @test_sext_v4i1_to_v4i8(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    ld.param.u64 %rd1, [test_sext_v4i1_to_v4i8_param_0];
 ; CHECK-NEXT:    ld.u32 %r1, [%rd1];
 ; CHECK-NEXT:    ld.u32 %r2, [%rd2];
-; CHECK-NEXT:    bfe.u32 %r3, %r2, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 24, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p1, %r4, %r3;
-; CHECK-NEXT:    bfe.u32 %r5, %r2, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r6, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r6, %r1, 16, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p2, %r6, %r5;
-; CHECK-NEXT:    bfe.u32 %r7, %r2, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r8, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r7, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r8, %r1, 8, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p3, %r8, %r7;
-; CHECK-NEXT:    bfe.u32 %r9, %r2, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r9, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r10, %r1, 0, 8;
 ; CHECK-NEXT:    setp.hi.u32 %p4, %r10, %r9;
 ; CHECK-NEXT:    selp.s32 %r11, -1, 0, %p4;
 ; CHECK-NEXT:    selp.s32 %r12, -1, 0, %p3;
-; CHECK-NEXT:    prmt.b32 %r13, %r12, %r11, 16435;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r11, 8, 8;
 ; CHECK-NEXT:    selp.s32 %r14, -1, 0, %p2;
-; CHECK-NEXT:    selp.s32 %r15, -1, 0, %p1;
-; CHECK-NEXT:    prmt.b32 %r16, %r15, %r14, 13120;
-; CHECK-NEXT:    prmt.b32 %r17, %r16, %r13, 21520;
+; CHECK-NEXT:    bfi.b32 %r15, %r14, %r13, 16, 8;
+; CHECK-NEXT:    selp.s32 %r16, -1, 0, %p1;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r15, 24, 8;
 ; CHECK-NEXT:    st.u32 [%rd3], %r17;
 ; CHECK-NEXT:    ret;
 entry:
diff --git a/llvm/test/CodeGen/NVPTX/lower-args.ll b/llvm/test/CodeGen/NVPTX/lower-args.ll
index 029f1944d596b32ff762af23a20034d5788aba37..d1bec032ec3a988a2ee9d4bcbc01c5450694cdcc 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args.ll
@@ -9,13 +9,13 @@ target triple = "nvptx64-nvidia-cuda"
 
 %class.outer = type <{ %class.inner, i32, [4 x i8] }>
 %class.inner = type { ptr, ptr }
+%class.padded = type { i8, i32 }
 
 ; Check that nvptx-lower-args preserves arg alignment
 ; COMMON-LABEL: load_alignment
 define void @load_alignment(ptr nocapture readonly byval(%class.outer) align 8 %arg) {
 entry:
-; IR: load %class.outer, ptr addrspace(101)
-; IR-SAME: align 8
+; IR: call void @llvm.memcpy.p0.p101.i64(ptr align 8
 ; PTX: ld.param.u64
 ; PTX-NOT: ld.param.u8
   %arg.idx.val = load ptr, ptr %arg, align 8
@@ -33,6 +33,36 @@ entry:
   ret void
 }
 
+; Check that nvptx-lower-args copies padding as the struct may have been a union
+; COMMON-LABEL: load_padding
+define void @load_padding(ptr nocapture readonly byval(%class.padded) %arg) {
+; PTX:       {
+; PTX-NEXT:    .local .align 8 .b8 __local_depot1[8];
+; PTX-NEXT:    .reg .b64 %SP;
+; PTX-NEXT:    .reg .b64 %SPL;
+; PTX-NEXT:    .reg .b64 %rd<5>;
+; PTX-EMPTY:
+; PTX-NEXT:  // %bb.0:
+; PTX-NEXT:    mov.u64 %SPL, __local_depot1;
+; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
+; PTX-NEXT:    ld.param.u64 %rd1, [load_padding_param_0];
+; PTX-NEXT:    st.u64 [%SP+0], %rd1;
+; PTX-NEXT:    add.u64 %rd2, %SP, 0;
+; PTX-NEXT:    { // callseq 1, 0
+; PTX-NEXT:    .param .b64 param0;
+; PTX-NEXT:    st.param.b64 [param0+0], %rd2;
+; PTX-NEXT:    .param .b64 retval0;
+; PTX-NEXT:    call.uni (retval0),
+; PTX-NEXT:    escape,
+; PTX-NEXT:    (
+; PTX-NEXT:    param0
+; PTX-NEXT:    );
+; PTX-NEXT:    ld.param.b64 %rd3, [retval0+0];
+; PTX-NEXT:    } // callseq 1
+; PTX-NEXT:    ret;
+  %tmp = call ptr @escape(ptr nonnull align 16 %arg)
+  ret void
+}
 
 ; COMMON-LABEL: ptr_generic
 define void @ptr_generic(ptr %out, ptr %in) {
diff --git a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
index a414a6c41cd5b251fcb657bf649e1bab0b9dd5be..a7dbc4c1620a5f68ccd509137abdf69a8c5e75b6 100644
--- a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
@@ -88,8 +88,7 @@ define dso_local void @read_only_gep_asc0(ptr nocapture noundef writeonly %out,
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
 ; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
@@ -115,8 +114,7 @@ define dso_local void @escape_ptr(ptr nocapture noundef readnone %out, ptr nound
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
 ; COMMON-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S3]])
@@ -134,8 +132,7 @@ define dso_local void @escape_ptr_gep(ptr nocapture noundef readnone %out, ptr n
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
 ; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
@@ -155,8 +152,7 @@ define dso_local void @escape_ptr_store(ptr nocapture noundef writeonly %out, pt
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
 ; COMMON-NEXT:    store ptr [[S3]], ptr [[OUT2]], align 8
@@ -174,8 +170,7 @@ define dso_local void @escape_ptr_gep_store(ptr nocapture noundef writeonly %out
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
 ; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
@@ -195,8 +190,7 @@ define dso_local void @escape_ptrtoint(ptr nocapture noundef writeonly %out, ptr
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
 ; COMMON-NEXT:    [[I:%.*]] = ptrtoint ptr [[S3]] to i64
@@ -232,8 +226,7 @@ define dso_local void @memcpy_to_param(ptr nocapture noundef readonly %in, ptr n
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
 ; COMMON-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[S3]], ptr [[IN2]], i64 16, i1 true)
@@ -251,8 +244,7 @@ define dso_local void @copy_on_store(ptr nocapture noundef readonly %in, ptr noc
 ; COMMON-NEXT:  [[BB:.*:]]
 ; COMMON-NEXT:    [[S3:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
-; COMMON-NEXT:    [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S3]], ptr addrspace(101) align 4 [[S4]], i64 8, i1 false)
 ; COMMON-NEXT:    [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
 ; COMMON-NEXT:    [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
 ; COMMON-NEXT:    [[I:%.*]] = load i32, ptr [[IN2]], align 4
@@ -273,12 +265,10 @@ define void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i32) %input2,
 ; SM_60-NEXT:    [[OUT8:%.*]] = addrspacecast ptr addrspace(1) [[OUT7]] to ptr
 ; SM_60-NEXT:    [[INPUT24:%.*]] = alloca i32, align 4
 ; SM_60-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; SM_60-NEXT:    [[INPUT26:%.*]] = load i32, ptr addrspace(101) [[INPUT25]], align 4
-; SM_60-NEXT:    store i32 [[INPUT26]], ptr [[INPUT24]], align 4
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT24]], ptr addrspace(101) align 4 [[INPUT25]], i64 4, i1 false)
 ; SM_60-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
 ; SM_60-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; SM_60-NEXT:    [[INPUT13:%.*]] = load i32, ptr addrspace(101) [[INPUT12]], align 4
-; SM_60-NEXT:    store i32 [[INPUT13]], ptr [[INPUT11]], align 4
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; SM_60-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT24]]
 ; SM_60-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
 ; SM_60-NEXT:    store i32 [[VALLOADED]], ptr [[OUT8]], align 4
@@ -313,12 +303,10 @@ define void @test_select_write(ptr byval(i32) align 4 %input1, ptr byval(i32) %i
 ; COMMON-NEXT:    [[OUT8:%.*]] = addrspacecast ptr addrspace(1) [[OUT7]] to ptr
 ; COMMON-NEXT:    [[INPUT24:%.*]] = alloca i32, align 4
 ; COMMON-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; COMMON-NEXT:    [[INPUT26:%.*]] = load i32, ptr addrspace(101) [[INPUT25]], align 4
-; COMMON-NEXT:    store i32 [[INPUT26]], ptr [[INPUT24]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT24]], ptr addrspace(101) align 4 [[INPUT25]], i64 4, i1 false)
 ; COMMON-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
 ; COMMON-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; COMMON-NEXT:    [[INPUT13:%.*]] = load i32, ptr addrspace(101) [[INPUT12]], align 4
-; COMMON-NEXT:    store i32 [[INPUT13]], ptr [[INPUT11]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; COMMON-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT24]]
 ; COMMON-NEXT:    store i32 1, ptr [[PTRNEW]], align 4
 ; COMMON-NEXT:    ret void
@@ -337,12 +325,10 @@ define void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval(%struct.S)
 ; SM_60-NEXT:    [[INOUT8:%.*]] = addrspacecast ptr addrspace(1) [[INOUT7]] to ptr
 ; SM_60-NEXT:    [[INPUT24:%.*]] = alloca [[STRUCT_S]], align 8
 ; SM_60-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; SM_60-NEXT:    [[INPUT26:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT25]], align 8
-; SM_60-NEXT:    store [[STRUCT_S]] [[INPUT26]], ptr [[INPUT24]], align 4
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT24]], ptr addrspace(101) align 8 [[INPUT25]], i64 8, i1 false)
 ; SM_60-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
 ; SM_60-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; SM_60-NEXT:    [[INPUT13:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT12]], align 4
-; SM_60-NEXT:    store [[STRUCT_S]] [[INPUT13]], ptr [[INPUT11]], align 4
+; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; SM_60-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; SM_60:       [[FIRST]]:
 ; SM_60-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT11]], i32 0, i32 0
@@ -402,12 +388,10 @@ define void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr byval(%str
 ; COMMON-NEXT:  [[BB:.*:]]
 ; COMMON-NEXT:    [[INPUT24:%.*]] = alloca [[STRUCT_S]], align 8
 ; COMMON-NEXT:    [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
-; COMMON-NEXT:    [[INPUT26:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT25]], align 8
-; COMMON-NEXT:    store [[STRUCT_S]] [[INPUT26]], ptr [[INPUT24]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT24]], ptr addrspace(101) align 8 [[INPUT25]], i64 8, i1 false)
 ; COMMON-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
 ; COMMON-NEXT:    [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
-; COMMON-NEXT:    [[INPUT13:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT12]], align 4
-; COMMON-NEXT:    store [[STRUCT_S]] [[INPUT13]], ptr [[INPUT11]], align 4
+; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; COMMON-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; COMMON:       [[FIRST]]:
 ; COMMON-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT11]], i32 0, i32 0
diff --git a/llvm/test/CodeGen/NVPTX/sext-setcc.ll b/llvm/test/CodeGen/NVPTX/sext-setcc.ll
index 8b7e5235443f05e6422d92e64ad0ab88df21258b..f471d47077cf0d56bde3dbab1ef2c608a2b80851 100644
--- a/llvm/test/CodeGen/NVPTX/sext-setcc.ll
+++ b/llvm/test/CodeGen/NVPTX/sext-setcc.ll
@@ -33,35 +33,35 @@ define <4 x i8> @sext_setcc_v4i1_to_v4i8(ptr %p) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
 ; CHECK-NEXT:    .reg .b16 %rs<9>;
-; CHECK-NEXT:    .reg .b32 %r<13>;
+; CHECK-NEXT:    .reg .b32 %r<14>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.u64 %rd1, [sext_setcc_v4i1_to_v4i8_param_0];
 ; CHECK-NEXT:    ld.u32 %r1, [%rd1];
-; CHECK-NEXT:    bfe.u32 %r2, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r2, %r1, 24, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs1, %r2;
 ; CHECK-NEXT:    and.b16 %rs2, %rs1, 255;
 ; CHECK-NEXT:    setp.eq.s16 %p1, %rs2, 0;
-; CHECK-NEXT:    bfe.u32 %r3, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs3, %r3;
 ; CHECK-NEXT:    and.b16 %rs4, %rs3, 255;
 ; CHECK-NEXT:    setp.eq.s16 %p2, %rs4, 0;
-; CHECK-NEXT:    bfe.u32 %r4, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs5, %r4;
 ; CHECK-NEXT:    and.b16 %rs6, %rs5, 255;
 ; CHECK-NEXT:    setp.eq.s16 %p3, %rs6, 0;
-; CHECK-NEXT:    bfe.u32 %r5, %r1, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.u16.u32 %rs7, %r5;
 ; CHECK-NEXT:    and.b16 %rs8, %rs7, 255;
 ; CHECK-NEXT:    setp.eq.s16 %p4, %rs8, 0;
 ; CHECK-NEXT:    selp.s32 %r6, -1, 0, %p4;
 ; CHECK-NEXT:    selp.s32 %r7, -1, 0, %p3;
-; CHECK-NEXT:    prmt.b32 %r8, %r7, %r6, 16435;
+; CHECK-NEXT:    bfi.b32 %r8, %r7, %r6, 8, 8;
 ; CHECK-NEXT:    selp.s32 %r9, -1, 0, %p2;
-; CHECK-NEXT:    selp.s32 %r10, -1, 0, %p1;
-; CHECK-NEXT:    prmt.b32 %r11, %r10, %r9, 13120;
-; CHECK-NEXT:    prmt.b32 %r12, %r11, %r8, 21520;
+; CHECK-NEXT:    bfi.b32 %r10, %r9, %r8, 16, 8;
+; CHECK-NEXT:    selp.s32 %r11, -1, 0, %p1;
+; CHECK-NEXT:    bfi.b32 %r12, %r11, %r10, 24, 8;
 ; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r12;
 ; CHECK-NEXT:    ret;
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/aix-p8vector-liveins.ll b/llvm/test/CodeGen/PowerPC/aix-p8vector-liveins.ll
index 07ec88840dc3d838ac8a8def1d3c32cad44d0adf..5269617a00bde98960296ddb5635eac4c49dc0ae 100644
--- a/llvm/test/CodeGen/PowerPC/aix-p8vector-liveins.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-p8vector-liveins.ll
@@ -18,15 +18,15 @@ entry:
 }
 
 ; POWR8-LABEL: name:            vssr
-; POWR8:       - { id: 0, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 1, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 2, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 3, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 4, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 5, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 6, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 7, class: vssrc, preferred-register: '' }
-; POWR8:       - { id: 8, class: vssrc, preferred-register: '' }
+; POWR8:       - { id: 0, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 1, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 2, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 3, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 4, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 5, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 6, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 7, class: vssrc, preferred-register: '', flags: [  ] }
+; POWR8:       - { id: 8, class: vssrc, preferred-register: '', flags: [  ] }
 ; POWR8:       %4:vssrc = COPY $f5
 ; POWR8:       %3:vssrc = COPY $f4
 ; POWR8:       %2:vssrc = COPY $f3
@@ -40,15 +40,15 @@ entry:
 
 ; NOP8V-LABEL: name:            vssr
 ; NOP8V:       registers:
-; NOP8V:       - { id: 0, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 1, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 2, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 3, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 4, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 5, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 6, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 7, class: f4rc, preferred-register: '' }
-; NOP8V:       - { id: 8, class: f4rc, preferred-register: '' }
+; NOP8V:       - { id: 0, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 1, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 2, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 3, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 4, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 5, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 6, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 7, class: f4rc, preferred-register: '', flags: [  ] }
+; NOP8V:       - { id: 8, class: f4rc, preferred-register: '', flags: [  ] }
 ; NOP8V:        %4:f4rc = COPY $f5
 ; NOP8V:        %3:f4rc = COPY $f4
 ; NOP8V:        %2:f4rc = COPY $f3
@@ -71,15 +71,15 @@ entry:
 
 ; VSX-LABEL:   vsfr
 ; VSX:         registers:
-; VSX:          - { id: 0, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 1, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 2, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 3, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 4, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 5, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 6, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 7, class: vsfrc, preferred-register: '' }
-; VSX:          - { id: 8, class: vsfrc, preferred-register: '' }
+; VSX:          - { id: 0, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 1, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 2, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 3, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 4, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 5, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 6, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 7, class: vsfrc, preferred-register: '', flags: [  ] }
+; VSX:          - { id: 8, class: vsfrc, preferred-register: '', flags: [  ] }
 ; VSX:          %4:vsfrc = COPY $f5
 ; VSX:          %3:vsfrc = COPY $f4
 ; VSX:          %2:vsfrc = COPY $f3
@@ -93,15 +93,15 @@ entry:
 
 ; NOVSX-LABEL:  vsfr
 ; NOVSX:        registers:
-; NOVSX:        - { id: 0, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 1, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 2, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 3, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 4, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 5, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 6, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 7, class: f8rc, preferred-register: '' }
-; NOVSX:        - { id: 8, class: f8rc, preferred-register: '' }
+; NOVSX:        - { id: 0, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 1, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 2, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 3, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 4, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 5, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 6, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 7, class: f8rc, preferred-register: '', flags: [  ] }
+; NOVSX:        - { id: 8, class: f8rc, preferred-register: '', flags: [  ] }
 ; NOVSX:        %4:f8rc = COPY $f5
 ; NOVSX:        %3:f8rc = COPY $f4
 ; NOVSX:        %2:f8rc = COPY $f3
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
index 3df4aca40ec9428105cd570f62221d2201c1b167..6c0e322a252c72d4a24bdfdf32d6db86073a7ad4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
@@ -69,12 +69,12 @@ define i32 @va1(ptr %fmt, ...) {
 ; RV64-NEXT:    sd a2, 32(sp)
 ; RV64-NEXT:    sd a3, 40(sp)
 ; RV64-NEXT:    sd a4, 48(sp)
-; RV64-NEXT:    sd a5, 56(sp)
 ; RV64-NEXT:    addi a0, sp, 8
 ; RV64-NEXT:    addi a1, sp, 24
 ; RV64-NEXT:    sd a1, 8(sp)
 ; RV64-NEXT:    lw a0, 4(a0)
 ; RV64-NEXT:    lwu a1, 8(sp)
+; RV64-NEXT:    sd a5, 56(sp)
 ; RV64-NEXT:    sd a6, 64(sp)
 ; RV64-NEXT:    sd a7, 72(sp)
 ; RV64-NEXT:    slli a0, a0, 32
@@ -129,12 +129,12 @@ define i32 @va1(ptr %fmt, ...) {
 ; RV64-WITHFP-NEXT:    sd a2, 16(s0)
 ; RV64-WITHFP-NEXT:    sd a3, 24(s0)
 ; RV64-WITHFP-NEXT:    sd a4, 32(s0)
-; RV64-WITHFP-NEXT:    sd a5, 40(s0)
 ; RV64-WITHFP-NEXT:    addi a0, s0, -24
 ; RV64-WITHFP-NEXT:    addi a1, s0, 8
 ; RV64-WITHFP-NEXT:    sd a1, -24(s0)
 ; RV64-WITHFP-NEXT:    lw a0, 4(a0)
 ; RV64-WITHFP-NEXT:    lwu a1, -24(s0)
+; RV64-WITHFP-NEXT:    sd a5, 40(s0)
 ; RV64-WITHFP-NEXT:    sd a6, 48(s0)
 ; RV64-WITHFP-NEXT:    sd a7, 56(s0)
 ; RV64-WITHFP-NEXT:    slli a0, a0, 32
@@ -844,11 +844,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-LABEL: va3:
 ; ILP32:       # %bb.0:
 ; ILP32-NEXT:    addi sp, sp, -32
-; ILP32-NEXT:    sw a3, 12(sp)
-; ILP32-NEXT:    sw a4, 16(sp)
 ; ILP32-NEXT:    addi a0, sp, 12
 ; ILP32-NEXT:    sw a0, 4(sp)
 ; ILP32-NEXT:    lw a0, 4(sp)
+; ILP32-NEXT:    sw a3, 12(sp)
+; ILP32-NEXT:    sw a4, 16(sp)
 ; ILP32-NEXT:    sw a5, 20(sp)
 ; ILP32-NEXT:    sw a6, 24(sp)
 ; ILP32-NEXT:    sw a7, 28(sp)
@@ -868,11 +868,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32-LABEL: va3:
 ; RV32D-ILP32:       # %bb.0:
 ; RV32D-ILP32-NEXT:    addi sp, sp, -48
-; RV32D-ILP32-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-NEXT:    addi a0, sp, 28
 ; RV32D-ILP32-NEXT:    sw a0, 20(sp)
 ; RV32D-ILP32-NEXT:    lw a0, 20(sp)
+; RV32D-ILP32-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-NEXT:    sw a5, 36(sp)
 ; RV32D-ILP32-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32-NEXT:    sw a7, 44(sp)
@@ -894,11 +894,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32F-LABEL: va3:
 ; RV32D-ILP32F:       # %bb.0:
 ; RV32D-ILP32F-NEXT:    addi sp, sp, -48
-; RV32D-ILP32F-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32F-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32F-NEXT:    addi a0, sp, 28
 ; RV32D-ILP32F-NEXT:    sw a0, 20(sp)
 ; RV32D-ILP32F-NEXT:    lw a0, 20(sp)
+; RV32D-ILP32F-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32F-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32F-NEXT:    sw a5, 36(sp)
 ; RV32D-ILP32F-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32F-NEXT:    sw a7, 44(sp)
@@ -920,11 +920,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32D-LABEL: va3:
 ; RV32D-ILP32D:       # %bb.0:
 ; RV32D-ILP32D-NEXT:    addi sp, sp, -48
-; RV32D-ILP32D-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32D-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32D-NEXT:    addi a0, sp, 28
 ; RV32D-ILP32D-NEXT:    sw a0, 20(sp)
 ; RV32D-ILP32D-NEXT:    lw a0, 20(sp)
+; RV32D-ILP32D-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32D-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32D-NEXT:    sw a5, 36(sp)
 ; RV32D-ILP32D-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32D-NEXT:    sw a7, 44(sp)
@@ -946,12 +946,12 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV64-LABEL: va3:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi sp, sp, -64
-; RV64-NEXT:    sd a2, 16(sp)
-; RV64-NEXT:    sd a3, 24(sp)
-; RV64-NEXT:    sd a4, 32(sp)
 ; RV64-NEXT:    addi a0, sp, 16
 ; RV64-NEXT:    sd a0, 8(sp)
 ; RV64-NEXT:    ld a0, 8(sp)
+; RV64-NEXT:    sd a2, 16(sp)
+; RV64-NEXT:    sd a3, 24(sp)
+; RV64-NEXT:    sd a4, 32(sp)
 ; RV64-NEXT:    sd a5, 40(sp)
 ; RV64-NEXT:    sd a6, 48(sp)
 ; RV64-NEXT:    sd a7, 56(sp)
@@ -970,11 +970,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV32-WITHFP-NEXT:    sw ra, 20(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    sw s0, 16(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    addi s0, sp, 24
-; RV32-WITHFP-NEXT:    sw a3, 4(s0)
-; RV32-WITHFP-NEXT:    sw a4, 8(s0)
 ; RV32-WITHFP-NEXT:    addi a0, s0, 4
 ; RV32-WITHFP-NEXT:    sw a0, -12(s0)
 ; RV32-WITHFP-NEXT:    lw a0, -12(s0)
+; RV32-WITHFP-NEXT:    sw a3, 4(s0)
+; RV32-WITHFP-NEXT:    sw a4, 8(s0)
 ; RV32-WITHFP-NEXT:    sw a5, 12(s0)
 ; RV32-WITHFP-NEXT:    sw a6, 16(s0)
 ; RV32-WITHFP-NEXT:    sw a7, 20(s0)
@@ -999,12 +999,12 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV64-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    addi s0, sp, 32
-; RV64-WITHFP-NEXT:    sd a2, 0(s0)
-; RV64-WITHFP-NEXT:    sd a3, 8(s0)
-; RV64-WITHFP-NEXT:    sd a4, 16(s0)
 ; RV64-WITHFP-NEXT:    mv a0, s0
 ; RV64-WITHFP-NEXT:    sd a0, -24(s0)
 ; RV64-WITHFP-NEXT:    ld a0, -24(s0)
+; RV64-WITHFP-NEXT:    sd a2, 0(s0)
+; RV64-WITHFP-NEXT:    sd a3, 8(s0)
+; RV64-WITHFP-NEXT:    sd a4, 16(s0)
 ; RV64-WITHFP-NEXT:    sd a5, 24(s0)
 ; RV64-WITHFP-NEXT:    sd a6, 32(s0)
 ; RV64-WITHFP-NEXT:    sd a7, 40(s0)
@@ -1622,9 +1622,6 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-NEXT:    lui a0, 24414
 ; RV64-NEXT:    add a0, sp, a0
 ; RV64-NEXT:    sd a4, 304(a0)
-; RV64-NEXT:    lui a0, 24414
-; RV64-NEXT:    add a0, sp, a0
-; RV64-NEXT:    sd a5, 312(a0)
 ; RV64-NEXT:    addi a0, sp, 8
 ; RV64-NEXT:    lui a1, 24414
 ; RV64-NEXT:    addiw a1, a1, 280
@@ -1634,6 +1631,9 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-NEXT:    lwu a1, 8(sp)
 ; RV64-NEXT:    lui a2, 24414
 ; RV64-NEXT:    add a2, sp, a2
+; RV64-NEXT:    sd a5, 312(a2)
+; RV64-NEXT:    lui a2, 24414
+; RV64-NEXT:    add a2, sp, a2
 ; RV64-NEXT:    sd a6, 320(a2)
 ; RV64-NEXT:    lui a2, 24414
 ; RV64-NEXT:    add a2, sp, a2
diff --git a/llvm/test/CodeGen/RISCV/abds-neg.ll b/llvm/test/CodeGen/RISCV/abds-neg.ll
index b6064198935a61dcb2ed189783ab6f686cae6d49..e7fd87bd78387653677c610d20c01c895a32af60 100644
--- a/llvm/test/CodeGen/RISCV/abds-neg.ll
+++ b/llvm/test/CodeGen/RISCV/abds-neg.ll
@@ -705,8 +705,8 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a4, a4, a3
 ; RV32I-NEXT:    neg a1, a1
 ; RV32I-NEXT:    sw a1, 0(a0)
-; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a2, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a5, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -824,8 +824,8 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a4, a4, a3
 ; RV32ZBB-NEXT:    neg a1, a1
 ; RV32ZBB-NEXT:    sw a1, 0(a0)
-; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a2, 4(a0)
+; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a5, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
@@ -952,8 +952,8 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a4, a4, a3
 ; RV32I-NEXT:    neg a1, a1
 ; RV32I-NEXT:    sw a1, 0(a0)
-; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a2, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a5, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1071,8 +1071,8 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a4, a4, a3
 ; RV32ZBB-NEXT:    neg a1, a1
 ; RV32ZBB-NEXT:    sw a1, 0(a0)
-; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a2, 4(a0)
+; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a5, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
@@ -1918,9 +1918,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a1, a1, t2
 ; RV32I-NEXT:    sub a2, a2, a3
 ; RV32I-NEXT:  .LBB22_11:
-; RV32I-NEXT:    sw a6, 8(a0)
-; RV32I-NEXT:    sw a1, 4(a0)
 ; RV32I-NEXT:    sw a2, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a6, 8(a0)
 ; RV32I-NEXT:    sw a5, 12(a0)
 ; RV32I-NEXT:    ret
 ;
@@ -2005,9 +2005,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a1, a1, t2
 ; RV32ZBB-NEXT:    sub a2, a2, a3
 ; RV32ZBB-NEXT:  .LBB22_11:
-; RV32ZBB-NEXT:    sw a6, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 4(a0)
 ; RV32ZBB-NEXT:    sw a2, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    sw a6, 8(a0)
 ; RV32ZBB-NEXT:    sw a5, 12(a0)
 ; RV32ZBB-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/abds.ll b/llvm/test/CodeGen/RISCV/abds.ll
index 91b044902a5201b13a5a51c14c8554bb043bfed4..e639d4b30d4c94160d6b863fcb11644c23c743de 100644
--- a/llvm/test/CodeGen/RISCV/abds.ll
+++ b/llvm/test/CodeGen/RISCV/abds.ll
@@ -599,9 +599,9 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a1, a4, t5
 ; RV32I-NEXT:    sub a4, a6, t6
 ; RV32I-NEXT:  .LBB11_13:
-; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 4(a0)
 ; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -695,9 +695,9 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a1, a4, t5
 ; RV32ZBB-NEXT:    sub a4, a6, t6
 ; RV32ZBB-NEXT:  .LBB11_13:
-; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 4(a0)
 ; RV32ZBB-NEXT:    sw a3, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
@@ -799,9 +799,9 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a1, a4, t5
 ; RV32I-NEXT:    sub a4, a6, t6
 ; RV32I-NEXT:  .LBB12_13:
-; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 4(a0)
 ; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -895,9 +895,9 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a1, a4, t5
 ; RV32ZBB-NEXT:    sub a4, a6, t6
 ; RV32ZBB-NEXT:  .LBB12_13:
-; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 4(a0)
 ; RV32ZBB-NEXT:    sw a3, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
@@ -1188,9 +1188,9 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a1, a4, t5
 ; RV32I-NEXT:    sub a4, a6, t6
 ; RV32I-NEXT:  .LBB17_13:
-; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 4(a0)
 ; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1284,9 +1284,9 @@ define i128 @abd_minmax_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a1, a4, t5
 ; RV32ZBB-NEXT:    sub a4, a6, t6
 ; RV32ZBB-NEXT:  .LBB17_13:
-; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 4(a0)
 ; RV32ZBB-NEXT:    sw a3, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
@@ -1579,9 +1579,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a1, a4, t5
 ; RV32I-NEXT:    sub a4, a6, t6
 ; RV32I-NEXT:  .LBB22_13:
-; RV32I-NEXT:    sw a4, 8(a0)
-; RV32I-NEXT:    sw a1, 4(a0)
 ; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
 ; RV32I-NEXT:    sw a2, 12(a0)
 ; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1675,9 +1675,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a1, a4, t5
 ; RV32ZBB-NEXT:    sub a4, a6, t6
 ; RV32ZBB-NEXT:  .LBB22_13:
-; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 4(a0)
 ; RV32ZBB-NEXT:    sw a3, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
@@ -2043,8 +2043,8 @@ define i64 @abd_subnsw_i64_undef(i64 %a, i64 %b) nounwind {
 define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-LABEL: abd_subnsw_i128:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lw a3, 0(a2)
-; RV32I-NEXT:    lw a4, 4(a2)
+; RV32I-NEXT:    lw a4, 0(a2)
+; RV32I-NEXT:    lw a3, 4(a2)
 ; RV32I-NEXT:    lw a6, 8(a2)
 ; RV32I-NEXT:    lw t0, 12(a2)
 ; RV32I-NEXT:    lw a2, 8(a1)
@@ -2053,24 +2053,24 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a7, 4(a1)
 ; RV32I-NEXT:    sltu a1, a2, a6
 ; RV32I-NEXT:    sub t1, t1, t0
-; RV32I-NEXT:    sltu t0, a5, a3
+; RV32I-NEXT:    sltu t0, a5, a4
 ; RV32I-NEXT:    sub a1, t1, a1
 ; RV32I-NEXT:    mv t1, t0
-; RV32I-NEXT:    beq a7, a4, .LBB31_2
+; RV32I-NEXT:    beq a7, a3, .LBB31_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    sltu t1, a7, a4
+; RV32I-NEXT:    sltu t1, a7, a3
 ; RV32I-NEXT:  .LBB31_2:
 ; RV32I-NEXT:    sub a2, a2, a6
 ; RV32I-NEXT:    sltu a6, a2, t1
 ; RV32I-NEXT:    sub a1, a1, a6
 ; RV32I-NEXT:    sub a2, a2, t1
-; RV32I-NEXT:    sub a4, a7, a4
-; RV32I-NEXT:    sub a4, a4, t0
-; RV32I-NEXT:    sub a3, a5, a3
+; RV32I-NEXT:    sub a3, a7, a3
+; RV32I-NEXT:    sub a3, a3, t0
+; RV32I-NEXT:    sub a4, a5, a4
 ; RV32I-NEXT:    bgez a1, .LBB31_4
 ; RV32I-NEXT:  # %bb.3:
-; RV32I-NEXT:    snez a5, a4
-; RV32I-NEXT:    snez a6, a3
+; RV32I-NEXT:    snez a5, a3
+; RV32I-NEXT:    snez a6, a4
 ; RV32I-NEXT:    or a5, a6, a5
 ; RV32I-NEXT:    neg a7, a2
 ; RV32I-NEXT:    sltu t0, a7, a5
@@ -2079,12 +2079,12 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    neg a1, a1
 ; RV32I-NEXT:    sub a1, a1, t0
 ; RV32I-NEXT:    sub a2, a7, a5
-; RV32I-NEXT:    neg a3, a3
-; RV32I-NEXT:    add a4, a4, a6
 ; RV32I-NEXT:    neg a4, a4
+; RV32I-NEXT:    add a3, a3, a6
+; RV32I-NEXT:    neg a3, a3
 ; RV32I-NEXT:  .LBB31_4:
-; RV32I-NEXT:    sw a4, 4(a0)
-; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a4, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a2, 8(a0)
 ; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
@@ -2106,8 +2106,8 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ;
 ; RV32ZBB-LABEL: abd_subnsw_i128:
 ; RV32ZBB:       # %bb.0:
-; RV32ZBB-NEXT:    lw a3, 0(a2)
-; RV32ZBB-NEXT:    lw a4, 4(a2)
+; RV32ZBB-NEXT:    lw a4, 0(a2)
+; RV32ZBB-NEXT:    lw a3, 4(a2)
 ; RV32ZBB-NEXT:    lw a6, 8(a2)
 ; RV32ZBB-NEXT:    lw t0, 12(a2)
 ; RV32ZBB-NEXT:    lw a2, 8(a1)
@@ -2116,24 +2116,24 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a7, 4(a1)
 ; RV32ZBB-NEXT:    sltu a1, a2, a6
 ; RV32ZBB-NEXT:    sub t1, t1, t0
-; RV32ZBB-NEXT:    sltu t0, a5, a3
+; RV32ZBB-NEXT:    sltu t0, a5, a4
 ; RV32ZBB-NEXT:    sub a1, t1, a1
 ; RV32ZBB-NEXT:    mv t1, t0
-; RV32ZBB-NEXT:    beq a7, a4, .LBB31_2
+; RV32ZBB-NEXT:    beq a7, a3, .LBB31_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    sltu t1, a7, a4
+; RV32ZBB-NEXT:    sltu t1, a7, a3
 ; RV32ZBB-NEXT:  .LBB31_2:
 ; RV32ZBB-NEXT:    sub a2, a2, a6
 ; RV32ZBB-NEXT:    sltu a6, a2, t1
 ; RV32ZBB-NEXT:    sub a1, a1, a6
 ; RV32ZBB-NEXT:    sub a2, a2, t1
-; RV32ZBB-NEXT:    sub a4, a7, a4
-; RV32ZBB-NEXT:    sub a4, a4, t0
-; RV32ZBB-NEXT:    sub a3, a5, a3
+; RV32ZBB-NEXT:    sub a3, a7, a3
+; RV32ZBB-NEXT:    sub a3, a3, t0
+; RV32ZBB-NEXT:    sub a4, a5, a4
 ; RV32ZBB-NEXT:    bgez a1, .LBB31_4
 ; RV32ZBB-NEXT:  # %bb.3:
-; RV32ZBB-NEXT:    snez a5, a4
-; RV32ZBB-NEXT:    snez a6, a3
+; RV32ZBB-NEXT:    snez a5, a3
+; RV32ZBB-NEXT:    snez a6, a4
 ; RV32ZBB-NEXT:    or a5, a6, a5
 ; RV32ZBB-NEXT:    neg a7, a2
 ; RV32ZBB-NEXT:    sltu t0, a7, a5
@@ -2142,12 +2142,12 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    neg a1, a1
 ; RV32ZBB-NEXT:    sub a1, a1, t0
 ; RV32ZBB-NEXT:    sub a2, a7, a5
-; RV32ZBB-NEXT:    neg a3, a3
-; RV32ZBB-NEXT:    add a4, a4, a6
 ; RV32ZBB-NEXT:    neg a4, a4
+; RV32ZBB-NEXT:    add a3, a3, a6
+; RV32ZBB-NEXT:    neg a3, a3
 ; RV32ZBB-NEXT:  .LBB31_4:
-; RV32ZBB-NEXT:    sw a4, 4(a0)
-; RV32ZBB-NEXT:    sw a3, 0(a0)
+; RV32ZBB-NEXT:    sw a4, 0(a0)
+; RV32ZBB-NEXT:    sw a3, 4(a0)
 ; RV32ZBB-NEXT:    sw a2, 8(a0)
 ; RV32ZBB-NEXT:    sw a1, 12(a0)
 ; RV32ZBB-NEXT:    ret
@@ -2174,8 +2174,8 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32I-LABEL: abd_subnsw_i128_undef:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lw a3, 0(a2)
-; RV32I-NEXT:    lw a4, 4(a2)
+; RV32I-NEXT:    lw a4, 0(a2)
+; RV32I-NEXT:    lw a3, 4(a2)
 ; RV32I-NEXT:    lw a6, 8(a2)
 ; RV32I-NEXT:    lw t0, 12(a2)
 ; RV32I-NEXT:    lw a2, 8(a1)
@@ -2184,24 +2184,24 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a7, 4(a1)
 ; RV32I-NEXT:    sltu a1, a2, a6
 ; RV32I-NEXT:    sub t1, t1, t0
-; RV32I-NEXT:    sltu t0, a5, a3
+; RV32I-NEXT:    sltu t0, a5, a4
 ; RV32I-NEXT:    sub a1, t1, a1
 ; RV32I-NEXT:    mv t1, t0
-; RV32I-NEXT:    beq a7, a4, .LBB32_2
+; RV32I-NEXT:    beq a7, a3, .LBB32_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    sltu t1, a7, a4
+; RV32I-NEXT:    sltu t1, a7, a3
 ; RV32I-NEXT:  .LBB32_2:
 ; RV32I-NEXT:    sub a2, a2, a6
 ; RV32I-NEXT:    sltu a6, a2, t1
 ; RV32I-NEXT:    sub a1, a1, a6
 ; RV32I-NEXT:    sub a2, a2, t1
-; RV32I-NEXT:    sub a4, a7, a4
-; RV32I-NEXT:    sub a4, a4, t0
-; RV32I-NEXT:    sub a3, a5, a3
+; RV32I-NEXT:    sub a3, a7, a3
+; RV32I-NEXT:    sub a3, a3, t0
+; RV32I-NEXT:    sub a4, a5, a4
 ; RV32I-NEXT:    bgez a1, .LBB32_4
 ; RV32I-NEXT:  # %bb.3:
-; RV32I-NEXT:    snez a5, a4
-; RV32I-NEXT:    snez a6, a3
+; RV32I-NEXT:    snez a5, a3
+; RV32I-NEXT:    snez a6, a4
 ; RV32I-NEXT:    or a5, a6, a5
 ; RV32I-NEXT:    neg a7, a2
 ; RV32I-NEXT:    sltu t0, a7, a5
@@ -2210,12 +2210,12 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    neg a1, a1
 ; RV32I-NEXT:    sub a1, a1, t0
 ; RV32I-NEXT:    sub a2, a7, a5
-; RV32I-NEXT:    neg a3, a3
-; RV32I-NEXT:    add a4, a4, a6
 ; RV32I-NEXT:    neg a4, a4
+; RV32I-NEXT:    add a3, a3, a6
+; RV32I-NEXT:    neg a3, a3
 ; RV32I-NEXT:  .LBB32_4:
-; RV32I-NEXT:    sw a4, 4(a0)
-; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a4, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a2, 8(a0)
 ; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
@@ -2237,8 +2237,8 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ;
 ; RV32ZBB-LABEL: abd_subnsw_i128_undef:
 ; RV32ZBB:       # %bb.0:
-; RV32ZBB-NEXT:    lw a3, 0(a2)
-; RV32ZBB-NEXT:    lw a4, 4(a2)
+; RV32ZBB-NEXT:    lw a4, 0(a2)
+; RV32ZBB-NEXT:    lw a3, 4(a2)
 ; RV32ZBB-NEXT:    lw a6, 8(a2)
 ; RV32ZBB-NEXT:    lw t0, 12(a2)
 ; RV32ZBB-NEXT:    lw a2, 8(a1)
@@ -2247,24 +2247,24 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    lw a7, 4(a1)
 ; RV32ZBB-NEXT:    sltu a1, a2, a6
 ; RV32ZBB-NEXT:    sub t1, t1, t0
-; RV32ZBB-NEXT:    sltu t0, a5, a3
+; RV32ZBB-NEXT:    sltu t0, a5, a4
 ; RV32ZBB-NEXT:    sub a1, t1, a1
 ; RV32ZBB-NEXT:    mv t1, t0
-; RV32ZBB-NEXT:    beq a7, a4, .LBB32_2
+; RV32ZBB-NEXT:    beq a7, a3, .LBB32_2
 ; RV32ZBB-NEXT:  # %bb.1:
-; RV32ZBB-NEXT:    sltu t1, a7, a4
+; RV32ZBB-NEXT:    sltu t1, a7, a3
 ; RV32ZBB-NEXT:  .LBB32_2:
 ; RV32ZBB-NEXT:    sub a2, a2, a6
 ; RV32ZBB-NEXT:    sltu a6, a2, t1
 ; RV32ZBB-NEXT:    sub a1, a1, a6
 ; RV32ZBB-NEXT:    sub a2, a2, t1
-; RV32ZBB-NEXT:    sub a4, a7, a4
-; RV32ZBB-NEXT:    sub a4, a4, t0
-; RV32ZBB-NEXT:    sub a3, a5, a3
+; RV32ZBB-NEXT:    sub a3, a7, a3
+; RV32ZBB-NEXT:    sub a3, a3, t0
+; RV32ZBB-NEXT:    sub a4, a5, a4
 ; RV32ZBB-NEXT:    bgez a1, .LBB32_4
 ; RV32ZBB-NEXT:  # %bb.3:
-; RV32ZBB-NEXT:    snez a5, a4
-; RV32ZBB-NEXT:    snez a6, a3
+; RV32ZBB-NEXT:    snez a5, a3
+; RV32ZBB-NEXT:    snez a6, a4
 ; RV32ZBB-NEXT:    or a5, a6, a5
 ; RV32ZBB-NEXT:    neg a7, a2
 ; RV32ZBB-NEXT:    sltu t0, a7, a5
@@ -2273,12 +2273,12 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    neg a1, a1
 ; RV32ZBB-NEXT:    sub a1, a1, t0
 ; RV32ZBB-NEXT:    sub a2, a7, a5
-; RV32ZBB-NEXT:    neg a3, a3
-; RV32ZBB-NEXT:    add a4, a4, a6
 ; RV32ZBB-NEXT:    neg a4, a4
+; RV32ZBB-NEXT:    add a3, a3, a6
+; RV32ZBB-NEXT:    neg a3, a3
 ; RV32ZBB-NEXT:  .LBB32_4:
-; RV32ZBB-NEXT:    sw a4, 4(a0)
-; RV32ZBB-NEXT:    sw a3, 0(a0)
+; RV32ZBB-NEXT:    sw a4, 0(a0)
+; RV32ZBB-NEXT:    sw a3, 4(a0)
 ; RV32ZBB-NEXT:    sw a2, 8(a0)
 ; RV32ZBB-NEXT:    sw a1, 12(a0)
 ; RV32ZBB-NEXT:    ret
@@ -2711,9 +2711,9 @@ define i128 @abd_select_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a1, a4, t5
 ; RV32ZBB-NEXT:    sub a4, a6, t6
 ; RV32ZBB-NEXT:  .LBB38_13:
-; RV32ZBB-NEXT:    sw a4, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 4(a0)
 ; RV32ZBB-NEXT:    sw a3, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    sw a4, 8(a0)
 ; RV32ZBB-NEXT:    sw a2, 12(a0)
 ; RV32ZBB-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32ZBB-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/abdu-neg.ll b/llvm/test/CodeGen/RISCV/abdu-neg.ll
index 54075f416943926b10e689b3b6a16ba26509443d..b39285c3d343f578c7c7ca3f6f7d91190236d825 100644
--- a/llvm/test/CodeGen/RISCV/abdu-neg.ll
+++ b/llvm/test/CodeGen/RISCV/abdu-neg.ll
@@ -1855,9 +1855,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sub a1, a1, t2
 ; RV32I-NEXT:    sub a2, a2, a3
 ; RV32I-NEXT:  .LBB22_11:
-; RV32I-NEXT:    sw a6, 8(a0)
-; RV32I-NEXT:    sw a1, 4(a0)
 ; RV32I-NEXT:    sw a2, 0(a0)
+; RV32I-NEXT:    sw a1, 4(a0)
+; RV32I-NEXT:    sw a6, 8(a0)
 ; RV32I-NEXT:    sw a5, 12(a0)
 ; RV32I-NEXT:    ret
 ;
@@ -1942,9 +1942,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
 ; RV32ZBB-NEXT:    sub a1, a1, t2
 ; RV32ZBB-NEXT:    sub a2, a2, a3
 ; RV32ZBB-NEXT:  .LBB22_11:
-; RV32ZBB-NEXT:    sw a6, 8(a0)
-; RV32ZBB-NEXT:    sw a1, 4(a0)
 ; RV32ZBB-NEXT:    sw a2, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    sw a6, 8(a0)
 ; RV32ZBB-NEXT:    sw a5, 12(a0)
 ; RV32ZBB-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/add-before-shl.ll b/llvm/test/CodeGen/RISCV/add-before-shl.ll
index 823918f1c42e7ac68b3f93b661382883785aee1b..db7498340d3951b774e7c7865140b576ebab3ca5 100644
--- a/llvm/test/CodeGen/RISCV/add-before-shl.ll
+++ b/llvm/test/CodeGen/RISCV/add-before-shl.ll
@@ -182,8 +182,8 @@ define i128 @add_wide_operand(i128 %a) nounwind {
 ; RV32I-NEXT:    lui a4, 128
 ; RV32I-NEXT:    add a1, a1, a4
 ; RV32I-NEXT:    sw a2, 0(a0)
-; RV32I-NEXT:    sw a3, 8(a0)
 ; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw a3, 8(a0)
 ; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    jalr zero, 0(ra)
 ;
@@ -217,8 +217,8 @@ define i128 @add_wide_operand(i128 %a) nounwind {
 ; RV32C-NEXT:    c.or a1, a3
 ; RV32C-NEXT:    c.slli a6, 3
 ; RV32C-NEXT:    sw a6, 0(a0)
-; RV32C-NEXT:    c.sw a1, 8(a0)
 ; RV32C-NEXT:    c.sw a4, 4(a0)
+; RV32C-NEXT:    c.sw a1, 8(a0)
 ; RV32C-NEXT:    c.sw a2, 12(a0)
 ; RV32C-NEXT:    c.jr ra
 ;
diff --git a/llvm/test/CodeGen/RISCV/alloca.ll b/llvm/test/CodeGen/RISCV/alloca.ll
index bcb0592c18f59f18f32737011bd3269c97051be8..975fc93c830af216ffe9c00f97b65625ff480b0b 100644
--- a/llvm/test/CodeGen/RISCV/alloca.ll
+++ b/llvm/test/CodeGen/RISCV/alloca.ll
@@ -76,13 +76,10 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-NEXT:    sub a0, sp, a0
 ; RV32I-NEXT:    mv sp, a0
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    li a1, 12
-; RV32I-NEXT:    sw a1, 12(sp)
-; RV32I-NEXT:    li a1, 11
-; RV32I-NEXT:    sw a1, 8(sp)
-; RV32I-NEXT:    li a1, 10
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    li t0, 9
+; RV32I-NEXT:    li t0, 12
+; RV32I-NEXT:    li t1, 11
+; RV32I-NEXT:    li t2, 10
+; RV32I-NEXT:    li t3, 9
 ; RV32I-NEXT:    li a1, 2
 ; RV32I-NEXT:    li a2, 3
 ; RV32I-NEXT:    li a3, 4
@@ -90,7 +87,10 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-NEXT:    li a5, 6
 ; RV32I-NEXT:    li a6, 7
 ; RV32I-NEXT:    li a7, 8
-; RV32I-NEXT:    sw t0, 0(sp)
+; RV32I-NEXT:    sw t3, 0(sp)
+; RV32I-NEXT:    sw t2, 4(sp)
+; RV32I-NEXT:    sw t1, 8(sp)
+; RV32I-NEXT:    sw t0, 12(sp)
 ; RV32I-NEXT:    call func
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    addi sp, s0, -16
diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
index 9908503adb9c300a9689edf240a112c79f9969a7..a87b49e61a8dbcd48cb134d198f8893f6411acc6 100644
--- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
@@ -4882,8 +4882,8 @@ define void @cmpxchg_i64_monotonic_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounw
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a4
@@ -4898,8 +4898,8 @@ define void @cmpxchg_i64_monotonic_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounw
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a4
@@ -4964,8 +4964,8 @@ define void @cmpxchg_i64_acquire_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 2
 ; RV32I-NEXT:    mv a2, a3
@@ -4981,8 +4981,8 @@ define void @cmpxchg_i64_acquire_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 2
 ; RV32IA-NEXT:    mv a2, a3
@@ -5057,8 +5057,8 @@ define void @cmpxchg_i64_acquire_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 2
 ; RV32I-NEXT:    li a5, 2
@@ -5074,8 +5074,8 @@ define void @cmpxchg_i64_acquire_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 2
 ; RV32IA-NEXT:    li a5, 2
@@ -5150,8 +5150,8 @@ define void @cmpxchg_i64_release_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 3
 ; RV32I-NEXT:    mv a2, a3
@@ -5167,8 +5167,8 @@ define void @cmpxchg_i64_release_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 3
 ; RV32IA-NEXT:    mv a2, a3
@@ -5243,8 +5243,8 @@ define void @cmpxchg_i64_release_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 3
 ; RV32I-NEXT:    li a5, 2
@@ -5260,8 +5260,8 @@ define void @cmpxchg_i64_release_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 3
 ; RV32IA-NEXT:    li a5, 2
@@ -5336,8 +5336,8 @@ define void @cmpxchg_i64_acq_rel_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 4
 ; RV32I-NEXT:    mv a2, a3
@@ -5353,8 +5353,8 @@ define void @cmpxchg_i64_acq_rel_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 4
 ; RV32IA-NEXT:    mv a2, a3
@@ -5429,8 +5429,8 @@ define void @cmpxchg_i64_acq_rel_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 4
 ; RV32I-NEXT:    li a5, 2
@@ -5446,8 +5446,8 @@ define void @cmpxchg_i64_acq_rel_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 4
 ; RV32IA-NEXT:    li a5, 2
@@ -5522,8 +5522,8 @@ define void @cmpxchg_i64_seq_cst_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 5
 ; RV32I-NEXT:    mv a2, a3
@@ -5539,8 +5539,8 @@ define void @cmpxchg_i64_seq_cst_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 5
 ; RV32IA-NEXT:    mv a2, a3
@@ -5615,8 +5615,8 @@ define void @cmpxchg_i64_seq_cst_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 5
 ; RV32I-NEXT:    li a5, 2
@@ -5632,8 +5632,8 @@ define void @cmpxchg_i64_seq_cst_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 5
 ; RV32IA-NEXT:    li a5, 2
@@ -5708,8 +5708,8 @@ define void @cmpxchg_i64_seq_cst_seq_cst(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a4, 5
 ; RV32I-NEXT:    li a5, 5
@@ -5725,8 +5725,8 @@ define void @cmpxchg_i64_seq_cst_seq_cst(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
-; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
+; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    mv a1, sp
 ; RV32IA-NEXT:    li a4, 5
 ; RV32IA-NEXT:    li a5, 5
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
index 6d2263f74062df8ccf2a3759352e1e642ff861a4..f26425c297ab077546eae679be096d240b202f30 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
@@ -2767,14 +2767,14 @@ define void @varargs(...) {
 ; RV32I-NEXT:    .cfi_def_cfa_offset 48
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    .cfi_offset ra, -36
-; RV32I-NEXT:    sw a7, 44(sp)
-; RV32I-NEXT:    sw a6, 40(sp)
-; RV32I-NEXT:    sw a5, 36(sp)
 ; RV32I-NEXT:    sw a4, 32(sp)
-; RV32I-NEXT:    sw a3, 28(sp)
-; RV32I-NEXT:    sw a2, 24(sp)
-; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a5, 36(sp)
+; RV32I-NEXT:    sw a6, 40(sp)
+; RV32I-NEXT:    sw a7, 44(sp)
 ; RV32I-NEXT:    sw a0, 16(sp)
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a2, 24(sp)
+; RV32I-NEXT:    sw a3, 28(sp)
 ; RV32I-NEXT:    call callee
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -2786,12 +2786,12 @@ define void @varargs(...) {
 ; RV32I-ILP32E-NEXT:    .cfi_def_cfa_offset 28
 ; RV32I-ILP32E-NEXT:    sw ra, 0(sp) # 4-byte Folded Spill
 ; RV32I-ILP32E-NEXT:    .cfi_offset ra, -28
-; RV32I-ILP32E-NEXT:    sw a5, 24(sp)
 ; RV32I-ILP32E-NEXT:    sw a4, 20(sp)
-; RV32I-ILP32E-NEXT:    sw a3, 16(sp)
-; RV32I-ILP32E-NEXT:    sw a2, 12(sp)
-; RV32I-ILP32E-NEXT:    sw a1, 8(sp)
+; RV32I-ILP32E-NEXT:    sw a5, 24(sp)
 ; RV32I-ILP32E-NEXT:    sw a0, 4(sp)
+; RV32I-ILP32E-NEXT:    sw a1, 8(sp)
+; RV32I-ILP32E-NEXT:    sw a2, 12(sp)
+; RV32I-ILP32E-NEXT:    sw a3, 16(sp)
 ; RV32I-ILP32E-NEXT:    call callee
 ; RV32I-ILP32E-NEXT:    lw ra, 0(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    addi sp, sp, 28
@@ -2807,14 +2807,14 @@ define void @varargs(...) {
 ; RV32I-WITH-FP-NEXT:    .cfi_offset s0, -40
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 16
 ; RV32I-WITH-FP-NEXT:    .cfi_def_cfa s0, 32
-; RV32I-WITH-FP-NEXT:    sw a7, 28(s0)
-; RV32I-WITH-FP-NEXT:    sw a6, 24(s0)
-; RV32I-WITH-FP-NEXT:    sw a5, 20(s0)
 ; RV32I-WITH-FP-NEXT:    sw a4, 16(s0)
-; RV32I-WITH-FP-NEXT:    sw a3, 12(s0)
-; RV32I-WITH-FP-NEXT:    sw a2, 8(s0)
-; RV32I-WITH-FP-NEXT:    sw a1, 4(s0)
+; RV32I-WITH-FP-NEXT:    sw a5, 20(s0)
+; RV32I-WITH-FP-NEXT:    sw a6, 24(s0)
+; RV32I-WITH-FP-NEXT:    sw a7, 28(s0)
 ; RV32I-WITH-FP-NEXT:    sw a0, 0(s0)
+; RV32I-WITH-FP-NEXT:    sw a1, 4(s0)
+; RV32I-WITH-FP-NEXT:    sw a2, 8(s0)
+; RV32I-WITH-FP-NEXT:    sw a3, 12(s0)
 ; RV32I-WITH-FP-NEXT:    call callee
 ; RV32I-WITH-FP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -2827,14 +2827,14 @@ define void @varargs(...) {
 ; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 48
 ; RV32IZCMP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IZCMP-NEXT:    .cfi_offset ra, -36
-; RV32IZCMP-NEXT:    sw a7, 44(sp)
-; RV32IZCMP-NEXT:    sw a6, 40(sp)
-; RV32IZCMP-NEXT:    sw a5, 36(sp)
 ; RV32IZCMP-NEXT:    sw a4, 32(sp)
-; RV32IZCMP-NEXT:    sw a3, 28(sp)
-; RV32IZCMP-NEXT:    sw a2, 24(sp)
-; RV32IZCMP-NEXT:    sw a1, 20(sp)
+; RV32IZCMP-NEXT:    sw a5, 36(sp)
+; RV32IZCMP-NEXT:    sw a6, 40(sp)
+; RV32IZCMP-NEXT:    sw a7, 44(sp)
 ; RV32IZCMP-NEXT:    sw a0, 16(sp)
+; RV32IZCMP-NEXT:    sw a1, 20(sp)
+; RV32IZCMP-NEXT:    sw a2, 24(sp)
+; RV32IZCMP-NEXT:    sw a3, 28(sp)
 ; RV32IZCMP-NEXT:    call callee
 ; RV32IZCMP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-NEXT:    addi sp, sp, 48
@@ -2850,14 +2850,14 @@ define void @varargs(...) {
 ; RV32IZCMP-WITH-FP-NEXT:    .cfi_offset s0, -40
 ; RV32IZCMP-WITH-FP-NEXT:    addi s0, sp, 16
 ; RV32IZCMP-WITH-FP-NEXT:    .cfi_def_cfa s0, 32
-; RV32IZCMP-WITH-FP-NEXT:    sw a7, 28(s0)
-; RV32IZCMP-WITH-FP-NEXT:    sw a6, 24(s0)
-; RV32IZCMP-WITH-FP-NEXT:    sw a5, 20(s0)
 ; RV32IZCMP-WITH-FP-NEXT:    sw a4, 16(s0)
-; RV32IZCMP-WITH-FP-NEXT:    sw a3, 12(s0)
-; RV32IZCMP-WITH-FP-NEXT:    sw a2, 8(s0)
-; RV32IZCMP-WITH-FP-NEXT:    sw a1, 4(s0)
+; RV32IZCMP-WITH-FP-NEXT:    sw a5, 20(s0)
+; RV32IZCMP-WITH-FP-NEXT:    sw a6, 24(s0)
+; RV32IZCMP-WITH-FP-NEXT:    sw a7, 28(s0)
 ; RV32IZCMP-WITH-FP-NEXT:    sw a0, 0(s0)
+; RV32IZCMP-WITH-FP-NEXT:    sw a1, 4(s0)
+; RV32IZCMP-WITH-FP-NEXT:    sw a2, 8(s0)
+; RV32IZCMP-WITH-FP-NEXT:    sw a3, 12(s0)
 ; RV32IZCMP-WITH-FP-NEXT:    call callee
 ; RV32IZCMP-WITH-FP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZCMP-WITH-FP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -2870,14 +2870,14 @@ define void @varargs(...) {
 ; RV64I-NEXT:    .cfi_def_cfa_offset 80
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -72
-; RV64I-NEXT:    sd a7, 72(sp)
-; RV64I-NEXT:    sd a6, 64(sp)
-; RV64I-NEXT:    sd a5, 56(sp)
 ; RV64I-NEXT:    sd a4, 48(sp)
-; RV64I-NEXT:    sd a3, 40(sp)
-; RV64I-NEXT:    sd a2, 32(sp)
-; RV64I-NEXT:    sd a1, 24(sp)
+; RV64I-NEXT:    sd a5, 56(sp)
+; RV64I-NEXT:    sd a6, 64(sp)
+; RV64I-NEXT:    sd a7, 72(sp)
 ; RV64I-NEXT:    sd a0, 16(sp)
+; RV64I-NEXT:    sd a1, 24(sp)
+; RV64I-NEXT:    sd a2, 32(sp)
+; RV64I-NEXT:    sd a3, 40(sp)
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 80
@@ -2889,12 +2889,12 @@ define void @varargs(...) {
 ; RV64I-LP64E-NEXT:    .cfi_def_cfa_offset 56
 ; RV64I-LP64E-NEXT:    sd ra, 0(sp) # 8-byte Folded Spill
 ; RV64I-LP64E-NEXT:    .cfi_offset ra, -56
-; RV64I-LP64E-NEXT:    sd a5, 48(sp)
 ; RV64I-LP64E-NEXT:    sd a4, 40(sp)
-; RV64I-LP64E-NEXT:    sd a3, 32(sp)
-; RV64I-LP64E-NEXT:    sd a2, 24(sp)
-; RV64I-LP64E-NEXT:    sd a1, 16(sp)
+; RV64I-LP64E-NEXT:    sd a5, 48(sp)
 ; RV64I-LP64E-NEXT:    sd a0, 8(sp)
+; RV64I-LP64E-NEXT:    sd a1, 16(sp)
+; RV64I-LP64E-NEXT:    sd a2, 24(sp)
+; RV64I-LP64E-NEXT:    sd a3, 32(sp)
 ; RV64I-LP64E-NEXT:    call callee
 ; RV64I-LP64E-NEXT:    ld ra, 0(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    addi sp, sp, 56
@@ -2910,14 +2910,14 @@ define void @varargs(...) {
 ; RV64I-WITH-FP-NEXT:    .cfi_offset s0, -80
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 16
 ; RV64I-WITH-FP-NEXT:    .cfi_def_cfa s0, 64
-; RV64I-WITH-FP-NEXT:    sd a7, 56(s0)
-; RV64I-WITH-FP-NEXT:    sd a6, 48(s0)
-; RV64I-WITH-FP-NEXT:    sd a5, 40(s0)
 ; RV64I-WITH-FP-NEXT:    sd a4, 32(s0)
-; RV64I-WITH-FP-NEXT:    sd a3, 24(s0)
-; RV64I-WITH-FP-NEXT:    sd a2, 16(s0)
-; RV64I-WITH-FP-NEXT:    sd a1, 8(s0)
+; RV64I-WITH-FP-NEXT:    sd a5, 40(s0)
+; RV64I-WITH-FP-NEXT:    sd a6, 48(s0)
+; RV64I-WITH-FP-NEXT:    sd a7, 56(s0)
 ; RV64I-WITH-FP-NEXT:    sd a0, 0(s0)
+; RV64I-WITH-FP-NEXT:    sd a1, 8(s0)
+; RV64I-WITH-FP-NEXT:    sd a2, 16(s0)
+; RV64I-WITH-FP-NEXT:    sd a3, 24(s0)
 ; RV64I-WITH-FP-NEXT:    call callee
 ; RV64I-WITH-FP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
@@ -2930,14 +2930,14 @@ define void @varargs(...) {
 ; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 80
 ; RV64IZCMP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IZCMP-NEXT:    .cfi_offset ra, -72
-; RV64IZCMP-NEXT:    sd a7, 72(sp)
-; RV64IZCMP-NEXT:    sd a6, 64(sp)
-; RV64IZCMP-NEXT:    sd a5, 56(sp)
 ; RV64IZCMP-NEXT:    sd a4, 48(sp)
-; RV64IZCMP-NEXT:    sd a3, 40(sp)
-; RV64IZCMP-NEXT:    sd a2, 32(sp)
-; RV64IZCMP-NEXT:    sd a1, 24(sp)
+; RV64IZCMP-NEXT:    sd a5, 56(sp)
+; RV64IZCMP-NEXT:    sd a6, 64(sp)
+; RV64IZCMP-NEXT:    sd a7, 72(sp)
 ; RV64IZCMP-NEXT:    sd a0, 16(sp)
+; RV64IZCMP-NEXT:    sd a1, 24(sp)
+; RV64IZCMP-NEXT:    sd a2, 32(sp)
+; RV64IZCMP-NEXT:    sd a3, 40(sp)
 ; RV64IZCMP-NEXT:    call callee
 ; RV64IZCMP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IZCMP-NEXT:    addi sp, sp, 80
@@ -2953,14 +2953,14 @@ define void @varargs(...) {
 ; RV64IZCMP-WITH-FP-NEXT:    .cfi_offset s0, -80
 ; RV64IZCMP-WITH-FP-NEXT:    addi s0, sp, 16
 ; RV64IZCMP-WITH-FP-NEXT:    .cfi_def_cfa s0, 64
-; RV64IZCMP-WITH-FP-NEXT:    sd a7, 56(s0)
-; RV64IZCMP-WITH-FP-NEXT:    sd a6, 48(s0)
-; RV64IZCMP-WITH-FP-NEXT:    sd a5, 40(s0)
 ; RV64IZCMP-WITH-FP-NEXT:    sd a4, 32(s0)
-; RV64IZCMP-WITH-FP-NEXT:    sd a3, 24(s0)
-; RV64IZCMP-WITH-FP-NEXT:    sd a2, 16(s0)
-; RV64IZCMP-WITH-FP-NEXT:    sd a1, 8(s0)
+; RV64IZCMP-WITH-FP-NEXT:    sd a5, 40(s0)
+; RV64IZCMP-WITH-FP-NEXT:    sd a6, 48(s0)
+; RV64IZCMP-WITH-FP-NEXT:    sd a7, 56(s0)
 ; RV64IZCMP-WITH-FP-NEXT:    sd a0, 0(s0)
+; RV64IZCMP-WITH-FP-NEXT:    sd a1, 8(s0)
+; RV64IZCMP-WITH-FP-NEXT:    sd a2, 16(s0)
+; RV64IZCMP-WITH-FP-NEXT:    sd a3, 24(s0)
 ; RV64IZCMP-WITH-FP-NEXT:    call callee
 ; RV64IZCMP-WITH-FP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IZCMP-WITH-FP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
index 278187f62cd75ee769e33f63744cfbdf33769b84..e97a3bff32fac700ce149f2ecfd5c49c55a177fb 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
@@ -146,30 +146,27 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -64
 ; RV32I-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    li a0, 18
+; RV32I-FPELIM-NEXT:    li a1, 17
+; RV32I-FPELIM-NEXT:    sw a1, 20(sp)
 ; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
-; RV32I-FPELIM-NEXT:    li a0, 17
-; RV32I-FPELIM-NEXT:    sw a0, 20(sp)
 ; RV32I-FPELIM-NEXT:    li a0, 16
+; RV32I-FPELIM-NEXT:    lui a1, 262236
+; RV32I-FPELIM-NEXT:    addi a1, a1, 655
+; RV32I-FPELIM-NEXT:    lui a2, 377487
+; RV32I-FPELIM-NEXT:    addi a2, a2, 1475
+; RV32I-FPELIM-NEXT:    li a3, 15
+; RV32I-FPELIM-NEXT:    sw a3, 0(sp)
+; RV32I-FPELIM-NEXT:    sw a2, 8(sp)
+; RV32I-FPELIM-NEXT:    sw a1, 12(sp)
 ; RV32I-FPELIM-NEXT:    sw a0, 16(sp)
-; RV32I-FPELIM-NEXT:    lui a0, 262236
-; RV32I-FPELIM-NEXT:    addi a0, a0, 655
-; RV32I-FPELIM-NEXT:    sw a0, 12(sp)
-; RV32I-FPELIM-NEXT:    lui a0, 377487
-; RV32I-FPELIM-NEXT:    addi a0, a0, 1475
-; RV32I-FPELIM-NEXT:    sw a0, 8(sp)
-; RV32I-FPELIM-NEXT:    li a0, 15
-; RV32I-FPELIM-NEXT:    sw a0, 0(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 262153
-; RV32I-FPELIM-NEXT:    addi a0, a0, 491
-; RV32I-FPELIM-NEXT:    sw a0, 44(sp)
+; RV32I-FPELIM-NEXT:    addi t0, a0, 491
 ; RV32I-FPELIM-NEXT:    lui a0, 545260
-; RV32I-FPELIM-NEXT:    addi a0, a0, -1967
-; RV32I-FPELIM-NEXT:    sw a0, 40(sp)
+; RV32I-FPELIM-NEXT:    addi t1, a0, -1967
 ; RV32I-FPELIM-NEXT:    lui a0, 964690
-; RV32I-FPELIM-NEXT:    addi a0, a0, -328
-; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
+; RV32I-FPELIM-NEXT:    addi t2, a0, -328
 ; RV32I-FPELIM-NEXT:    lui a0, 335544
-; RV32I-FPELIM-NEXT:    addi t0, a0, 1311
+; RV32I-FPELIM-NEXT:    addi t3, a0, 1311
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a5, a0, -2048
 ; RV32I-FPELIM-NEXT:    li a0, 1
@@ -179,7 +176,10 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    li a4, 13
 ; RV32I-FPELIM-NEXT:    li a6, 4
 ; RV32I-FPELIM-NEXT:    li a7, 14
-; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
+; RV32I-FPELIM-NEXT:    sw t3, 32(sp)
+; RV32I-FPELIM-NEXT:    sw t2, 36(sp)
+; RV32I-FPELIM-NEXT:    sw t1, 40(sp)
+; RV32I-FPELIM-NEXT:    sw t0, 44(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -192,30 +192,27 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 64
 ; RV32I-WITHFP-NEXT:    li a0, 18
+; RV32I-WITHFP-NEXT:    li a1, 17
+; RV32I-WITHFP-NEXT:    sw a1, 20(sp)
 ; RV32I-WITHFP-NEXT:    sw a0, 24(sp)
-; RV32I-WITHFP-NEXT:    li a0, 17
-; RV32I-WITHFP-NEXT:    sw a0, 20(sp)
 ; RV32I-WITHFP-NEXT:    li a0, 16
+; RV32I-WITHFP-NEXT:    lui a1, 262236
+; RV32I-WITHFP-NEXT:    addi a1, a1, 655
+; RV32I-WITHFP-NEXT:    lui a2, 377487
+; RV32I-WITHFP-NEXT:    addi a2, a2, 1475
+; RV32I-WITHFP-NEXT:    li a3, 15
+; RV32I-WITHFP-NEXT:    sw a3, 0(sp)
+; RV32I-WITHFP-NEXT:    sw a2, 8(sp)
+; RV32I-WITHFP-NEXT:    sw a1, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw a0, 16(sp)
-; RV32I-WITHFP-NEXT:    lui a0, 262236
-; RV32I-WITHFP-NEXT:    addi a0, a0, 655
-; RV32I-WITHFP-NEXT:    sw a0, 12(sp)
-; RV32I-WITHFP-NEXT:    lui a0, 377487
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1475
-; RV32I-WITHFP-NEXT:    sw a0, 8(sp)
-; RV32I-WITHFP-NEXT:    li a0, 15
-; RV32I-WITHFP-NEXT:    sw a0, 0(sp)
 ; RV32I-WITHFP-NEXT:    lui a0, 262153
-; RV32I-WITHFP-NEXT:    addi a0, a0, 491
-; RV32I-WITHFP-NEXT:    sw a0, -20(s0)
+; RV32I-WITHFP-NEXT:    addi t0, a0, 491
 ; RV32I-WITHFP-NEXT:    lui a0, 545260
-; RV32I-WITHFP-NEXT:    addi a0, a0, -1967
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi t1, a0, -1967
 ; RV32I-WITHFP-NEXT:    lui a0, 964690
-; RV32I-WITHFP-NEXT:    addi a0, a0, -328
-; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
+; RV32I-WITHFP-NEXT:    addi t2, a0, -328
 ; RV32I-WITHFP-NEXT:    lui a0, 335544
-; RV32I-WITHFP-NEXT:    addi t0, a0, 1311
+; RV32I-WITHFP-NEXT:    addi t3, a0, 1311
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a5, a0, -2048
 ; RV32I-WITHFP-NEXT:    li a0, 1
@@ -225,7 +222,10 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    li a4, 13
 ; RV32I-WITHFP-NEXT:    li a6, 4
 ; RV32I-WITHFP-NEXT:    li a7, 14
-; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
+; RV32I-WITHFP-NEXT:    sw t3, -32(s0)
+; RV32I-WITHFP-NEXT:    sw t2, -28(s0)
+; RV32I-WITHFP-NEXT:    sw t1, -24(s0)
+; RV32I-WITHFP-NEXT:    sw t0, -20(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
index bb082b0314d59989f67c6a03f877d64d773ac4eb..9e4c8a6e3320c2b9365415e14652102cb829b34c 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
@@ -142,8 +142,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-FPELIM-NEXT:    li a0, 8
-; RV32I-FPELIM-NEXT:    sw a0, 4(sp)
+; RV32I-FPELIM-NEXT:    li a4, 8
 ; RV32I-FPELIM-NEXT:    li a0, 1
 ; RV32I-FPELIM-NEXT:    li a1, 2
 ; RV32I-FPELIM-NEXT:    li a2, 3
@@ -152,6 +151,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    li a6, 6
 ; RV32I-FPELIM-NEXT:    li a7, 7
 ; RV32I-FPELIM-NEXT:    sw zero, 0(sp)
+; RV32I-FPELIM-NEXT:    sw a4, 4(sp)
 ; RV32I-FPELIM-NEXT:    li a4, 0
 ; RV32I-FPELIM-NEXT:    call callee_many_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -164,8 +164,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    li a0, 8
-; RV32I-WITHFP-NEXT:    sw a0, 4(sp)
+; RV32I-WITHFP-NEXT:    li a4, 8
 ; RV32I-WITHFP-NEXT:    li a0, 1
 ; RV32I-WITHFP-NEXT:    li a1, 2
 ; RV32I-WITHFP-NEXT:    li a2, 3
@@ -174,6 +173,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    li a6, 6
 ; RV32I-WITHFP-NEXT:    li a7, 7
 ; RV32I-WITHFP-NEXT:    sw zero, 0(sp)
+; RV32I-WITHFP-NEXT:    sw a4, 4(sp)
 ; RV32I-WITHFP-NEXT:    li a4, 0
 ; RV32I-WITHFP-NEXT:    call callee_many_scalars
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -246,17 +246,17 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -48
 ; RV32I-FPELIM-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    lui a0, 524272
-; RV32I-FPELIM-NEXT:    sw a0, 12(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 8(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 4(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 0(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 36(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 32(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 28(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 4(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 8(sp)
+; RV32I-FPELIM-NEXT:    sw a0, 12(sp)
 ; RV32I-FPELIM-NEXT:    li a2, 1
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 24
 ; RV32I-FPELIM-NEXT:    mv a1, sp
 ; RV32I-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 28(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 32(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 36(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 48
@@ -269,17 +269,17 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 48
 ; RV32I-WITHFP-NEXT:    lui a0, 524272
-; RV32I-WITHFP-NEXT:    sw a0, -36(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -40(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -44(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -48(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -44(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -40(s0)
+; RV32I-WITHFP-NEXT:    sw a0, -36(s0)
 ; RV32I-WITHFP-NEXT:    li a2, 1
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -24
 ; RV32I-WITHFP-NEXT:    addi a1, s0, -48
 ; RV32I-WITHFP-NEXT:    sw a2, -24(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars
 ; RV32I-WITHFP-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
@@ -354,18 +354,15 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -64
 ; RV32I-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 16
+; RV32I-FPELIM-NEXT:    li a1, 9
+; RV32I-FPELIM-NEXT:    sw a1, 0(sp)
 ; RV32I-FPELIM-NEXT:    sw a0, 4(sp)
-; RV32I-FPELIM-NEXT:    li a0, 9
-; RV32I-FPELIM-NEXT:    sw a0, 0(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 524272
-; RV32I-FPELIM-NEXT:    sw a0, 28(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 24(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 20(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 16(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 52(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 48(sp)
-; RV32I-FPELIM-NEXT:    li a0, 8
-; RV32I-FPELIM-NEXT:    sw a0, 40(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 20(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 24(sp)
+; RV32I-FPELIM-NEXT:    sw a0, 28(sp)
+; RV32I-FPELIM-NEXT:    li t0, 8
 ; RV32I-FPELIM-NEXT:    li a0, 1
 ; RV32I-FPELIM-NEXT:    li a1, 2
 ; RV32I-FPELIM-NEXT:    li a2, 3
@@ -374,7 +371,10 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    li a5, 6
 ; RV32I-FPELIM-NEXT:    li a6, 7
 ; RV32I-FPELIM-NEXT:    addi a7, sp, 40
+; RV32I-FPELIM-NEXT:    sw t0, 40(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 44(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 48(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 52(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -387,18 +387,15 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 64
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -48
+; RV32I-WITHFP-NEXT:    li a1, 9
+; RV32I-WITHFP-NEXT:    sw a1, 0(sp)
 ; RV32I-WITHFP-NEXT:    sw a0, 4(sp)
-; RV32I-WITHFP-NEXT:    li a0, 9
-; RV32I-WITHFP-NEXT:    sw a0, 0(sp)
 ; RV32I-WITHFP-NEXT:    lui a0, 524272
-; RV32I-WITHFP-NEXT:    sw a0, -36(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -40(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -44(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -48(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
-; RV32I-WITHFP-NEXT:    li a0, 8
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -44(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -40(s0)
+; RV32I-WITHFP-NEXT:    sw a0, -36(s0)
+; RV32I-WITHFP-NEXT:    li t0, 8
 ; RV32I-WITHFP-NEXT:    li a0, 1
 ; RV32I-WITHFP-NEXT:    li a1, 2
 ; RV32I-WITHFP-NEXT:    li a2, 3
@@ -407,7 +404,10 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    li a5, 6
 ; RV32I-WITHFP-NEXT:    li a6, 7
 ; RV32I-WITHFP-NEXT:    addi a7, s0, -24
+; RV32I-WITHFP-NEXT:    sw t0, -24(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
@@ -553,12 +553,12 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -48
 ; RV32I-FPELIM-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    li a0, 1
-; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
 ; RV32I-FPELIM-NEXT:    li a1, 2
-; RV32I-FPELIM-NEXT:    sw a1, 28(sp)
 ; RV32I-FPELIM-NEXT:    li a2, 3
-; RV32I-FPELIM-NEXT:    sw a2, 32(sp)
 ; RV32I-FPELIM-NEXT:    li a3, 4
+; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
+; RV32I-FPELIM-NEXT:    sw a1, 28(sp)
+; RV32I-FPELIM-NEXT:    sw a2, 32(sp)
 ; RV32I-FPELIM-NEXT:    sw a3, 36(sp)
 ; RV32I-FPELIM-NEXT:    sw a0, 8(sp)
 ; RV32I-FPELIM-NEXT:    sw a1, 12(sp)
@@ -577,12 +577,12 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 48
 ; RV32I-WITHFP-NEXT:    li a0, 1
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
 ; RV32I-WITHFP-NEXT:    li a1, 2
-; RV32I-WITHFP-NEXT:    sw a1, -20(s0)
 ; RV32I-WITHFP-NEXT:    li a2, 3
-; RV32I-WITHFP-NEXT:    sw a2, -16(s0)
 ; RV32I-WITHFP-NEXT:    li a3, 4
+; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    sw a1, -20(s0)
+; RV32I-WITHFP-NEXT:    sw a2, -16(s0)
 ; RV32I-WITHFP-NEXT:    sw a3, -12(s0)
 ; RV32I-WITHFP-NEXT:    sw a0, -40(s0)
 ; RV32I-WITHFP-NEXT:    sw a1, -36(s0)
@@ -665,27 +665,24 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -64
 ; RV32I-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    li a0, 19
+; RV32I-FPELIM-NEXT:    li a1, 18
+; RV32I-FPELIM-NEXT:    sw a1, 20(sp)
 ; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
-; RV32I-FPELIM-NEXT:    li a0, 18
-; RV32I-FPELIM-NEXT:    sw a0, 20(sp)
 ; RV32I-FPELIM-NEXT:    li a0, 17
-; RV32I-FPELIM-NEXT:    sw a0, 16(sp)
+; RV32I-FPELIM-NEXT:    li a1, 16
+; RV32I-FPELIM-NEXT:    li a2, 15
+; RV32I-FPELIM-NEXT:    sw a2, 0(sp)
+; RV32I-FPELIM-NEXT:    sw a1, 8(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 12(sp)
-; RV32I-FPELIM-NEXT:    li a0, 16
-; RV32I-FPELIM-NEXT:    sw a0, 8(sp)
-; RV32I-FPELIM-NEXT:    li a0, 15
-; RV32I-FPELIM-NEXT:    sw a0, 0(sp)
+; RV32I-FPELIM-NEXT:    sw a0, 16(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 262153
-; RV32I-FPELIM-NEXT:    addi a0, a0, 491
-; RV32I-FPELIM-NEXT:    sw a0, 44(sp)
+; RV32I-FPELIM-NEXT:    addi t0, a0, 491
 ; RV32I-FPELIM-NEXT:    lui a0, 545260
-; RV32I-FPELIM-NEXT:    addi a0, a0, -1967
-; RV32I-FPELIM-NEXT:    sw a0, 40(sp)
+; RV32I-FPELIM-NEXT:    addi t1, a0, -1967
 ; RV32I-FPELIM-NEXT:    lui a0, 964690
-; RV32I-FPELIM-NEXT:    addi a0, a0, -328
-; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
+; RV32I-FPELIM-NEXT:    addi t2, a0, -328
 ; RV32I-FPELIM-NEXT:    lui a0, 335544
-; RV32I-FPELIM-NEXT:    addi t0, a0, 1311
+; RV32I-FPELIM-NEXT:    addi t3, a0, 1311
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a5, a0, -2048
 ; RV32I-FPELIM-NEXT:    li a0, 1
@@ -695,7 +692,10 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    li a4, 13
 ; RV32I-FPELIM-NEXT:    li a6, 4
 ; RV32I-FPELIM-NEXT:    li a7, 14
-; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
+; RV32I-FPELIM-NEXT:    sw t3, 32(sp)
+; RV32I-FPELIM-NEXT:    sw t2, 36(sp)
+; RV32I-FPELIM-NEXT:    sw t1, 40(sp)
+; RV32I-FPELIM-NEXT:    sw t0, 44(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -708,27 +708,24 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 64
 ; RV32I-WITHFP-NEXT:    li a0, 19
+; RV32I-WITHFP-NEXT:    li a1, 18
+; RV32I-WITHFP-NEXT:    sw a1, 20(sp)
 ; RV32I-WITHFP-NEXT:    sw a0, 24(sp)
-; RV32I-WITHFP-NEXT:    li a0, 18
-; RV32I-WITHFP-NEXT:    sw a0, 20(sp)
 ; RV32I-WITHFP-NEXT:    li a0, 17
-; RV32I-WITHFP-NEXT:    sw a0, 16(sp)
+; RV32I-WITHFP-NEXT:    li a1, 16
+; RV32I-WITHFP-NEXT:    li a2, 15
+; RV32I-WITHFP-NEXT:    sw a2, 0(sp)
+; RV32I-WITHFP-NEXT:    sw a1, 8(sp)
 ; RV32I-WITHFP-NEXT:    sw zero, 12(sp)
-; RV32I-WITHFP-NEXT:    li a0, 16
-; RV32I-WITHFP-NEXT:    sw a0, 8(sp)
-; RV32I-WITHFP-NEXT:    li a0, 15
-; RV32I-WITHFP-NEXT:    sw a0, 0(sp)
+; RV32I-WITHFP-NEXT:    sw a0, 16(sp)
 ; RV32I-WITHFP-NEXT:    lui a0, 262153
-; RV32I-WITHFP-NEXT:    addi a0, a0, 491
-; RV32I-WITHFP-NEXT:    sw a0, -20(s0)
+; RV32I-WITHFP-NEXT:    addi t0, a0, 491
 ; RV32I-WITHFP-NEXT:    lui a0, 545260
-; RV32I-WITHFP-NEXT:    addi a0, a0, -1967
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi t1, a0, -1967
 ; RV32I-WITHFP-NEXT:    lui a0, 964690
-; RV32I-WITHFP-NEXT:    addi a0, a0, -328
-; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
+; RV32I-WITHFP-NEXT:    addi t2, a0, -328
 ; RV32I-WITHFP-NEXT:    lui a0, 335544
-; RV32I-WITHFP-NEXT:    addi t0, a0, 1311
+; RV32I-WITHFP-NEXT:    addi t3, a0, 1311
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a5, a0, -2048
 ; RV32I-WITHFP-NEXT:    li a0, 1
@@ -738,7 +735,10 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    li a4, 13
 ; RV32I-WITHFP-NEXT:    li a6, 4
 ; RV32I-WITHFP-NEXT:    li a7, 14
-; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
+; RV32I-WITHFP-NEXT:    sw t3, -32(s0)
+; RV32I-WITHFP-NEXT:    sw t2, -28(s0)
+; RV32I-WITHFP-NEXT:    sw t1, -24(s0)
+; RV32I-WITHFP-NEXT:    sw t0, -20(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
@@ -881,10 +881,10 @@ define fp128 @callee_large_scalar_ret() nounwind {
 ; RV32I-FPELIM-LABEL: callee_large_scalar_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lui a1, 524272
-; RV32I-FPELIM-NEXT:    sw a1, 12(a0)
-; RV32I-FPELIM-NEXT:    sw zero, 8(a0)
-; RV32I-FPELIM-NEXT:    sw zero, 4(a0)
 ; RV32I-FPELIM-NEXT:    sw zero, 0(a0)
+; RV32I-FPELIM-NEXT:    sw zero, 4(a0)
+; RV32I-FPELIM-NEXT:    sw zero, 8(a0)
+; RV32I-FPELIM-NEXT:    sw a1, 12(a0)
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_large_scalar_ret:
@@ -894,10 +894,10 @@ define fp128 @callee_large_scalar_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lui a1, 524272
-; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    sw zero, 8(a0)
-; RV32I-WITHFP-NEXT:    sw zero, 4(a0)
 ; RV32I-WITHFP-NEXT:    sw zero, 0(a0)
+; RV32I-WITHFP-NEXT:    sw zero, 4(a0)
+; RV32I-WITHFP-NEXT:    sw zero, 8(a0)
+; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -938,13 +938,13 @@ define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result
 ; RV32I-FPELIM-LABEL: callee_large_struct_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    li a1, 1
+; RV32I-FPELIM-NEXT:    li a2, 2
+; RV32I-FPELIM-NEXT:    li a3, 3
+; RV32I-FPELIM-NEXT:    li a4, 4
 ; RV32I-FPELIM-NEXT:    sw a1, 0(a0)
-; RV32I-FPELIM-NEXT:    li a1, 2
-; RV32I-FPELIM-NEXT:    sw a1, 4(a0)
-; RV32I-FPELIM-NEXT:    li a1, 3
-; RV32I-FPELIM-NEXT:    sw a1, 8(a0)
-; RV32I-FPELIM-NEXT:    li a1, 4
-; RV32I-FPELIM-NEXT:    sw a1, 12(a0)
+; RV32I-FPELIM-NEXT:    sw a2, 4(a0)
+; RV32I-FPELIM-NEXT:    sw a3, 8(a0)
+; RV32I-FPELIM-NEXT:    sw a4, 12(a0)
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_large_struct_ret:
@@ -954,13 +954,13 @@ define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    li a1, 1
+; RV32I-WITHFP-NEXT:    li a2, 2
+; RV32I-WITHFP-NEXT:    li a3, 3
+; RV32I-WITHFP-NEXT:    li a4, 4
 ; RV32I-WITHFP-NEXT:    sw a1, 0(a0)
-; RV32I-WITHFP-NEXT:    li a1, 2
-; RV32I-WITHFP-NEXT:    sw a1, 4(a0)
-; RV32I-WITHFP-NEXT:    li a1, 3
-; RV32I-WITHFP-NEXT:    sw a1, 8(a0)
-; RV32I-WITHFP-NEXT:    li a1, 4
-; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
+; RV32I-WITHFP-NEXT:    sw a2, 4(a0)
+; RV32I-WITHFP-NEXT:    sw a3, 8(a0)
+; RV32I-WITHFP-NEXT:    sw a4, 12(a0)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
index bcceea7ac35b3e244282647a4a7a95cd336d5f03..1321413fbc57e8fd0d91699eb571eff280d5b25c 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
@@ -203,8 +203,7 @@ define i32 @caller_double_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-ILP32D-NEXT:    lui a0, 262816
-; RV32-ILP32D-NEXT:    sw a0, 4(sp)
+; RV32-ILP32D-NEXT:    lui a1, 262816
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_0)
 ; RV32-ILP32D-NEXT:    fld fa0, %lo(.LCPI9_0)(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_1)
@@ -226,6 +225,7 @@ define i32 @caller_double_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    li a4, 5
 ; RV32-ILP32D-NEXT:    li a6, 7
 ; RV32-ILP32D-NEXT:    sw zero, 0(sp)
+; RV32-ILP32D-NEXT:    sw a1, 4(sp)
 ; RV32-ILP32D-NEXT:    li a1, 0
 ; RV32-ILP32D-NEXT:    li a3, 0
 ; RV32-ILP32D-NEXT:    li a5, 0
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32e.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32e.ll
index 708cb00d1c45c64f3c8c987ab18e7608e9770df1..948645948f64d23cc86d910018842f9c4f93208d 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32e.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32e.ll
@@ -199,14 +199,14 @@ define i32 @caller_float_on_stack() {
 ; ILP32E-FPELIM-NEXT:    .cfi_def_cfa_offset 16
 ; ILP32E-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32E-FPELIM-NEXT:    .cfi_offset ra, -4
-; ILP32E-FPELIM-NEXT:    lui a0, 264704
-; ILP32E-FPELIM-NEXT:    sw a0, 8(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 4(sp)
-; ILP32E-FPELIM-NEXT:    li a1, 4
+; ILP32E-FPELIM-NEXT:    lui a1, 264704
+; ILP32E-FPELIM-NEXT:    li a3, 4
 ; ILP32E-FPELIM-NEXT:    li a0, 1
 ; ILP32E-FPELIM-NEXT:    li a2, 2
 ; ILP32E-FPELIM-NEXT:    li a4, 3
-; ILP32E-FPELIM-NEXT:    sw a1, 0(sp)
+; ILP32E-FPELIM-NEXT:    sw a3, 0(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 4(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
 ; ILP32E-FPELIM-NEXT:    li a1, 0
 ; ILP32E-FPELIM-NEXT:    li a3, 0
 ; ILP32E-FPELIM-NEXT:    li a5, 0
@@ -225,14 +225,14 @@ define i32 @caller_float_on_stack() {
 ; ILP32E-WITHFP-NEXT:    .cfi_offset s0, -8
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 20
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
-; ILP32E-WITHFP-NEXT:    lui a0, 264704
-; ILP32E-WITHFP-NEXT:    sw a0, 8(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 4(sp)
-; ILP32E-WITHFP-NEXT:    li a1, 4
+; ILP32E-WITHFP-NEXT:    lui a1, 264704
+; ILP32E-WITHFP-NEXT:    li a3, 4
 ; ILP32E-WITHFP-NEXT:    li a0, 1
 ; ILP32E-WITHFP-NEXT:    li a2, 2
 ; ILP32E-WITHFP-NEXT:    li a4, 3
-; ILP32E-WITHFP-NEXT:    sw a1, 0(sp)
+; ILP32E-WITHFP-NEXT:    sw a3, 0(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 4(sp)
+; ILP32E-WITHFP-NEXT:    sw a1, 8(sp)
 ; ILP32E-WITHFP-NEXT:    li a1, 0
 ; ILP32E-WITHFP-NEXT:    li a3, 0
 ; ILP32E-WITHFP-NEXT:    li a5, 0
@@ -248,14 +248,14 @@ define i32 @caller_float_on_stack() {
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, sp, -12
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_def_cfa_offset 16
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_offset ra, -4
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 264704
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 8(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 4
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a1, 264704
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 4
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 1
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 2
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a4, 3
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 0(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a3, 0(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 8(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 0
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 0
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a5, 0
@@ -272,14 +272,14 @@ define i32 @caller_float_on_stack() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_offset s0, -8
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi s0, sp, 20
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 264704
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 8(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 4
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a1, 264704
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 4
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 1
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 2
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a4, 3
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 0(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a3, 0(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 8(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a5, 0
@@ -630,34 +630,31 @@ define void @caller_aligned_stack() {
 ; ILP32E-FPELIM-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-FPELIM-NEXT:    andi sp, sp, -16
 ; ILP32E-FPELIM-NEXT:    li a0, 18
+; ILP32E-FPELIM-NEXT:    li a1, 17
+; ILP32E-FPELIM-NEXT:    li a2, 16
+; ILP32E-FPELIM-NEXT:    lui a3, 262236
+; ILP32E-FPELIM-NEXT:    addi a3, a3, 655
+; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 20(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 24(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 28(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 17
-; ILP32E-FPELIM-NEXT:    sw a0, 24(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 16
-; ILP32E-FPELIM-NEXT:    sw a0, 20(sp)
-; ILP32E-FPELIM-NEXT:    lui a0, 262236
-; ILP32E-FPELIM-NEXT:    addi a0, a0, 655
-; ILP32E-FPELIM-NEXT:    sw a0, 16(sp)
 ; ILP32E-FPELIM-NEXT:    lui a0, 377487
 ; ILP32E-FPELIM-NEXT:    addi a0, a0, 1475
+; ILP32E-FPELIM-NEXT:    li a1, 15
+; ILP32E-FPELIM-NEXT:    li a2, 14
+; ILP32E-FPELIM-NEXT:    li a3, 4
+; ILP32E-FPELIM-NEXT:    sw a3, 0(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 4(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 15
-; ILP32E-FPELIM-NEXT:    sw a0, 8(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 14
-; ILP32E-FPELIM-NEXT:    sw a0, 4(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 4
-; ILP32E-FPELIM-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-NEXT:    lui a0, 262153
-; ILP32E-FPELIM-NEXT:    addi a0, a0, 491
-; ILP32E-FPELIM-NEXT:    sw a0, 44(sp)
+; ILP32E-FPELIM-NEXT:    addi a6, a0, 491
 ; ILP32E-FPELIM-NEXT:    lui a0, 545260
-; ILP32E-FPELIM-NEXT:    addi a0, a0, -1967
-; ILP32E-FPELIM-NEXT:    sw a0, 40(sp)
+; ILP32E-FPELIM-NEXT:    addi a7, a0, -1967
 ; ILP32E-FPELIM-NEXT:    lui a0, 964690
-; ILP32E-FPELIM-NEXT:    addi a0, a0, -328
-; ILP32E-FPELIM-NEXT:    sw a0, 36(sp)
+; ILP32E-FPELIM-NEXT:    addi t0, a0, -328
 ; ILP32E-FPELIM-NEXT:    lui a0, 335544
-; ILP32E-FPELIM-NEXT:    addi a6, a0, 1311
+; ILP32E-FPELIM-NEXT:    addi t1, a0, 1311
 ; ILP32E-FPELIM-NEXT:    lui a0, 688509
 ; ILP32E-FPELIM-NEXT:    addi a5, a0, -2048
 ; ILP32E-FPELIM-NEXT:    li a0, 1
@@ -665,7 +662,10 @@ define void @caller_aligned_stack() {
 ; ILP32E-FPELIM-NEXT:    addi a2, sp, 32
 ; ILP32E-FPELIM-NEXT:    li a3, 12
 ; ILP32E-FPELIM-NEXT:    li a4, 13
-; ILP32E-FPELIM-NEXT:    sw a6, 32(sp)
+; ILP32E-FPELIM-NEXT:    sw t1, 32(sp)
+; ILP32E-FPELIM-NEXT:    sw t0, 36(sp)
+; ILP32E-FPELIM-NEXT:    sw a7, 40(sp)
+; ILP32E-FPELIM-NEXT:    sw a6, 44(sp)
 ; ILP32E-FPELIM-NEXT:    call callee_aligned_stack
 ; ILP32E-FPELIM-NEXT:    addi sp, s0, -64
 ; ILP32E-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
@@ -685,34 +685,31 @@ define void @caller_aligned_stack() {
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-NEXT:    andi sp, sp, -16
 ; ILP32E-WITHFP-NEXT:    li a0, 18
+; ILP32E-WITHFP-NEXT:    li a1, 17
+; ILP32E-WITHFP-NEXT:    li a2, 16
+; ILP32E-WITHFP-NEXT:    lui a3, 262236
+; ILP32E-WITHFP-NEXT:    addi a3, a3, 655
+; ILP32E-WITHFP-NEXT:    sw a3, 16(sp)
+; ILP32E-WITHFP-NEXT:    sw a2, 20(sp)
+; ILP32E-WITHFP-NEXT:    sw a1, 24(sp)
 ; ILP32E-WITHFP-NEXT:    sw a0, 28(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 17
-; ILP32E-WITHFP-NEXT:    sw a0, 24(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 16
-; ILP32E-WITHFP-NEXT:    sw a0, 20(sp)
-; ILP32E-WITHFP-NEXT:    lui a0, 262236
-; ILP32E-WITHFP-NEXT:    addi a0, a0, 655
-; ILP32E-WITHFP-NEXT:    sw a0, 16(sp)
 ; ILP32E-WITHFP-NEXT:    lui a0, 377487
 ; ILP32E-WITHFP-NEXT:    addi a0, a0, 1475
+; ILP32E-WITHFP-NEXT:    li a1, 15
+; ILP32E-WITHFP-NEXT:    li a2, 14
+; ILP32E-WITHFP-NEXT:    li a3, 4
+; ILP32E-WITHFP-NEXT:    sw a3, 0(sp)
+; ILP32E-WITHFP-NEXT:    sw a2, 4(sp)
+; ILP32E-WITHFP-NEXT:    sw a1, 8(sp)
 ; ILP32E-WITHFP-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 15
-; ILP32E-WITHFP-NEXT:    sw a0, 8(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 14
-; ILP32E-WITHFP-NEXT:    sw a0, 4(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 4
-; ILP32E-WITHFP-NEXT:    sw a0, 0(sp)
 ; ILP32E-WITHFP-NEXT:    lui a0, 262153
-; ILP32E-WITHFP-NEXT:    addi a0, a0, 491
-; ILP32E-WITHFP-NEXT:    sw a0, 44(sp)
+; ILP32E-WITHFP-NEXT:    addi a6, a0, 491
 ; ILP32E-WITHFP-NEXT:    lui a0, 545260
-; ILP32E-WITHFP-NEXT:    addi a0, a0, -1967
-; ILP32E-WITHFP-NEXT:    sw a0, 40(sp)
+; ILP32E-WITHFP-NEXT:    addi a7, a0, -1967
 ; ILP32E-WITHFP-NEXT:    lui a0, 964690
-; ILP32E-WITHFP-NEXT:    addi a0, a0, -328
-; ILP32E-WITHFP-NEXT:    sw a0, 36(sp)
+; ILP32E-WITHFP-NEXT:    addi t0, a0, -328
 ; ILP32E-WITHFP-NEXT:    lui a0, 335544
-; ILP32E-WITHFP-NEXT:    addi a6, a0, 1311
+; ILP32E-WITHFP-NEXT:    addi t1, a0, 1311
 ; ILP32E-WITHFP-NEXT:    lui a0, 688509
 ; ILP32E-WITHFP-NEXT:    addi a5, a0, -2048
 ; ILP32E-WITHFP-NEXT:    li a0, 1
@@ -720,7 +717,10 @@ define void @caller_aligned_stack() {
 ; ILP32E-WITHFP-NEXT:    addi a2, sp, 32
 ; ILP32E-WITHFP-NEXT:    li a3, 12
 ; ILP32E-WITHFP-NEXT:    li a4, 13
-; ILP32E-WITHFP-NEXT:    sw a6, 32(sp)
+; ILP32E-WITHFP-NEXT:    sw t1, 32(sp)
+; ILP32E-WITHFP-NEXT:    sw t0, 36(sp)
+; ILP32E-WITHFP-NEXT:    sw a7, 40(sp)
+; ILP32E-WITHFP-NEXT:    sw a6, 44(sp)
 ; ILP32E-WITHFP-NEXT:    call callee_aligned_stack
 ; ILP32E-WITHFP-NEXT:    addi sp, s0, -64
 ; ILP32E-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
@@ -739,34 +739,31 @@ define void @caller_aligned_stack() {
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    andi sp, sp, -16
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 18
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 17
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 16
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a3, 262236
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a3, a3, 655
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a3, 16(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a2, 20(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 24(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 28(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 17
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 24(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 16
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 20(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 262236
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, a0, 655
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 16(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 377487
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, a0, 1475
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 15
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 14
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 4
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a3, 0(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a2, 4(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 8(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 15
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 8(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 14
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 4(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 4
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 262153
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, a0, 491
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 44(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a6, a0, 491
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 545260
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, a0, -1967
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 40(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a7, a0, -1967
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 964690
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, a0, -328
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 36(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi t0, a0, -328
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 335544
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a6, a0, 1311
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi t1, a0, 1311
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 688509
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a5, a0, -2048
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 1
@@ -774,7 +771,10 @@ define void @caller_aligned_stack() {
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a2, sp, 32
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 12
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a4, 13
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a6, 32(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw t1, 32(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw t0, 36(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a7, 40(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a6, 44(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    call callee_aligned_stack
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, s0, -64
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, sp, 56
@@ -791,34 +791,31 @@ define void @caller_aligned_stack() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    andi sp, sp, -16
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 18
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 17
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 16
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a3, 262236
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a3, a3, 655
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a3, 16(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a2, 20(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 24(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 28(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 17
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 24(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 16
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 20(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 262236
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, a0, 655
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 16(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 377487
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, a0, 1475
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 15
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 14
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 4
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a3, 0(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a2, 4(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 8(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 15
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 8(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 14
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 4(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 4
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 0(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 262153
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, a0, 491
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 44(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a6, a0, 491
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 545260
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, a0, -1967
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 40(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a7, a0, -1967
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 964690
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, a0, -328
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 36(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi t0, a0, -328
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 335544
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a6, a0, 1311
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi t1, a0, 1311
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 688509
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a5, a0, -2048
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 1
@@ -826,7 +823,10 @@ define void @caller_aligned_stack() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a2, sp, 32
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 12
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a4, 13
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a6, 32(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw t1, 32(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw t0, 36(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a7, 40(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a6, 44(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    call callee_aligned_stack
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi sp, s0, -64
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi sp, sp, 56
@@ -1163,18 +1163,18 @@ define i32 @caller_many_scalars() {
 ; ILP32E-FPELIM-NEXT:    .cfi_def_cfa_offset 20
 ; ILP32E-FPELIM-NEXT:    sw ra, 16(sp) # 4-byte Folded Spill
 ; ILP32E-FPELIM-NEXT:    .cfi_offset ra, -4
-; ILP32E-FPELIM-NEXT:    li a0, 8
-; ILP32E-FPELIM-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 8(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 7
-; ILP32E-FPELIM-NEXT:    sw a0, 4(sp)
-; ILP32E-FPELIM-NEXT:    li a4, 6
+; ILP32E-FPELIM-NEXT:    li a4, 8
+; ILP32E-FPELIM-NEXT:    li a6, 7
+; ILP32E-FPELIM-NEXT:    li a7, 6
 ; ILP32E-FPELIM-NEXT:    li a0, 1
 ; ILP32E-FPELIM-NEXT:    li a1, 2
 ; ILP32E-FPELIM-NEXT:    li a2, 3
 ; ILP32E-FPELIM-NEXT:    li a3, 4
 ; ILP32E-FPELIM-NEXT:    li a5, 5
-; ILP32E-FPELIM-NEXT:    sw a4, 0(sp)
+; ILP32E-FPELIM-NEXT:    sw a7, 0(sp)
+; ILP32E-FPELIM-NEXT:    sw a6, 4(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 12(sp)
 ; ILP32E-FPELIM-NEXT:    li a4, 0
 ; ILP32E-FPELIM-NEXT:    call callee_many_scalars
 ; ILP32E-FPELIM-NEXT:    lw ra, 16(sp) # 4-byte Folded Reload
@@ -1191,18 +1191,18 @@ define i32 @caller_many_scalars() {
 ; ILP32E-WITHFP-NEXT:    .cfi_offset s0, -8
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 24
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
-; ILP32E-WITHFP-NEXT:    li a0, 8
-; ILP32E-WITHFP-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 8(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 7
-; ILP32E-WITHFP-NEXT:    sw a0, 4(sp)
-; ILP32E-WITHFP-NEXT:    li a4, 6
+; ILP32E-WITHFP-NEXT:    li a4, 8
+; ILP32E-WITHFP-NEXT:    li a6, 7
+; ILP32E-WITHFP-NEXT:    li a7, 6
 ; ILP32E-WITHFP-NEXT:    li a0, 1
 ; ILP32E-WITHFP-NEXT:    li a1, 2
 ; ILP32E-WITHFP-NEXT:    li a2, 3
 ; ILP32E-WITHFP-NEXT:    li a3, 4
 ; ILP32E-WITHFP-NEXT:    li a5, 5
-; ILP32E-WITHFP-NEXT:    sw a4, 0(sp)
+; ILP32E-WITHFP-NEXT:    sw a7, 0(sp)
+; ILP32E-WITHFP-NEXT:    sw a6, 4(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 8(sp)
+; ILP32E-WITHFP-NEXT:    sw a4, 12(sp)
 ; ILP32E-WITHFP-NEXT:    li a4, 0
 ; ILP32E-WITHFP-NEXT:    call callee_many_scalars
 ; ILP32E-WITHFP-NEXT:    lw ra, 20(sp) # 4-byte Folded Reload
@@ -1216,18 +1216,18 @@ define i32 @caller_many_scalars() {
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, sp, -16
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_def_cfa_offset 20
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_offset ra, -4
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 8
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 7
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 4(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a4, 6
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a4, 8
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a6, 7
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a7, 6
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 1
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 2
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 3
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 4
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a5, 5
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a4, 0(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a7, 0(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a6, 4(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a4, 12(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a4, 0
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    call callee_many_scalars
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, sp, 16
@@ -1242,18 +1242,18 @@ define i32 @caller_many_scalars() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_offset s0, -8
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi s0, sp, 24
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 8
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 7
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 4(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a4, 6
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a4, 8
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a6, 7
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a7, 6
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 1
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 2
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 3
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 4
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a5, 5
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a4, 0(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a7, 0(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a6, 4(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a4, 12(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a4, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    call callee_many_scalars
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi sp, sp, 16
@@ -1380,17 +1380,17 @@ define i32 @caller_large_scalars() {
 ; ILP32E-FPELIM-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-FPELIM-NEXT:    andi sp, sp, -16
 ; ILP32E-FPELIM-NEXT:    lui a0, 524272
-; ILP32E-FPELIM-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 8(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 4(sp)
 ; ILP32E-FPELIM-NEXT:    sw zero, 0(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 36(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 32(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 28(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 4(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a0, 12(sp)
 ; ILP32E-FPELIM-NEXT:    li a2, 1
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 24
 ; ILP32E-FPELIM-NEXT:    mv a1, sp
 ; ILP32E-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 28(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 32(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 36(sp)
 ; ILP32E-FPELIM-NEXT:    call callee_large_scalars
 ; ILP32E-FPELIM-NEXT:    addi sp, s0, -48
 ; ILP32E-FPELIM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
@@ -1410,17 +1410,17 @@ define i32 @caller_large_scalars() {
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-NEXT:    andi sp, sp, -16
 ; ILP32E-WITHFP-NEXT:    lui a0, 524272
-; ILP32E-WITHFP-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 8(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 4(sp)
 ; ILP32E-WITHFP-NEXT:    sw zero, 0(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 36(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 32(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 28(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 4(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 8(sp)
+; ILP32E-WITHFP-NEXT:    sw a0, 12(sp)
 ; ILP32E-WITHFP-NEXT:    li a2, 1
 ; ILP32E-WITHFP-NEXT:    addi a0, sp, 24
 ; ILP32E-WITHFP-NEXT:    mv a1, sp
 ; ILP32E-WITHFP-NEXT:    sw a2, 24(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 28(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 32(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 36(sp)
 ; ILP32E-WITHFP-NEXT:    call callee_large_scalars
 ; ILP32E-WITHFP-NEXT:    addi sp, s0, -48
 ; ILP32E-WITHFP-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
@@ -1439,17 +1439,17 @@ define i32 @caller_large_scalars() {
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    andi sp, sp, -16
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 524272
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 0(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 36(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 32(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 28(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 1
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, sp, 24
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    mv a1, sp
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a2, 24(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 28(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 32(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 36(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    call callee_large_scalars
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, s0, -48
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, sp, 40
@@ -1466,17 +1466,17 @@ define i32 @caller_large_scalars() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    andi sp, sp, -16
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 524272
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 0(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 36(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 32(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 28(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 4(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 8(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 1
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, sp, 24
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    mv a1, sp
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a2, 24(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 28(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 32(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 36(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    call callee_large_scalars
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi sp, s0, -48
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi sp, sp, 40
@@ -1613,29 +1613,29 @@ define i32 @caller_large_scalars_exhausted_regs() {
 ; ILP32E-FPELIM-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-FPELIM-NEXT:    andi sp, sp, -16
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 16
+; ILP32E-FPELIM-NEXT:    li a1, 9
+; ILP32E-FPELIM-NEXT:    addi a2, sp, 40
+; ILP32E-FPELIM-NEXT:    li a3, 7
+; ILP32E-FPELIM-NEXT:    sw a3, 0(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 4(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 9
-; ILP32E-FPELIM-NEXT:    sw a0, 8(sp)
-; ILP32E-FPELIM-NEXT:    addi a0, sp, 40
-; ILP32E-FPELIM-NEXT:    sw a0, 4(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 7
-; ILP32E-FPELIM-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-NEXT:    lui a0, 524272
-; ILP32E-FPELIM-NEXT:    sw a0, 28(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 24(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 20(sp)
 ; ILP32E-FPELIM-NEXT:    sw zero, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 52(sp)
-; ILP32E-FPELIM-NEXT:    sw zero, 48(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 8
-; ILP32E-FPELIM-NEXT:    sw a0, 40(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 20(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 24(sp)
+; ILP32E-FPELIM-NEXT:    sw a0, 28(sp)
+; ILP32E-FPELIM-NEXT:    li a6, 8
 ; ILP32E-FPELIM-NEXT:    li a0, 1
 ; ILP32E-FPELIM-NEXT:    li a1, 2
 ; ILP32E-FPELIM-NEXT:    li a2, 3
 ; ILP32E-FPELIM-NEXT:    li a3, 4
 ; ILP32E-FPELIM-NEXT:    li a4, 5
 ; ILP32E-FPELIM-NEXT:    li a5, 6
+; ILP32E-FPELIM-NEXT:    sw a6, 40(sp)
 ; ILP32E-FPELIM-NEXT:    sw zero, 44(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 48(sp)
+; ILP32E-FPELIM-NEXT:    sw zero, 52(sp)
 ; ILP32E-FPELIM-NEXT:    call callee_large_scalars_exhausted_regs
 ; ILP32E-FPELIM-NEXT:    addi sp, s0, -64
 ; ILP32E-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
@@ -1655,29 +1655,29 @@ define i32 @caller_large_scalars_exhausted_regs() {
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-NEXT:    andi sp, sp, -16
 ; ILP32E-WITHFP-NEXT:    addi a0, sp, 16
+; ILP32E-WITHFP-NEXT:    li a1, 9
+; ILP32E-WITHFP-NEXT:    addi a2, sp, 40
+; ILP32E-WITHFP-NEXT:    li a3, 7
+; ILP32E-WITHFP-NEXT:    sw a3, 0(sp)
+; ILP32E-WITHFP-NEXT:    sw a2, 4(sp)
+; ILP32E-WITHFP-NEXT:    sw a1, 8(sp)
 ; ILP32E-WITHFP-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 9
-; ILP32E-WITHFP-NEXT:    sw a0, 8(sp)
-; ILP32E-WITHFP-NEXT:    addi a0, sp, 40
-; ILP32E-WITHFP-NEXT:    sw a0, 4(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 7
-; ILP32E-WITHFP-NEXT:    sw a0, 0(sp)
 ; ILP32E-WITHFP-NEXT:    lui a0, 524272
-; ILP32E-WITHFP-NEXT:    sw a0, 28(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 24(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 20(sp)
 ; ILP32E-WITHFP-NEXT:    sw zero, 16(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 52(sp)
-; ILP32E-WITHFP-NEXT:    sw zero, 48(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 8
-; ILP32E-WITHFP-NEXT:    sw a0, 40(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 20(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 24(sp)
+; ILP32E-WITHFP-NEXT:    sw a0, 28(sp)
+; ILP32E-WITHFP-NEXT:    li a6, 8
 ; ILP32E-WITHFP-NEXT:    li a0, 1
 ; ILP32E-WITHFP-NEXT:    li a1, 2
 ; ILP32E-WITHFP-NEXT:    li a2, 3
 ; ILP32E-WITHFP-NEXT:    li a3, 4
 ; ILP32E-WITHFP-NEXT:    li a4, 5
 ; ILP32E-WITHFP-NEXT:    li a5, 6
+; ILP32E-WITHFP-NEXT:    sw a6, 40(sp)
 ; ILP32E-WITHFP-NEXT:    sw zero, 44(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 48(sp)
+; ILP32E-WITHFP-NEXT:    sw zero, 52(sp)
 ; ILP32E-WITHFP-NEXT:    call callee_large_scalars_exhausted_regs
 ; ILP32E-WITHFP-NEXT:    addi sp, s0, -64
 ; ILP32E-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
@@ -1696,29 +1696,29 @@ define i32 @caller_large_scalars_exhausted_regs() {
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    andi sp, sp, -16
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, sp, 16
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 9
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a2, sp, 40
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 7
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a3, 0(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a2, 4(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 8(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 9
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 8(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi a0, sp, 40
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 4(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 7
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a0, 524272
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 28(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 24(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 20(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 16(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 52(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 48(sp)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 8
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 40(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 20(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 24(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 28(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a6, 8
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 1
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 2
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 3
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 4
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a4, 5
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a5, 6
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a6, 40(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 44(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 48(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 52(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    call callee_large_scalars_exhausted_regs
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, s0, -64
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    addi sp, sp, 56
@@ -1735,29 +1735,29 @@ define i32 @caller_large_scalars_exhausted_regs() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    andi sp, sp, -16
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, sp, 16
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 9
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a2, sp, 40
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 7
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a3, 0(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a2, 4(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 8(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 9
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 8(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi a0, sp, 40
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 4(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 7
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 0(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a0, 524272
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 28(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 24(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 20(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 16(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 52(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 48(sp)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 8
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 40(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 20(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 24(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, 28(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a6, 8
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 1
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 2
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 3
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 4
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a4, 5
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a5, 6
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a6, 40(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 44(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 48(sp)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 52(sp)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    call callee_large_scalars_exhausted_regs
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi sp, s0, -64
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi sp, sp, 56
@@ -2034,12 +2034,12 @@ define i32 @caller_large_struct() {
 ; ILP32E-FPELIM-NEXT:    sw ra, 32(sp) # 4-byte Folded Spill
 ; ILP32E-FPELIM-NEXT:    .cfi_offset ra, -4
 ; ILP32E-FPELIM-NEXT:    li a0, 1
-; ILP32E-FPELIM-NEXT:    sw a0, 16(sp)
 ; ILP32E-FPELIM-NEXT:    li a1, 2
-; ILP32E-FPELIM-NEXT:    sw a1, 20(sp)
 ; ILP32E-FPELIM-NEXT:    li a2, 3
-; ILP32E-FPELIM-NEXT:    sw a2, 24(sp)
 ; ILP32E-FPELIM-NEXT:    li a3, 4
+; ILP32E-FPELIM-NEXT:    sw a0, 16(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 24(sp)
 ; ILP32E-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-NEXT:    sw a1, 4(sp)
@@ -2062,12 +2062,12 @@ define i32 @caller_large_struct() {
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 40
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-NEXT:    li a0, 1
-; ILP32E-WITHFP-NEXT:    sw a0, -24(s0)
 ; ILP32E-WITHFP-NEXT:    li a1, 2
-; ILP32E-WITHFP-NEXT:    sw a1, -20(s0)
 ; ILP32E-WITHFP-NEXT:    li a2, 3
-; ILP32E-WITHFP-NEXT:    sw a2, -16(s0)
 ; ILP32E-WITHFP-NEXT:    li a3, 4
+; ILP32E-WITHFP-NEXT:    sw a0, -24(s0)
+; ILP32E-WITHFP-NEXT:    sw a1, -20(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, -16(s0)
 ; ILP32E-WITHFP-NEXT:    sw a3, -12(s0)
 ; ILP32E-WITHFP-NEXT:    sw a0, -40(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, -36(s0)
@@ -2087,12 +2087,12 @@ define i32 @caller_large_struct() {
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_def_cfa_offset 36
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    .cfi_offset ra, -4
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a0, 1
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 16(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 2
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 20(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 3
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a2, 24(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 4
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 16(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 20(sp)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a2, 24(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a3, 28(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 4(sp)
@@ -2113,12 +2113,12 @@ define i32 @caller_large_struct() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi s0, sp, 40
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a0, 1
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, -24(s0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 2
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, -20(s0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 3
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a2, -16(s0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 4
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, -24(s0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, -20(s0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a2, -16(s0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a3, -12(s0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a0, -40(s0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, -36(s0)
@@ -2252,10 +2252,10 @@ define fp128 @callee_large_scalar_ret() {
 ; ILP32E-FPELIM-LABEL: callee_large_scalar_ret:
 ; ILP32E-FPELIM:       # %bb.0:
 ; ILP32E-FPELIM-NEXT:    lui a1, 524272
-; ILP32E-FPELIM-NEXT:    sw a1, 12(a0)
-; ILP32E-FPELIM-NEXT:    sw zero, 8(a0)
-; ILP32E-FPELIM-NEXT:    sw zero, 4(a0)
 ; ILP32E-FPELIM-NEXT:    sw zero, 0(a0)
+; ILP32E-FPELIM-NEXT:    sw zero, 4(a0)
+; ILP32E-FPELIM-NEXT:    sw zero, 8(a0)
+; ILP32E-FPELIM-NEXT:    sw a1, 12(a0)
 ; ILP32E-FPELIM-NEXT:    ret
 ;
 ; ILP32E-WITHFP-LABEL: callee_large_scalar_ret:
@@ -2269,10 +2269,10 @@ define fp128 @callee_large_scalar_ret() {
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 8
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-NEXT:    lui a1, 524272
-; ILP32E-WITHFP-NEXT:    sw a1, 12(a0)
-; ILP32E-WITHFP-NEXT:    sw zero, 8(a0)
-; ILP32E-WITHFP-NEXT:    sw zero, 4(a0)
 ; ILP32E-WITHFP-NEXT:    sw zero, 0(a0)
+; ILP32E-WITHFP-NEXT:    sw zero, 4(a0)
+; ILP32E-WITHFP-NEXT:    sw zero, 8(a0)
+; ILP32E-WITHFP-NEXT:    sw a1, 12(a0)
 ; ILP32E-WITHFP-NEXT:    lw ra, 4(sp) # 4-byte Folded Reload
 ; ILP32E-WITHFP-NEXT:    lw s0, 0(sp) # 4-byte Folded Reload
 ; ILP32E-WITHFP-NEXT:    addi sp, sp, 8
@@ -2281,10 +2281,10 @@ define fp128 @callee_large_scalar_ret() {
 ; ILP32E-FPELIM-SAVE-RESTORE-LABEL: callee_large_scalar_ret:
 ; ILP32E-FPELIM-SAVE-RESTORE:       # %bb.0:
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    lui a1, 524272
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 12(a0)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 8(a0)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 4(a0)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 0(a0)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 4(a0)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw zero, 8(a0)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 12(a0)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    ret
 ;
 ; ILP32E-WITHFP-SAVE-RESTORE-LABEL: callee_large_scalar_ret:
@@ -2296,10 +2296,10 @@ define fp128 @callee_large_scalar_ret() {
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi s0, sp, 8
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    lui a1, 524272
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 12(a0)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 8(a0)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 4(a0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 0(a0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 4(a0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw zero, 8(a0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 12(a0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    tail __riscv_restore_1
   ret fp128 0xL00000000000000007FFF000000000000
 }
@@ -2384,13 +2384,13 @@ define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result
 ; ILP32E-FPELIM-LABEL: callee_large_struct_ret:
 ; ILP32E-FPELIM:       # %bb.0:
 ; ILP32E-FPELIM-NEXT:    li a1, 1
+; ILP32E-FPELIM-NEXT:    li a2, 2
+; ILP32E-FPELIM-NEXT:    li a3, 3
+; ILP32E-FPELIM-NEXT:    li a4, 4
 ; ILP32E-FPELIM-NEXT:    sw a1, 0(a0)
-; ILP32E-FPELIM-NEXT:    li a1, 2
-; ILP32E-FPELIM-NEXT:    sw a1, 4(a0)
-; ILP32E-FPELIM-NEXT:    li a1, 3
-; ILP32E-FPELIM-NEXT:    sw a1, 8(a0)
-; ILP32E-FPELIM-NEXT:    li a1, 4
-; ILP32E-FPELIM-NEXT:    sw a1, 12(a0)
+; ILP32E-FPELIM-NEXT:    sw a2, 4(a0)
+; ILP32E-FPELIM-NEXT:    sw a3, 8(a0)
+; ILP32E-FPELIM-NEXT:    sw a4, 12(a0)
 ; ILP32E-FPELIM-NEXT:    ret
 ;
 ; ILP32E-WITHFP-LABEL: callee_large_struct_ret:
@@ -2404,13 +2404,13 @@ define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 8
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-NEXT:    li a1, 1
+; ILP32E-WITHFP-NEXT:    li a2, 2
+; ILP32E-WITHFP-NEXT:    li a3, 3
+; ILP32E-WITHFP-NEXT:    li a4, 4
 ; ILP32E-WITHFP-NEXT:    sw a1, 0(a0)
-; ILP32E-WITHFP-NEXT:    li a1, 2
-; ILP32E-WITHFP-NEXT:    sw a1, 4(a0)
-; ILP32E-WITHFP-NEXT:    li a1, 3
-; ILP32E-WITHFP-NEXT:    sw a1, 8(a0)
-; ILP32E-WITHFP-NEXT:    li a1, 4
-; ILP32E-WITHFP-NEXT:    sw a1, 12(a0)
+; ILP32E-WITHFP-NEXT:    sw a2, 4(a0)
+; ILP32E-WITHFP-NEXT:    sw a3, 8(a0)
+; ILP32E-WITHFP-NEXT:    sw a4, 12(a0)
 ; ILP32E-WITHFP-NEXT:    lw ra, 4(sp) # 4-byte Folded Reload
 ; ILP32E-WITHFP-NEXT:    lw s0, 0(sp) # 4-byte Folded Reload
 ; ILP32E-WITHFP-NEXT:    addi sp, sp, 8
@@ -2419,13 +2419,13 @@ define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result
 ; ILP32E-FPELIM-SAVE-RESTORE-LABEL: callee_large_struct_ret:
 ; ILP32E-FPELIM-SAVE-RESTORE:       # %bb.0:
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 1
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a2, 2
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a3, 3
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a4, 4
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 0(a0)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 2
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 4(a0)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 3
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 8(a0)
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    li a1, 4
-; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a1, 12(a0)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a2, 4(a0)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a3, 8(a0)
+; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    sw a4, 12(a0)
 ; ILP32E-FPELIM-SAVE-RESTORE-NEXT:    ret
 ;
 ; ILP32E-WITHFP-SAVE-RESTORE-LABEL: callee_large_struct_ret:
@@ -2437,13 +2437,13 @@ define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    addi s0, sp, 8
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    .cfi_def_cfa s0, 0
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 1
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a2, 2
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a3, 3
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a4, 4
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 0(a0)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 2
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 4(a0)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 3
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 8(a0)
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    li a1, 4
-; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a1, 12(a0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a2, 4(a0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a3, 8(a0)
+; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    sw a4, 12(a0)
 ; ILP32E-WITHFP-SAVE-RESTORE-NEXT:    tail __riscv_restore_1
   %a = getelementptr inbounds %struct.large, ptr %agg.result, i32 0, i32 0
   store i32 1, ptr %a, align 4
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
index a0e1b002b7260dc9c9987d42646d56d58de04f69..cbd2cef981d71f2d918f11542757aa644a80bc48 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -82,8 +82,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
 ; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    li a0, 8
-; RV64I-NEXT:    sd a0, 8(sp)
+; RV64I-NEXT:    li a4, 8
 ; RV64I-NEXT:    li a0, 1
 ; RV64I-NEXT:    li a1, 2
 ; RV64I-NEXT:    li a2, 3
@@ -92,6 +91,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV64I-NEXT:    li a6, 6
 ; RV64I-NEXT:    li a7, 7
 ; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    sd a4, 8(sp)
 ; RV64I-NEXT:    li a4, 0
 ; RV64I-NEXT:    call callee_many_scalars
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -133,18 +133,18 @@ define i64 @caller_large_scalars() nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -80
 ; RV64I-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    li a0, 2
 ; RV64I-NEXT:    sd a0, 0(sp)
-; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    sd zero, 48(sp)
-; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    li a2, 1
 ; RV64I-NEXT:    addi a0, sp, 32
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    sd a2, 32(sp)
+; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    sd zero, 48(sp)
+; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    call callee_large_scalars
 ; RV64I-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 80
@@ -189,18 +189,15 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-NEXT:    addi sp, sp, -96
 ; RV64I-NEXT:    sd ra, 88(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, sp, 16
+; RV64I-NEXT:    li a1, 9
+; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    sd a0, 8(sp)
-; RV64I-NEXT:    li a0, 9
-; RV64I-NEXT:    sd a0, 0(sp)
-; RV64I-NEXT:    sd zero, 40(sp)
-; RV64I-NEXT:    sd zero, 32(sp)
-; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    li a0, 10
 ; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    sd zero, 72(sp)
-; RV64I-NEXT:    sd zero, 64(sp)
-; RV64I-NEXT:    li a0, 8
-; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
+; RV64I-NEXT:    sd zero, 32(sp)
+; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    li t0, 8
 ; RV64I-NEXT:    li a0, 1
 ; RV64I-NEXT:    li a1, 2
 ; RV64I-NEXT:    li a2, 3
@@ -209,7 +206,10 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-NEXT:    li a5, 6
 ; RV64I-NEXT:    li a6, 7
 ; RV64I-NEXT:    addi a7, sp, 48
+; RV64I-NEXT:    sd t0, 48(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
+; RV64I-NEXT:    sd zero, 64(sp)
+; RV64I-NEXT:    sd zero, 72(sp)
 ; RV64I-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV64I-NEXT:    ld ra, 88(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 96
@@ -293,12 +293,12 @@ define i64 @caller_large_struct() nounwind {
 ; RV64I-NEXT:    addi sp, sp, -80
 ; RV64I-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    li a1, 2
-; RV64I-NEXT:    sd a1, 48(sp)
 ; RV64I-NEXT:    li a2, 3
-; RV64I-NEXT:    sd a2, 56(sp)
 ; RV64I-NEXT:    li a3, 4
+; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    sd a1, 48(sp)
+; RV64I-NEXT:    sd a2, 56(sp)
 ; RV64I-NEXT:    sd a3, 64(sp)
 ; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    sd a1, 16(sp)
@@ -357,15 +357,12 @@ define void @caller_aligned_stack() nounwind {
 ; RV64I-NEXT:    addi sp, sp, -64
 ; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    li a0, 12
+; RV64I-NEXT:    li a1, 11
+; RV64I-NEXT:    sd a1, 40(sp)
 ; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    li a0, 11
-; RV64I-NEXT:    sd a0, 40(sp)
-; RV64I-NEXT:    li a0, 10
-; RV64I-NEXT:    sd a0, 32(sp)
-; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    li a0, 9
-; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    li a6, 8
+; RV64I-NEXT:    li a6, 10
+; RV64I-NEXT:    li t0, 9
+; RV64I-NEXT:    li t1, 8
 ; RV64I-NEXT:    li a0, 1
 ; RV64I-NEXT:    li a1, 2
 ; RV64I-NEXT:    li a2, 3
@@ -373,7 +370,10 @@ define void @caller_aligned_stack() nounwind {
 ; RV64I-NEXT:    li a4, 5
 ; RV64I-NEXT:    li a5, 6
 ; RV64I-NEXT:    li a7, 7
-; RV64I-NEXT:    sd a6, 0(sp)
+; RV64I-NEXT:    sd t1, 0(sp)
+; RV64I-NEXT:    sd t0, 16(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
+; RV64I-NEXT:    sd a6, 32(sp)
 ; RV64I-NEXT:    li a6, 0
 ; RV64I-NEXT:    call callee_aligned_stack
 ; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
@@ -449,12 +449,12 @@ define i256 @callee_large_scalar_ret() nounwind {
 ; RV64I-LABEL: callee_large_scalar_ret:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    li a1, -1
-; RV64I-NEXT:    sd a1, 24(a0)
-; RV64I-NEXT:    sd a1, 16(a0)
+; RV64I-NEXT:    lui a2, 1018435
+; RV64I-NEXT:    addiw a2, a2, 747
+; RV64I-NEXT:    sd a2, 0(a0)
 ; RV64I-NEXT:    sd a1, 8(a0)
-; RV64I-NEXT:    lui a1, 1018435
-; RV64I-NEXT:    addiw a1, a1, 747
-; RV64I-NEXT:    sd a1, 0(a0)
+; RV64I-NEXT:    sd a1, 16(a0)
+; RV64I-NEXT:    sd a1, 24(a0)
 ; RV64I-NEXT:    ret
   ret i256 -123456789
 }
@@ -478,18 +478,18 @@ define void @caller_large_scalar_ret() nounwind {
 define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result) nounwind {
 ; RV64I-LABEL: callee_large_struct_ret:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sw zero, 4(a0)
 ; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    li a2, 2
 ; RV64I-NEXT:    sw a1, 0(a0)
+; RV64I-NEXT:    sw zero, 4(a0)
+; RV64I-NEXT:    sw a2, 8(a0)
 ; RV64I-NEXT:    sw zero, 12(a0)
-; RV64I-NEXT:    li a1, 2
-; RV64I-NEXT:    sw a1, 8(a0)
-; RV64I-NEXT:    sw zero, 20(a0)
 ; RV64I-NEXT:    li a1, 3
+; RV64I-NEXT:    li a2, 4
 ; RV64I-NEXT:    sw a1, 16(a0)
+; RV64I-NEXT:    sw zero, 20(a0)
+; RV64I-NEXT:    sw a2, 24(a0)
 ; RV64I-NEXT:    sw zero, 28(a0)
-; RV64I-NEXT:    li a1, 4
-; RV64I-NEXT:    sw a1, 24(a0)
 ; RV64I-NEXT:    ret
   store i64 1, ptr %agg.result, align 4
   %b = getelementptr inbounds %struct.large, ptr %agg.result, i64 0, i32 1
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64e.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64e.ll
index bb2fd5934025122bd61d9510d0fff0c20f037b82..985135a086e24776040d142cdc10f9385dbb72df 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64e.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64e.ll
@@ -114,14 +114,14 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-LP64E-FPELIM-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; RV64I-LP64E-FPELIM-NEXT:    addi s0, sp, 48
 ; RV64I-LP64E-FPELIM-NEXT:    andi sp, sp, -16
-; RV64I-LP64E-FPELIM-NEXT:    lui a0, 264704
-; RV64I-LP64E-FPELIM-NEXT:    sd a0, 16(sp)
-; RV64I-LP64E-FPELIM-NEXT:    sd zero, 8(sp)
-; RV64I-LP64E-FPELIM-NEXT:    li a1, 4
+; RV64I-LP64E-FPELIM-NEXT:    lui a1, 264704
+; RV64I-LP64E-FPELIM-NEXT:    li a3, 4
 ; RV64I-LP64E-FPELIM-NEXT:    li a0, 1
 ; RV64I-LP64E-FPELIM-NEXT:    li a2, 2
 ; RV64I-LP64E-FPELIM-NEXT:    li a4, 3
-; RV64I-LP64E-FPELIM-NEXT:    sd a1, 0(sp)
+; RV64I-LP64E-FPELIM-NEXT:    sd a3, 0(sp)
+; RV64I-LP64E-FPELIM-NEXT:    sd zero, 8(sp)
+; RV64I-LP64E-FPELIM-NEXT:    sd a1, 16(sp)
 ; RV64I-LP64E-FPELIM-NEXT:    li a1, 0
 ; RV64I-LP64E-FPELIM-NEXT:    li a3, 0
 ; RV64I-LP64E-FPELIM-NEXT:    li a5, 0
@@ -139,14 +139,14 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-LP64E-WITHFP-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; RV64I-LP64E-WITHFP-NEXT:    addi s0, sp, 48
 ; RV64I-LP64E-WITHFP-NEXT:    andi sp, sp, -16
-; RV64I-LP64E-WITHFP-NEXT:    lui a0, 264704
-; RV64I-LP64E-WITHFP-NEXT:    sd a0, 16(sp)
-; RV64I-LP64E-WITHFP-NEXT:    sd zero, 8(sp)
-; RV64I-LP64E-WITHFP-NEXT:    li a1, 4
+; RV64I-LP64E-WITHFP-NEXT:    lui a1, 264704
+; RV64I-LP64E-WITHFP-NEXT:    li a3, 4
 ; RV64I-LP64E-WITHFP-NEXT:    li a0, 1
 ; RV64I-LP64E-WITHFP-NEXT:    li a2, 2
 ; RV64I-LP64E-WITHFP-NEXT:    li a4, 3
-; RV64I-LP64E-WITHFP-NEXT:    sd a1, 0(sp)
+; RV64I-LP64E-WITHFP-NEXT:    sd a3, 0(sp)
+; RV64I-LP64E-WITHFP-NEXT:    sd zero, 8(sp)
+; RV64I-LP64E-WITHFP-NEXT:    sd a1, 16(sp)
 ; RV64I-LP64E-WITHFP-NEXT:    li a1, 0
 ; RV64I-LP64E-WITHFP-NEXT:    li a3, 0
 ; RV64I-LP64E-WITHFP-NEXT:    li a5, 0
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
index 37d9eb6990b0e497890df5426ba7acaa70abfea7..eaba1acffa0547136ef366ff6b5106ff0fd63a00 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
@@ -32,13 +32,14 @@ define float @caller_onstack_f32_noop(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IF-NEXT:    sw a0, 4(sp)
-; RV32IF-NEXT:    lui a1, 264704
+; RV32IF-NEXT:    mv a1, a0
+; RV32IF-NEXT:    lui a3, 264704
 ; RV32IF-NEXT:    li a0, 1
 ; RV32IF-NEXT:    li a2, 2
 ; RV32IF-NEXT:    li a4, 3
 ; RV32IF-NEXT:    li a6, 4
-; RV32IF-NEXT:    sw a1, 0(sp)
+; RV32IF-NEXT:    sw a3, 0(sp)
+; RV32IF-NEXT:    sw a1, 4(sp)
 ; RV32IF-NEXT:    li a1, 0
 ; RV32IF-NEXT:    li a3, 0
 ; RV32IF-NEXT:    li a5, 0
@@ -60,12 +61,12 @@ define float @caller_onstack_f32_fadd(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa4, a0
 ; RV32IF-NEXT:    fadd.s fa3, fa4, fa5
 ; RV32IF-NEXT:    fsub.s fa5, fa5, fa4
-; RV32IF-NEXT:    fsw fa5, 4(sp)
 ; RV32IF-NEXT:    li a0, 1
 ; RV32IF-NEXT:    li a2, 2
 ; RV32IF-NEXT:    li a4, 3
 ; RV32IF-NEXT:    li a6, 4
 ; RV32IF-NEXT:    fsw fa3, 0(sp)
+; RV32IF-NEXT:    fsw fa5, 4(sp)
 ; RV32IF-NEXT:    li a1, 0
 ; RV32IF-NEXT:    li a3, 0
 ; RV32IF-NEXT:    li a5, 0
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32e.ll b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32e.ll
index d3530a4341330d55a482bda0ac731a330b0f2197..63d4ea5fee33137630cededf7e6f97ad8bc8e3e8 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32e.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32e.ll
@@ -32,15 +32,16 @@ define float @caller_onstack_f32_noop(float %a) nounwind {
 ; RV32IF-ILP32E:       # %bb.0:
 ; RV32IF-ILP32E-NEXT:    addi sp, sp, -20
 ; RV32IF-ILP32E-NEXT:    sw ra, 16(sp) # 4-byte Folded Spill
-; RV32IF-ILP32E-NEXT:    sw a0, 12(sp)
-; RV32IF-ILP32E-NEXT:    lui a0, 264704
-; RV32IF-ILP32E-NEXT:    sw a0, 8(sp)
-; RV32IF-ILP32E-NEXT:    sw zero, 4(sp)
-; RV32IF-ILP32E-NEXT:    li a1, 4
+; RV32IF-ILP32E-NEXT:    mv a1, a0
+; RV32IF-ILP32E-NEXT:    lui a3, 264704
+; RV32IF-ILP32E-NEXT:    li a5, 4
 ; RV32IF-ILP32E-NEXT:    li a0, 1
 ; RV32IF-ILP32E-NEXT:    li a2, 2
 ; RV32IF-ILP32E-NEXT:    li a4, 3
-; RV32IF-ILP32E-NEXT:    sw a1, 0(sp)
+; RV32IF-ILP32E-NEXT:    sw a5, 0(sp)
+; RV32IF-ILP32E-NEXT:    sw zero, 4(sp)
+; RV32IF-ILP32E-NEXT:    sw a3, 8(sp)
+; RV32IF-ILP32E-NEXT:    sw a1, 12(sp)
 ; RV32IF-ILP32E-NEXT:    li a1, 0
 ; RV32IF-ILP32E-NEXT:    li a3, 0
 ; RV32IF-ILP32E-NEXT:    li a5, 0
@@ -61,14 +62,14 @@ define float @caller_onstack_f32_fadd(float %a, float %b) nounwind {
 ; RV32IF-ILP32E-NEXT:    fmv.w.x fa4, a0
 ; RV32IF-ILP32E-NEXT:    fadd.s fa3, fa4, fa5
 ; RV32IF-ILP32E-NEXT:    fsub.s fa5, fa5, fa4
-; RV32IF-ILP32E-NEXT:    sw zero, 4(sp)
-; RV32IF-ILP32E-NEXT:    li a0, 4
-; RV32IF-ILP32E-NEXT:    sw a0, 0(sp)
-; RV32IF-ILP32E-NEXT:    fsw fa5, 12(sp)
+; RV32IF-ILP32E-NEXT:    li a1, 4
 ; RV32IF-ILP32E-NEXT:    li a0, 1
 ; RV32IF-ILP32E-NEXT:    li a2, 2
 ; RV32IF-ILP32E-NEXT:    li a4, 3
+; RV32IF-ILP32E-NEXT:    sw a1, 0(sp)
+; RV32IF-ILP32E-NEXT:    sw zero, 4(sp)
 ; RV32IF-ILP32E-NEXT:    fsw fa3, 8(sp)
+; RV32IF-ILP32E-NEXT:    fsw fa5, 12(sp)
 ; RV32IF-ILP32E-NEXT:    li a1, 0
 ; RV32IF-ILP32E-NEXT:    li a3, 0
 ; RV32IF-ILP32E-NEXT:    li a5, 0
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll b/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll
index 8ebe9b6301c44df01c355d289e33d4cbb63807ae..4153cad1ae88148574f645b7e005de17f0828a70 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-vector-float.ll
@@ -41,10 +41,10 @@ define <4 x float> @callee_v4f32(<4 x float> %x, <4 x float> %y) {
 ; RV64-NEXT:    fadd.s fa2, fa1, fa2
 ; RV64-NEXT:    fadd.s fa4, fa3, fa4
 ; RV64-NEXT:    fadd.s fa5, fa5, ft1
-; RV64-NEXT:    fsw fa5, 12(a0)
-; RV64-NEXT:    fsw fa4, 8(a0)
-; RV64-NEXT:    fsw fa2, 4(a0)
 ; RV64-NEXT:    fsw fa0, 0(a0)
+; RV64-NEXT:    fsw fa2, 4(a0)
+; RV64-NEXT:    fsw fa4, 8(a0)
+; RV64-NEXT:    fsw fa5, 12(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV64LP64F-LABEL: callee_v4f32:
@@ -53,10 +53,10 @@ define <4 x float> @callee_v4f32(<4 x float> %x, <4 x float> %y) {
 ; RV64LP64F-NEXT:    fadd.s fa5, fa1, fa5
 ; RV64LP64F-NEXT:    fadd.s fa2, fa2, fa6
 ; RV64LP64F-NEXT:    fadd.s fa3, fa3, fa7
-; RV64LP64F-NEXT:    fsw fa3, 12(a0)
-; RV64LP64F-NEXT:    fsw fa2, 8(a0)
-; RV64LP64F-NEXT:    fsw fa5, 4(a0)
 ; RV64LP64F-NEXT:    fsw fa4, 0(a0)
+; RV64LP64F-NEXT:    fsw fa5, 4(a0)
+; RV64LP64F-NEXT:    fsw fa2, 8(a0)
+; RV64LP64F-NEXT:    fsw fa3, 12(a0)
 ; RV64LP64F-NEXT:    ret
   %z = fadd <4 x float> %x, %y
   ret <4 x float> %z
diff --git a/llvm/test/CodeGen/RISCV/calls.ll b/llvm/test/CodeGen/RISCV/calls.ll
index f18bbb4ed84ee3e1ec98dc57e4ae38a6e89d7e3a..cf0e625f3c6c7813bd5d8adc8da4a81e8fd9c61e 100644
--- a/llvm/test/CodeGen/RISCV/calls.ll
+++ b/llvm/test/CodeGen/RISCV/calls.ll
@@ -566,8 +566,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    mv s0, a0
-; CHECK-NEXT:    sw a0, 4(sp)
 ; CHECK-NEXT:    sw a0, 0(sp)
+; CHECK-NEXT:    sw a0, 4(sp)
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:    mv a3, a0
@@ -588,8 +588,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
-; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    sd a0, 0(sp)
+; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    mv a1, a0
 ; RV64I-NEXT:    mv a2, a0
 ; RV64I-NEXT:    mv a3, a0
@@ -610,8 +610,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV64I-SMALL-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64I-SMALL-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64I-SMALL-NEXT:    mv s0, a0
-; RV64I-SMALL-NEXT:    sd a0, 8(sp)
 ; RV64I-SMALL-NEXT:    sd a0, 0(sp)
+; RV64I-SMALL-NEXT:    sd a0, 8(sp)
 ; RV64I-SMALL-NEXT:    mv a1, a0
 ; RV64I-SMALL-NEXT:    mv a2, a0
 ; RV64I-SMALL-NEXT:    mv a3, a0
@@ -632,8 +632,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV64I-MEDIUM-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64I-MEDIUM-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64I-MEDIUM-NEXT:    mv s0, a0
-; RV64I-MEDIUM-NEXT:    sd a0, 8(sp)
 ; RV64I-MEDIUM-NEXT:    sd a0, 0(sp)
+; RV64I-MEDIUM-NEXT:    sd a0, 8(sp)
 ; RV64I-MEDIUM-NEXT:    mv a1, a0
 ; RV64I-MEDIUM-NEXT:    mv a2, a0
 ; RV64I-MEDIUM-NEXT:    mv a3, a0
@@ -657,8 +657,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV64I-LARGE-NEXT:  .Lpcrel_hi4:
 ; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI8_0)
 ; RV64I-LARGE-NEXT:    ld t1, %pcrel_lo(.Lpcrel_hi4)(a0)
-; RV64I-LARGE-NEXT:    sd s0, 8(sp)
 ; RV64I-LARGE-NEXT:    sd s0, 0(sp)
+; RV64I-LARGE-NEXT:    sd s0, 8(sp)
 ; RV64I-LARGE-NEXT:    mv a0, s0
 ; RV64I-LARGE-NEXT:    mv a1, s0
 ; RV64I-LARGE-NEXT:    mv a2, s0
@@ -684,8 +684,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi4:
 ; RV64I-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI8_0)
 ; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi4)(a0)
-; RV64I-LARGE-ZICFILP-NEXT:    sd s0, 8(sp)
 ; RV64I-LARGE-ZICFILP-NEXT:    sd s0, 0(sp)
+; RV64I-LARGE-ZICFILP-NEXT:    sd s0, 8(sp)
 ; RV64I-LARGE-ZICFILP-NEXT:    mv a0, s0
 ; RV64I-LARGE-ZICFILP-NEXT:    mv a1, s0
 ; RV64I-LARGE-ZICFILP-NEXT:    mv a2, s0
@@ -751,8 +751,8 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw a0, 4(sp)
 ; CHECK-NEXT:    sw a0, 0(sp)
+; CHECK-NEXT:    sw a0, 4(sp)
 ; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:    mv a3, a0
@@ -769,8 +769,8 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
 ; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    sd a0, 0(sp)
+; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    mv a1, a0
 ; RV64I-NEXT:    mv a2, a0
 ; RV64I-NEXT:    mv a3, a0
@@ -787,8 +787,8 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV64I-SMALL:       # %bb.0:
 ; RV64I-SMALL-NEXT:    addi sp, sp, -32
 ; RV64I-SMALL-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-SMALL-NEXT:    sd a0, 8(sp)
 ; RV64I-SMALL-NEXT:    sd a0, 0(sp)
+; RV64I-SMALL-NEXT:    sd a0, 8(sp)
 ; RV64I-SMALL-NEXT:    mv a1, a0
 ; RV64I-SMALL-NEXT:    mv a2, a0
 ; RV64I-SMALL-NEXT:    mv a3, a0
@@ -805,8 +805,8 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV64I-MEDIUM:       # %bb.0:
 ; RV64I-MEDIUM-NEXT:    addi sp, sp, -32
 ; RV64I-MEDIUM-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-MEDIUM-NEXT:    sd a0, 8(sp)
 ; RV64I-MEDIUM-NEXT:    sd a0, 0(sp)
+; RV64I-MEDIUM-NEXT:    sd a0, 8(sp)
 ; RV64I-MEDIUM-NEXT:    mv a1, a0
 ; RV64I-MEDIUM-NEXT:    mv a2, a0
 ; RV64I-MEDIUM-NEXT:    mv a3, a0
@@ -826,8 +826,8 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV64I-LARGE-NEXT:  .Lpcrel_hi5:
 ; RV64I-LARGE-NEXT:    auipc a1, %pcrel_hi(.LCPI10_0)
 ; RV64I-LARGE-NEXT:    ld t1, %pcrel_lo(.Lpcrel_hi5)(a1)
-; RV64I-LARGE-NEXT:    sd a0, 8(sp)
 ; RV64I-LARGE-NEXT:    sd a0, 0(sp)
+; RV64I-LARGE-NEXT:    sd a0, 8(sp)
 ; RV64I-LARGE-NEXT:    mv a1, a0
 ; RV64I-LARGE-NEXT:    mv a2, a0
 ; RV64I-LARGE-NEXT:    mv a3, a0
@@ -848,8 +848,8 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi5:
 ; RV64I-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI10_0)
 ; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi5)(a1)
-; RV64I-LARGE-ZICFILP-NEXT:    sd a0, 8(sp)
 ; RV64I-LARGE-ZICFILP-NEXT:    sd a0, 0(sp)
+; RV64I-LARGE-ZICFILP-NEXT:    sd a0, 8(sp)
 ; RV64I-LARGE-ZICFILP-NEXT:    mv a1, a0
 ; RV64I-LARGE-ZICFILP-NEXT:    mv a2, a0
 ; RV64I-LARGE-ZICFILP-NEXT:    mv a3, a0
diff --git a/llvm/test/CodeGen/RISCV/double-calling-conv.ll b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
index 57aaa4c9f74e46b5a3337637248d506faedae1c6..b9e80dccd97b9aca906bb2cbeb0f5a0f1ea715cf 100644
--- a/llvm/test/CodeGen/RISCV/double-calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
@@ -181,19 +181,19 @@ define double @caller_double_stack() nounwind {
 ; RV32IFD-NEXT:    addi sp, sp, -32
 ; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lui a0, 262510
-; RV32IFD-NEXT:    addi a0, a0, 327
-; RV32IFD-NEXT:    sw a0, 4(sp)
+; RV32IFD-NEXT:    addi a1, a0, 327
 ; RV32IFD-NEXT:    lui a0, 713032
-; RV32IFD-NEXT:    addi a1, a0, -1311
-; RV32IFD-NEXT:    sw a1, 0(sp)
+; RV32IFD-NEXT:    addi a3, a0, -1311
 ; RV32IFD-NEXT:    lui a0, 262574
-; RV32IFD-NEXT:    addi a0, a0, 327
-; RV32IFD-NEXT:    sw a0, 12(sp)
+; RV32IFD-NEXT:    addi a5, a0, 327
 ; RV32IFD-NEXT:    li a0, 1
 ; RV32IFD-NEXT:    li a2, 2
 ; RV32IFD-NEXT:    li a4, 3
 ; RV32IFD-NEXT:    li a6, 4
-; RV32IFD-NEXT:    sw a1, 8(sp)
+; RV32IFD-NEXT:    sw a3, 0(sp)
+; RV32IFD-NEXT:    sw a1, 4(sp)
+; RV32IFD-NEXT:    sw a3, 8(sp)
+; RV32IFD-NEXT:    sw a5, 12(sp)
 ; RV32IFD-NEXT:    li a1, 0
 ; RV32IFD-NEXT:    li a3, 0
 ; RV32IFD-NEXT:    li a5, 0
@@ -208,19 +208,19 @@ define double @caller_double_stack() nounwind {
 ; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
 ; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    lui a0, 262510
-; RV32IZFINXZDINX-NEXT:    addi a0, a0, 327
-; RV32IZFINXZDINX-NEXT:    sw a0, 4(sp)
+; RV32IZFINXZDINX-NEXT:    addi a1, a0, 327
 ; RV32IZFINXZDINX-NEXT:    lui a0, 713032
-; RV32IZFINXZDINX-NEXT:    addi a1, a0, -1311
-; RV32IZFINXZDINX-NEXT:    sw a1, 0(sp)
+; RV32IZFINXZDINX-NEXT:    addi a3, a0, -1311
 ; RV32IZFINXZDINX-NEXT:    lui a0, 262574
-; RV32IZFINXZDINX-NEXT:    addi a0, a0, 327
-; RV32IZFINXZDINX-NEXT:    sw a0, 12(sp)
+; RV32IZFINXZDINX-NEXT:    addi a5, a0, 327
 ; RV32IZFINXZDINX-NEXT:    li a0, 1
 ; RV32IZFINXZDINX-NEXT:    li a2, 2
 ; RV32IZFINXZDINX-NEXT:    li a4, 3
 ; RV32IZFINXZDINX-NEXT:    li a6, 4
-; RV32IZFINXZDINX-NEXT:    sw a1, 8(sp)
+; RV32IZFINXZDINX-NEXT:    sw a3, 0(sp)
+; RV32IZFINXZDINX-NEXT:    sw a1, 4(sp)
+; RV32IZFINXZDINX-NEXT:    sw a3, 8(sp)
+; RV32IZFINXZDINX-NEXT:    sw a5, 12(sp)
 ; RV32IZFINXZDINX-NEXT:    li a1, 0
 ; RV32IZFINXZDINX-NEXT:    li a3, 0
 ; RV32IZFINXZDINX-NEXT:    li a5, 0
diff --git a/llvm/test/CodeGen/RISCV/double-convert-strict.ll b/llvm/test/CodeGen/RISCV/double-convert-strict.ll
index 3732978b8bd83ea2c43eac0ddcb5ce7fb287c2f9..2b1ec10fcaf17e24f9fb0d4d73946baba28a0ad1 100644
--- a/llvm/test/CodeGen/RISCV/double-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert-strict.ll
@@ -792,8 +792,8 @@ define signext i32 @fcvt_d_w_demanded_bits(i32 signext %0, ptr %1) nounwind stri
 ; RV32I-NEXT:    addi s1, a0, 1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __floatsidf
-; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -865,8 +865,8 @@ define signext i32 @fcvt_d_wu_demanded_bits(i32 signext %0, ptr %1) nounwind str
 ; RV32I-NEXT:    addi s1, a0, 1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __floatunsidf
-; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index ef2d8e7627be5484a6c0a55109e893b62c849aa5..a8b141618bbb3a7e74ff2f731fd4b39494f3c147 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -1474,8 +1474,8 @@ define signext i32 @fcvt_d_w_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV32I-NEXT:    addi s1, a0, 1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __floatsidf
-; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -1547,8 +1547,8 @@ define signext i32 @fcvt_d_wu_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV32I-NEXT:    addi s1, a0, 1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __floatunsidf
-; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/epi-trampoline.ll b/llvm/test/CodeGen/RISCV/epi-trampoline.ll
index 24022af9040f3b7bbec67d1ab4b032d31b982fb9..44f7b0a2d456fee3a9e41426c1d5e8823b6193e8 100644
--- a/llvm/test/CodeGen/RISCV/epi-trampoline.ll
+++ b/llvm/test/CodeGen/RISCV/epi-trampoline.ll
@@ -18,15 +18,15 @@ define i64 @test0(i64 %n, ptr %p) nounwind {
 ; RV64-NEXT:    lui a0, %hi(f)
 ; RV64-NEXT:    addi a0, a0, %lo(f)
 ; RV64-NEXT:    sd a0, 24(sp)
-; RV64-NEXT:    lui a0, %hi(.LCPI0_0)
-; RV64-NEXT:    ld a0, %lo(.LCPI0_0)(a0)
+; RV64-NEXT:    li a0, 919
+; RV64-NEXT:    lui a2, %hi(.LCPI0_0)
+; RV64-NEXT:    ld a2, %lo(.LCPI0_0)(a2)
+; RV64-NEXT:    lui a3, 6203
+; RV64-NEXT:    addi a3, a3, 643
+; RV64-NEXT:    sw a0, 0(sp)
+; RV64-NEXT:    sw a3, 4(sp)
+; RV64-NEXT:    sd a2, 8(sp)
 ; RV64-NEXT:    sd a1, 16(sp)
-; RV64-NEXT:    li a1, 919
-; RV64-NEXT:    sw a1, 0(sp)
-; RV64-NEXT:    sd a0, 8(sp)
-; RV64-NEXT:    lui a0, 6203
-; RV64-NEXT:    addi a0, a0, 643
-; RV64-NEXT:    sw a0, 4(sp)
 ; RV64-NEXT:    addi a1, sp, 32
 ; RV64-NEXT:    mv a0, sp
 ; RV64-NEXT:    mv s1, sp
diff --git a/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll b/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll
index e8524ce80bc16e5e2c72fd58ac0f0820adbe7033..f5b7ecfd903814de6d15a79968a837b42858eada 100644
--- a/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll
@@ -164,9 +164,9 @@ define dso_local void @foo(%struct.crypto_stream_chacha20_dolbeau_riscv_v_ECRYPT
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vadc.vim v26, v9, 0, v0
 ; CHECK-NEXT:    add a2, a2, s9
+; CHECK-NEXT:    srli a5, a2, 32
 ; CHECK-NEXT:    sw a2, 48(a0)
-; CHECK-NEXT:    srli a2, a2, 32
-; CHECK-NEXT:    sw a2, 52(a0)
+; CHECK-NEXT:    sw a5, 52(a0)
 ; CHECK-NEXT:    li a5, -2
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    li s1, 10
@@ -658,10 +658,10 @@ define dso_local void @foo(%struct.crypto_stream_chacha20_dolbeau_riscv_v_ECRYPT
 ; CHECK-NEXT:    lw a7, 132(sp)
 ; CHECK-NEXT:    lw t1, 136(sp)
 ; CHECK-NEXT:    lw t3, 140(sp)
-; CHECK-NEXT:    lw a5, 144(sp)
+; CHECK-NEXT:    lw s0, 144(sp)
 ; CHECK-NEXT:    lw s8, 148(sp)
-; CHECK-NEXT:    lw s7, 152(sp)
-; CHECK-NEXT:    lw s6, 156(sp)
+; CHECK-NEXT:    lw s6, 152(sp)
+; CHECK-NEXT:    lw s7, 156(sp)
 ; CHECK-NEXT:    lw s11, 176(sp)
 ; CHECK-NEXT:    lw a6, 180(sp)
 ; CHECK-NEXT:    lw t0, 184(sp)
@@ -677,73 +677,73 @@ define dso_local void @foo(%struct.crypto_stream_chacha20_dolbeau_riscv_v_ECRYPT
 ; CHECK-NEXT:  .LBB0_9: # %for.body5.i
 ; CHECK-NEXT:    # Parent Loop BB0_8 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    add ra, ra, a5
+; CHECK-NEXT:    add ra, ra, s0
 ; CHECK-NEXT:    xor a3, ra, s11
-; CHECK-NEXT:    srliw s1, a3, 16
+; CHECK-NEXT:    srliw a5, a3, 16
 ; CHECK-NEXT:    slli a3, a3, 16
-; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    or a3, a3, a5
 ; CHECK-NEXT:    add s10, s10, a3
-; CHECK-NEXT:    xor a5, s10, a5
+; CHECK-NEXT:    xor a5, s10, s0
 ; CHECK-NEXT:    srliw s1, a5, 20
 ; CHECK-NEXT:    slli a5, a5, 12
-; CHECK-NEXT:    or s1, s1, a5
-; CHECK-NEXT:    add ra, ra, s1
+; CHECK-NEXT:    or a5, a5, s1
+; CHECK-NEXT:    add ra, ra, a5
 ; CHECK-NEXT:    xor a3, ra, a3
-; CHECK-NEXT:    srliw s0, a3, 24
+; CHECK-NEXT:    srliw s1, a3, 24
 ; CHECK-NEXT:    slli a3, a3, 8
-; CHECK-NEXT:    or s11, a3, s0
+; CHECK-NEXT:    or s11, a3, s1
 ; CHECK-NEXT:    add s10, s10, s11
-; CHECK-NEXT:    xor a3, s10, s1
-; CHECK-NEXT:    srliw s1, a3, 25
+; CHECK-NEXT:    xor a3, s10, a5
+; CHECK-NEXT:    srliw a5, a3, 25
 ; CHECK-NEXT:    slli a3, a3, 7
-; CHECK-NEXT:    or t2, a3, s1
+; CHECK-NEXT:    or t2, a3, a5
 ; CHECK-NEXT:    add a7, a7, s8
 ; CHECK-NEXT:    xor a3, a7, a6
-; CHECK-NEXT:    srliw s1, a3, 16
+; CHECK-NEXT:    srliw a5, a3, 16
 ; CHECK-NEXT:    slli a3, a3, 16
-; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    or a3, a3, a5
 ; CHECK-NEXT:    add s9, s9, a3
-; CHECK-NEXT:    xor s1, s9, s8
-; CHECK-NEXT:    srliw s0, s1, 20
-; CHECK-NEXT:    slli s1, s1, 12
-; CHECK-NEXT:    or s1, s1, s0
-; CHECK-NEXT:    add a7, a7, s1
+; CHECK-NEXT:    xor a5, s9, s8
+; CHECK-NEXT:    srliw s1, a5, 20
+; CHECK-NEXT:    slli a5, a5, 12
+; CHECK-NEXT:    or a5, a5, s1
+; CHECK-NEXT:    add a7, a7, a5
 ; CHECK-NEXT:    xor a3, a7, a3
 ; CHECK-NEXT:    srliw a2, a3, 24
 ; CHECK-NEXT:    slli a3, a3, 8
 ; CHECK-NEXT:    or a6, a3, a2
 ; CHECK-NEXT:    add s9, s9, a6
-; CHECK-NEXT:    xor a2, s9, s1
+; CHECK-NEXT:    xor a2, s9, a5
 ; CHECK-NEXT:    srliw a3, a2, 25
 ; CHECK-NEXT:    slli a2, a2, 7
 ; CHECK-NEXT:    or a2, a2, a3
-; CHECK-NEXT:    add t1, t1, s7
-; CHECK-NEXT:    xor a3, t1, t0
-; CHECK-NEXT:    srliw s1, a3, 16
+; CHECK-NEXT:    add a5, t1, s6
+; CHECK-NEXT:    xor a3, a5, t0
+; CHECK-NEXT:    srliw a4, a3, 16
 ; CHECK-NEXT:    slli a3, a3, 16
-; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    or a3, a3, a4
 ; CHECK-NEXT:    add s4, s4, a3
-; CHECK-NEXT:    xor s1, s4, s7
-; CHECK-NEXT:    srliw a4, s1, 20
-; CHECK-NEXT:    slli s1, s1, 12
-; CHECK-NEXT:    or a4, a4, s1
-; CHECK-NEXT:    add t1, t1, a4
+; CHECK-NEXT:    xor a4, s4, s6
+; CHECK-NEXT:    srliw s0, a4, 20
+; CHECK-NEXT:    slli a4, a4, 12
+; CHECK-NEXT:    or a4, a4, s0
+; CHECK-NEXT:    add t1, a4, a5
 ; CHECK-NEXT:    xor a3, t1, a3
-; CHECK-NEXT:    srliw a5, a3, 24
+; CHECK-NEXT:    srliw s0, a3, 24
 ; CHECK-NEXT:    slli a3, a3, 8
-; CHECK-NEXT:    or t0, a3, a5
+; CHECK-NEXT:    or t0, a3, s0
 ; CHECK-NEXT:    add s4, s4, t0
 ; CHECK-NEXT:    xor a3, s4, a4
 ; CHECK-NEXT:    srliw a4, a3, 25
 ; CHECK-NEXT:    slli a3, a3, 7
 ; CHECK-NEXT:    or a4, a4, a3
-; CHECK-NEXT:    add a3, t3, s6
-; CHECK-NEXT:    xor a5, a3, s3
-; CHECK-NEXT:    srliw s0, a5, 16
-; CHECK-NEXT:    slli a5, a5, 16
+; CHECK-NEXT:    add a3, t3, s7
+; CHECK-NEXT:    xor s0, a3, s3
+; CHECK-NEXT:    srliw a5, s0, 16
+; CHECK-NEXT:    slli s0, s0, 16
 ; CHECK-NEXT:    or a5, a5, s0
 ; CHECK-NEXT:    add s5, s5, a5
-; CHECK-NEXT:    xor s0, s5, s6
+; CHECK-NEXT:    xor s0, s5, s7
 ; CHECK-NEXT:    srliw s1, s0, 20
 ; CHECK-NEXT:    slli s0, s0, 12
 ; CHECK-NEXT:    or s0, s0, s1
@@ -796,7 +796,7 @@ define dso_local void @foo(%struct.crypto_stream_chacha20_dolbeau_riscv_v_ECRYPT
 ; CHECK-NEXT:    xor a2, s5, a4
 ; CHECK-NEXT:    srliw a4, a2, 25
 ; CHECK-NEXT:    slli a2, a2, 7
-; CHECK-NEXT:    or s7, a2, a4
+; CHECK-NEXT:    or s6, a2, a4
 ; CHECK-NEXT:    add t1, t1, s0
 ; CHECK-NEXT:    xor a2, a6, t1
 ; CHECK-NEXT:    srliw a4, a2, 16
@@ -816,7 +816,7 @@ define dso_local void @foo(%struct.crypto_stream_chacha20_dolbeau_riscv_v_ECRYPT
 ; CHECK-NEXT:    xor a2, s10, a4
 ; CHECK-NEXT:    srliw a4, a2, 25
 ; CHECK-NEXT:    slli a2, a2, 7
-; CHECK-NEXT:    or s6, a2, a4
+; CHECK-NEXT:    or s7, a2, a4
 ; CHECK-NEXT:    add a3, a3, t2
 ; CHECK-NEXT:    xor a2, a3, t0
 ; CHECK-NEXT:    srliw a4, a2, 16
@@ -837,57 +837,57 @@ define dso_local void @foo(%struct.crypto_stream_chacha20_dolbeau_riscv_v_ECRYPT
 ; CHECK-NEXT:    srliw a3, a2, 25
 ; CHECK-NEXT:    slli a2, a2, 7
 ; CHECK-NEXT:    addiw t4, t4, -2
-; CHECK-NEXT:    or a5, a2, a3
+; CHECK-NEXT:    or s0, a2, a3
 ; CHECK-NEXT:    bltu t6, t4, .LBB0_9
 ; CHECK-NEXT:  # %bb.10: # %for.cond451.preheader.i
 ; CHECK-NEXT:    # in Loop: Header=BB0_8 Depth=1
 ; CHECK-NEXT:    lw a3, 0(a0)
 ; CHECK-NEXT:    lw a4, 4(a0)
-; CHECK-NEXT:    lw s1, 8(a0)
-; CHECK-NEXT:    lw s0, 12(a0)
+; CHECK-NEXT:    lw a5, 8(a0)
+; CHECK-NEXT:    lw s1, 12(a0)
 ; CHECK-NEXT:    add ra, ra, a3
 ; CHECK-NEXT:    add a7, a7, a4
-; CHECK-NEXT:    add t1, t1, s1
-; CHECK-NEXT:    sw t1, 136(sp)
-; CHECK-NEXT:    add t3, t3, s0
+; CHECK-NEXT:    add a5, a5, t1
+; CHECK-NEXT:    add t3, t3, s1
+; CHECK-NEXT:    sw ra, 128(sp)
+; CHECK-NEXT:    sw a7, 132(sp)
+; CHECK-NEXT:    sw a5, 136(sp)
 ; CHECK-NEXT:    sw t3, 140(sp)
 ; CHECK-NEXT:    lw a3, 16(a0)
 ; CHECK-NEXT:    lw a4, 20(a0)
-; CHECK-NEXT:    lw s1, 24(a0)
-; CHECK-NEXT:    lw s0, 28(a0)
-; CHECK-NEXT:    add a3, a3, a5
-; CHECK-NEXT:    sw ra, 128(sp)
-; CHECK-NEXT:    sw a7, 132(sp)
-; CHECK-NEXT:    sw a3, 144(sp)
+; CHECK-NEXT:    lw a5, 24(a0)
+; CHECK-NEXT:    lw s1, 28(a0)
+; CHECK-NEXT:    add a3, a3, s0
 ; CHECK-NEXT:    add a4, a4, s8
-; CHECK-NEXT:    sw a4, 148(sp)
+; CHECK-NEXT:    add a5, a5, s6
 ; CHECK-NEXT:    add s1, s1, s7
-; CHECK-NEXT:    sw s1, 152(sp)
-; CHECK-NEXT:    add s0, s0, s6
-; CHECK-NEXT:    sw s0, 156(sp)
+; CHECK-NEXT:    sw a3, 144(sp)
+; CHECK-NEXT:    sw a4, 148(sp)
+; CHECK-NEXT:    sw a5, 152(sp)
+; CHECK-NEXT:    sw s1, 156(sp)
 ; CHECK-NEXT:    lw a3, 32(a0)
 ; CHECK-NEXT:    lw a4, 36(a0)
 ; CHECK-NEXT:    lw a5, 40(a0)
 ; CHECK-NEXT:    lw s1, 44(a0)
 ; CHECK-NEXT:    add a3, a3, s10
-; CHECK-NEXT:    sw a3, 160(sp)
 ; CHECK-NEXT:    add a4, a4, s9
-; CHECK-NEXT:    sw a4, 164(sp)
 ; CHECK-NEXT:    add a5, a5, s4
-; CHECK-NEXT:    sw a5, 168(sp)
 ; CHECK-NEXT:    add s1, s1, s5
+; CHECK-NEXT:    sw a3, 160(sp)
+; CHECK-NEXT:    sw a4, 164(sp)
+; CHECK-NEXT:    sw a5, 168(sp)
 ; CHECK-NEXT:    sw s1, 172(sp)
 ; CHECK-NEXT:    lw t2, 48(a0)
 ; CHECK-NEXT:    lw t1, 52(a0)
 ; CHECK-NEXT:    lw a5, 56(a0)
 ; CHECK-NEXT:    lw s1, 60(a0)
 ; CHECK-NEXT:    add s11, s11, t2
-; CHECK-NEXT:    sw s11, 176(sp)
 ; CHECK-NEXT:    add a6, a6, t1
-; CHECK-NEXT:    sw a6, 180(sp)
 ; CHECK-NEXT:    add a5, a5, t0
-; CHECK-NEXT:    sw a5, 184(sp)
 ; CHECK-NEXT:    add s1, s1, s3
+; CHECK-NEXT:    sw s11, 176(sp)
+; CHECK-NEXT:    sw a6, 180(sp)
+; CHECK-NEXT:    sw a5, 184(sp)
 ; CHECK-NEXT:    sw s1, 188(sp)
 ; CHECK-NEXT:    srli a5, ra, 8
 ; CHECK-NEXT:    srli s1, ra, 16
diff --git a/llvm/test/CodeGen/RISCV/fastcc-bf16.ll b/llvm/test/CodeGen/RISCV/fastcc-bf16.ll
index 493bc63992547e23ab5a5ba6aa3961ffe73bc431..17356116081fff80599b0468fd87e65fb9dd55ed 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-bf16.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-bf16.ll
@@ -50,18 +50,18 @@ define bfloat @caller(<32 x bfloat> %A) nounwind {
 ; CHECK-NEXT:    fmv.h.x fa5, a5
 ; CHECK-NEXT:    fmv.h.x fa6, a6
 ; CHECK-NEXT:    fmv.h.x fa7, a7
-; CHECK-NEXT:    fsh fs11, 22(sp)
-; CHECK-NEXT:    fsh fs10, 20(sp)
-; CHECK-NEXT:    fsh fs9, 18(sp)
 ; CHECK-NEXT:    fsh fs8, 16(sp)
-; CHECK-NEXT:    fsh fs7, 14(sp)
-; CHECK-NEXT:    fsh fs6, 12(sp)
-; CHECK-NEXT:    fsh fs5, 10(sp)
+; CHECK-NEXT:    fsh fs9, 18(sp)
+; CHECK-NEXT:    fsh fs10, 20(sp)
+; CHECK-NEXT:    fsh fs11, 22(sp)
 ; CHECK-NEXT:    fsh fs4, 8(sp)
-; CHECK-NEXT:    fsh fs3, 6(sp)
-; CHECK-NEXT:    fsh fs2, 4(sp)
-; CHECK-NEXT:    fsh fs1, 2(sp)
+; CHECK-NEXT:    fsh fs5, 10(sp)
+; CHECK-NEXT:    fsh fs6, 12(sp)
+; CHECK-NEXT:    fsh fs7, 14(sp)
 ; CHECK-NEXT:    fsh fs0, 0(sp)
+; CHECK-NEXT:    fsh fs1, 2(sp)
+; CHECK-NEXT:    fsh fs2, 4(sp)
+; CHECK-NEXT:    fsh fs3, 6(sp)
 ; CHECK-NEXT:    call callee
 ; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/fastcc-float.ll b/llvm/test/CodeGen/RISCV/fastcc-float.ll
index 488c97d5a4506c441a33490bcfcafd6a8eaa0ab3..237a72d983de4ae130ae64709be730bfe766627b 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-float.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-float.ll
@@ -50,18 +50,18 @@ define float @caller(<32 x float> %A) nounwind {
 ; CHECK-NEXT:    fmv.w.x fa5, a5
 ; CHECK-NEXT:    fmv.w.x fa6, a6
 ; CHECK-NEXT:    fmv.w.x fa7, a7
-; CHECK-NEXT:    fsw fs11, 44(sp)
-; CHECK-NEXT:    fsw fs10, 40(sp)
-; CHECK-NEXT:    fsw fs9, 36(sp)
 ; CHECK-NEXT:    fsw fs8, 32(sp)
-; CHECK-NEXT:    fsw fs7, 28(sp)
-; CHECK-NEXT:    fsw fs6, 24(sp)
-; CHECK-NEXT:    fsw fs5, 20(sp)
+; CHECK-NEXT:    fsw fs9, 36(sp)
+; CHECK-NEXT:    fsw fs10, 40(sp)
+; CHECK-NEXT:    fsw fs11, 44(sp)
 ; CHECK-NEXT:    fsw fs4, 16(sp)
-; CHECK-NEXT:    fsw fs3, 12(sp)
-; CHECK-NEXT:    fsw fs2, 8(sp)
-; CHECK-NEXT:    fsw fs1, 4(sp)
+; CHECK-NEXT:    fsw fs5, 20(sp)
+; CHECK-NEXT:    fsw fs6, 24(sp)
+; CHECK-NEXT:    fsw fs7, 28(sp)
 ; CHECK-NEXT:    fsw fs0, 0(sp)
+; CHECK-NEXT:    fsw fs1, 4(sp)
+; CHECK-NEXT:    fsw fs2, 8(sp)
+; CHECK-NEXT:    fsw fs3, 12(sp)
 ; CHECK-NEXT:    call callee
 ; CHECK-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 64
diff --git a/llvm/test/CodeGen/RISCV/fastcc-half.ll b/llvm/test/CodeGen/RISCV/fastcc-half.ll
index 499b91b9d81c350617e169b1dc80bedc1ef1120a..bf8d4e8dcb98c4b410116e69112e5e119012244f 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-half.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-half.ll
@@ -50,18 +50,18 @@ define half @caller(<32 x half> %A) nounwind {
 ; CHECK-NEXT:    fmv.h.x fa5, a5
 ; CHECK-NEXT:    fmv.h.x fa6, a6
 ; CHECK-NEXT:    fmv.h.x fa7, a7
-; CHECK-NEXT:    fsh fs11, 22(sp)
-; CHECK-NEXT:    fsh fs10, 20(sp)
-; CHECK-NEXT:    fsh fs9, 18(sp)
 ; CHECK-NEXT:    fsh fs8, 16(sp)
-; CHECK-NEXT:    fsh fs7, 14(sp)
-; CHECK-NEXT:    fsh fs6, 12(sp)
-; CHECK-NEXT:    fsh fs5, 10(sp)
+; CHECK-NEXT:    fsh fs9, 18(sp)
+; CHECK-NEXT:    fsh fs10, 20(sp)
+; CHECK-NEXT:    fsh fs11, 22(sp)
 ; CHECK-NEXT:    fsh fs4, 8(sp)
-; CHECK-NEXT:    fsh fs3, 6(sp)
-; CHECK-NEXT:    fsh fs2, 4(sp)
-; CHECK-NEXT:    fsh fs1, 2(sp)
+; CHECK-NEXT:    fsh fs5, 10(sp)
+; CHECK-NEXT:    fsh fs6, 12(sp)
+; CHECK-NEXT:    fsh fs7, 14(sp)
 ; CHECK-NEXT:    fsh fs0, 0(sp)
+; CHECK-NEXT:    fsh fs1, 2(sp)
+; CHECK-NEXT:    fsh fs2, 4(sp)
+; CHECK-NEXT:    fsh fs3, 6(sp)
 ; CHECK-NEXT:    call callee
 ; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/fastcc-int.ll b/llvm/test/CodeGen/RISCV/fastcc-int.ll
index 75046b701b2352621c2d7114488511efb89a4d7c..3570ece269609f70b17345d0e6162a5c88b03954 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-int.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-int.ll
@@ -40,10 +40,10 @@ define i32 @caller(<16 x i32> %A) nounwind {
 ; RV32-NEXT:    lw t2, 52(a0)
 ; RV32-NEXT:    lw s0, 56(a0)
 ; RV32-NEXT:    lw a0, 60(a0)
-; RV32-NEXT:    sw a0, 12(sp)
-; RV32-NEXT:    sw s0, 8(sp)
-; RV32-NEXT:    sw t2, 4(sp)
 ; RV32-NEXT:    sw t1, 0(sp)
+; RV32-NEXT:    sw t2, 4(sp)
+; RV32-NEXT:    sw s0, 8(sp)
+; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    mv a0, t0
 ; RV32-NEXT:    call callee
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
@@ -72,10 +72,10 @@ define i32 @caller(<16 x i32> %A) nounwind {
 ; RV64-NEXT:    ld t2, 104(a0)
 ; RV64-NEXT:    ld s0, 112(a0)
 ; RV64-NEXT:    ld a0, 120(a0)
-; RV64-NEXT:    sd a0, 24(sp)
-; RV64-NEXT:    sd s0, 16(sp)
-; RV64-NEXT:    sd t2, 8(sp)
 ; RV64-NEXT:    sd t1, 0(sp)
+; RV64-NEXT:    sd t2, 8(sp)
+; RV64-NEXT:    sd s0, 16(sp)
+; RV64-NEXT:    sd a0, 24(sp)
 ; RV64-NEXT:    mv a0, t0
 ; RV64-NEXT:    call callee
 ; RV64-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
index 0eefc34ad552a98da66abc405ae7c1bea0e6be77..7523119c4ff778d1b392de37129086cd6262512f 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
@@ -289,8 +289,8 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZHINX32-NEXT:    lh t0, 124(sp)
 ; ZHINX32-NEXT:    sh t0, 52(sp) # 2-byte Folded Spill
 ; ZHINX32-NEXT:    lh t6, 128(sp)
-; ZHINX32-NEXT:    lh t5, 132(sp)
-; ZHINX32-NEXT:    lh t4, 136(sp)
+; ZHINX32-NEXT:    lh t4, 132(sp)
+; ZHINX32-NEXT:    lh t5, 136(sp)
 ; ZHINX32-NEXT:    lh s0, 140(sp)
 ; ZHINX32-NEXT:    lh s1, 144(sp)
 ; ZHINX32-NEXT:    lh s2, 148(sp)
@@ -304,30 +304,30 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZHINX32-NEXT:    lh s10, 180(sp)
 ; ZHINX32-NEXT:    lh s11, 184(sp)
 ; ZHINX32-NEXT:    lh ra, 188(sp)
-; ZHINX32-NEXT:    lh t3, 192(sp)
-; ZHINX32-NEXT:    lh t2, 196(sp)
-; ZHINX32-NEXT:    lh t1, 200(sp)
-; ZHINX32-NEXT:    lh t0, 204(sp)
-; ZHINX32-NEXT:    sh t0, 38(sp)
-; ZHINX32-NEXT:    sh t1, 36(sp)
-; ZHINX32-NEXT:    sh t2, 34(sp)
-; ZHINX32-NEXT:    sh t3, 32(sp)
-; ZHINX32-NEXT:    sh ra, 30(sp)
-; ZHINX32-NEXT:    sh s11, 28(sp)
-; ZHINX32-NEXT:    sh s10, 26(sp)
+; ZHINX32-NEXT:    lh t0, 192(sp)
+; ZHINX32-NEXT:    lh t1, 196(sp)
+; ZHINX32-NEXT:    lh t2, 200(sp)
+; ZHINX32-NEXT:    lh t3, 204(sp)
+; ZHINX32-NEXT:    sh t0, 32(sp)
+; ZHINX32-NEXT:    sh t1, 34(sp)
+; ZHINX32-NEXT:    sh t2, 36(sp)
+; ZHINX32-NEXT:    sh t3, 38(sp)
 ; ZHINX32-NEXT:    sh s9, 24(sp)
-; ZHINX32-NEXT:    sh s8, 22(sp)
-; ZHINX32-NEXT:    sh s7, 20(sp)
-; ZHINX32-NEXT:    sh s6, 18(sp)
+; ZHINX32-NEXT:    sh s10, 26(sp)
+; ZHINX32-NEXT:    sh s11, 28(sp)
+; ZHINX32-NEXT:    sh ra, 30(sp)
 ; ZHINX32-NEXT:    sh s5, 16(sp)
-; ZHINX32-NEXT:    sh s4, 14(sp)
-; ZHINX32-NEXT:    sh s3, 12(sp)
-; ZHINX32-NEXT:    sh s2, 10(sp)
+; ZHINX32-NEXT:    sh s6, 18(sp)
+; ZHINX32-NEXT:    sh s7, 20(sp)
+; ZHINX32-NEXT:    sh s8, 22(sp)
 ; ZHINX32-NEXT:    sh s1, 8(sp)
-; ZHINX32-NEXT:    sh s0, 6(sp)
-; ZHINX32-NEXT:    sh t4, 4(sp)
-; ZHINX32-NEXT:    sh t5, 2(sp)
+; ZHINX32-NEXT:    sh s2, 10(sp)
+; ZHINX32-NEXT:    sh s3, 12(sp)
+; ZHINX32-NEXT:    sh s4, 14(sp)
 ; ZHINX32-NEXT:    sh t6, 0(sp)
+; ZHINX32-NEXT:    sh t4, 2(sp)
+; ZHINX32-NEXT:    sh t5, 4(sp)
+; ZHINX32-NEXT:    sh s0, 6(sp)
 ; ZHINX32-NEXT:    lh t3, 58(sp) # 2-byte Folded Reload
 ; ZHINX32-NEXT:    lh t4, 56(sp) # 2-byte Folded Reload
 ; ZHINX32-NEXT:    lh t5, 54(sp) # 2-byte Folded Reload
@@ -374,8 +374,8 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZHINX64-NEXT:    lh t0, 184(sp)
 ; ZHINX64-NEXT:    sh t0, 48(sp) # 2-byte Folded Spill
 ; ZHINX64-NEXT:    lh t6, 192(sp)
-; ZHINX64-NEXT:    lh t5, 200(sp)
-; ZHINX64-NEXT:    lh t4, 208(sp)
+; ZHINX64-NEXT:    lh t4, 200(sp)
+; ZHINX64-NEXT:    lh t5, 208(sp)
 ; ZHINX64-NEXT:    lh s0, 216(sp)
 ; ZHINX64-NEXT:    lh s1, 224(sp)
 ; ZHINX64-NEXT:    lh s2, 232(sp)
@@ -389,30 +389,30 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZHINX64-NEXT:    lh s10, 296(sp)
 ; ZHINX64-NEXT:    lh s11, 304(sp)
 ; ZHINX64-NEXT:    lh ra, 312(sp)
-; ZHINX64-NEXT:    lh t3, 320(sp)
-; ZHINX64-NEXT:    lh t2, 328(sp)
-; ZHINX64-NEXT:    lh t1, 336(sp)
-; ZHINX64-NEXT:    lh t0, 344(sp)
-; ZHINX64-NEXT:    sh t0, 38(sp)
-; ZHINX64-NEXT:    sh t1, 36(sp)
-; ZHINX64-NEXT:    sh t2, 34(sp)
-; ZHINX64-NEXT:    sh t3, 32(sp)
-; ZHINX64-NEXT:    sh ra, 30(sp)
-; ZHINX64-NEXT:    sh s11, 28(sp)
-; ZHINX64-NEXT:    sh s10, 26(sp)
+; ZHINX64-NEXT:    lh t0, 320(sp)
+; ZHINX64-NEXT:    lh t1, 328(sp)
+; ZHINX64-NEXT:    lh t2, 336(sp)
+; ZHINX64-NEXT:    lh t3, 344(sp)
+; ZHINX64-NEXT:    sh t0, 32(sp)
+; ZHINX64-NEXT:    sh t1, 34(sp)
+; ZHINX64-NEXT:    sh t2, 36(sp)
+; ZHINX64-NEXT:    sh t3, 38(sp)
 ; ZHINX64-NEXT:    sh s9, 24(sp)
-; ZHINX64-NEXT:    sh s8, 22(sp)
-; ZHINX64-NEXT:    sh s7, 20(sp)
-; ZHINX64-NEXT:    sh s6, 18(sp)
+; ZHINX64-NEXT:    sh s10, 26(sp)
+; ZHINX64-NEXT:    sh s11, 28(sp)
+; ZHINX64-NEXT:    sh ra, 30(sp)
 ; ZHINX64-NEXT:    sh s5, 16(sp)
-; ZHINX64-NEXT:    sh s4, 14(sp)
-; ZHINX64-NEXT:    sh s3, 12(sp)
-; ZHINX64-NEXT:    sh s2, 10(sp)
+; ZHINX64-NEXT:    sh s6, 18(sp)
+; ZHINX64-NEXT:    sh s7, 20(sp)
+; ZHINX64-NEXT:    sh s8, 22(sp)
 ; ZHINX64-NEXT:    sh s1, 8(sp)
-; ZHINX64-NEXT:    sh s0, 6(sp)
-; ZHINX64-NEXT:    sh t4, 4(sp)
-; ZHINX64-NEXT:    sh t5, 2(sp)
+; ZHINX64-NEXT:    sh s2, 10(sp)
+; ZHINX64-NEXT:    sh s3, 12(sp)
+; ZHINX64-NEXT:    sh s4, 14(sp)
 ; ZHINX64-NEXT:    sh t6, 0(sp)
+; ZHINX64-NEXT:    sh t4, 2(sp)
+; ZHINX64-NEXT:    sh t5, 4(sp)
+; ZHINX64-NEXT:    sh s0, 6(sp)
 ; ZHINX64-NEXT:    lh t3, 54(sp) # 2-byte Folded Reload
 ; ZHINX64-NEXT:    lh t4, 52(sp) # 2-byte Folded Reload
 ; ZHINX64-NEXT:    lh t5, 50(sp) # 2-byte Folded Reload
@@ -450,67 +450,67 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZFINX32-NEXT:    sw s9, 116(sp) # 4-byte Folded Spill
 ; ZFINX32-NEXT:    sw s10, 112(sp) # 4-byte Folded Spill
 ; ZFINX32-NEXT:    sw s11, 108(sp) # 4-byte Folded Spill
-; ZFINX32-NEXT:    lw t0, 0(a0)
-; ZFINX32-NEXT:    lw a1, 4(a0)
-; ZFINX32-NEXT:    sw a1, 104(sp) # 4-byte Folded Spill
-; ZFINX32-NEXT:    lw a1, 8(a0)
-; ZFINX32-NEXT:    sw a1, 100(sp) # 4-byte Folded Spill
-; ZFINX32-NEXT:    lw a1, 12(a0)
-; ZFINX32-NEXT:    sw a1, 96(sp) # 4-byte Folded Spill
-; ZFINX32-NEXT:    lw a1, 16(a0)
-; ZFINX32-NEXT:    sw a1, 92(sp) # 4-byte Folded Spill
-; ZFINX32-NEXT:    lw a5, 20(a0)
-; ZFINX32-NEXT:    lw a6, 24(a0)
-; ZFINX32-NEXT:    lw a7, 28(a0)
-; ZFINX32-NEXT:    lw t3, 32(a0)
-; ZFINX32-NEXT:    lw t4, 36(a0)
-; ZFINX32-NEXT:    lw t5, 40(a0)
-; ZFINX32-NEXT:    lw t6, 44(a0)
-; ZFINX32-NEXT:    lw t1, 48(a0)
-; ZFINX32-NEXT:    lw t2, 52(a0)
-; ZFINX32-NEXT:    lw s0, 56(a0)
-; ZFINX32-NEXT:    lw s1, 60(a0)
-; ZFINX32-NEXT:    lw s2, 64(a0)
-; ZFINX32-NEXT:    lw s3, 68(a0)
-; ZFINX32-NEXT:    lw s4, 72(a0)
-; ZFINX32-NEXT:    lw s5, 76(a0)
-; ZFINX32-NEXT:    lw s6, 80(a0)
-; ZFINX32-NEXT:    lw s7, 84(a0)
-; ZFINX32-NEXT:    lw s8, 88(a0)
-; ZFINX32-NEXT:    lw s9, 92(a0)
-; ZFINX32-NEXT:    lw s10, 96(a0)
-; ZFINX32-NEXT:    lw s11, 100(a0)
-; ZFINX32-NEXT:    lw ra, 104(a0)
-; ZFINX32-NEXT:    lw a4, 108(a0)
-; ZFINX32-NEXT:    lw a3, 112(a0)
-; ZFINX32-NEXT:    lw a2, 116(a0)
-; ZFINX32-NEXT:    lw a1, 120(a0)
-; ZFINX32-NEXT:    lw a0, 124(a0)
-; ZFINX32-NEXT:    sw a0, 76(sp)
-; ZFINX32-NEXT:    sw a1, 72(sp)
-; ZFINX32-NEXT:    sw a2, 68(sp)
-; ZFINX32-NEXT:    sw a3, 64(sp)
-; ZFINX32-NEXT:    sw a4, 60(sp)
-; ZFINX32-NEXT:    sw ra, 56(sp)
-; ZFINX32-NEXT:    sw s11, 52(sp)
+; ZFINX32-NEXT:    mv t0, a0
+; ZFINX32-NEXT:    lw a0, 0(a0)
+; ZFINX32-NEXT:    sw a0, 104(sp) # 4-byte Folded Spill
+; ZFINX32-NEXT:    lw a0, 4(t0)
+; ZFINX32-NEXT:    sw a0, 100(sp) # 4-byte Folded Spill
+; ZFINX32-NEXT:    lw a0, 8(t0)
+; ZFINX32-NEXT:    sw a0, 96(sp) # 4-byte Folded Spill
+; ZFINX32-NEXT:    lw a0, 12(t0)
+; ZFINX32-NEXT:    sw a0, 92(sp) # 4-byte Folded Spill
+; ZFINX32-NEXT:    lw a4, 16(t0)
+; ZFINX32-NEXT:    lw a5, 20(t0)
+; ZFINX32-NEXT:    lw a6, 24(t0)
+; ZFINX32-NEXT:    lw a7, 28(t0)
+; ZFINX32-NEXT:    lw t3, 32(t0)
+; ZFINX32-NEXT:    lw t4, 36(t0)
+; ZFINX32-NEXT:    lw t5, 40(t0)
+; ZFINX32-NEXT:    lw t6, 44(t0)
+; ZFINX32-NEXT:    lw t1, 48(t0)
+; ZFINX32-NEXT:    lw t2, 52(t0)
+; ZFINX32-NEXT:    lw s0, 56(t0)
+; ZFINX32-NEXT:    lw s1, 60(t0)
+; ZFINX32-NEXT:    lw s2, 64(t0)
+; ZFINX32-NEXT:    lw s3, 68(t0)
+; ZFINX32-NEXT:    lw s4, 72(t0)
+; ZFINX32-NEXT:    lw s5, 76(t0)
+; ZFINX32-NEXT:    lw s6, 80(t0)
+; ZFINX32-NEXT:    lw s7, 84(t0)
+; ZFINX32-NEXT:    lw s8, 88(t0)
+; ZFINX32-NEXT:    lw s9, 92(t0)
+; ZFINX32-NEXT:    lw s10, 96(t0)
+; ZFINX32-NEXT:    lw s11, 100(t0)
+; ZFINX32-NEXT:    lw ra, 104(t0)
+; ZFINX32-NEXT:    lw a3, 108(t0)
+; ZFINX32-NEXT:    lw a0, 112(t0)
+; ZFINX32-NEXT:    lw a1, 116(t0)
+; ZFINX32-NEXT:    lw a2, 120(t0)
+; ZFINX32-NEXT:    lw t0, 124(t0)
+; ZFINX32-NEXT:    sw a0, 64(sp)
+; ZFINX32-NEXT:    sw a1, 68(sp)
+; ZFINX32-NEXT:    sw a2, 72(sp)
+; ZFINX32-NEXT:    sw t0, 76(sp)
 ; ZFINX32-NEXT:    sw s10, 48(sp)
-; ZFINX32-NEXT:    sw s9, 44(sp)
-; ZFINX32-NEXT:    sw s8, 40(sp)
-; ZFINX32-NEXT:    sw s7, 36(sp)
+; ZFINX32-NEXT:    sw s11, 52(sp)
+; ZFINX32-NEXT:    sw ra, 56(sp)
+; ZFINX32-NEXT:    sw a3, 60(sp)
 ; ZFINX32-NEXT:    sw s6, 32(sp)
-; ZFINX32-NEXT:    sw s5, 28(sp)
-; ZFINX32-NEXT:    sw s4, 24(sp)
-; ZFINX32-NEXT:    sw s3, 20(sp)
+; ZFINX32-NEXT:    sw s7, 36(sp)
+; ZFINX32-NEXT:    sw s8, 40(sp)
+; ZFINX32-NEXT:    sw s9, 44(sp)
 ; ZFINX32-NEXT:    sw s2, 16(sp)
-; ZFINX32-NEXT:    sw s1, 12(sp)
-; ZFINX32-NEXT:    sw s0, 8(sp)
-; ZFINX32-NEXT:    sw t2, 4(sp)
+; ZFINX32-NEXT:    sw s3, 20(sp)
+; ZFINX32-NEXT:    sw s4, 24(sp)
+; ZFINX32-NEXT:    sw s5, 28(sp)
 ; ZFINX32-NEXT:    sw t1, 0(sp)
-; ZFINX32-NEXT:    mv a0, t0
-; ZFINX32-NEXT:    lw a1, 104(sp) # 4-byte Folded Reload
-; ZFINX32-NEXT:    lw a2, 100(sp) # 4-byte Folded Reload
-; ZFINX32-NEXT:    lw a3, 96(sp) # 4-byte Folded Reload
-; ZFINX32-NEXT:    lw a4, 92(sp) # 4-byte Folded Reload
+; ZFINX32-NEXT:    sw t2, 4(sp)
+; ZFINX32-NEXT:    sw s0, 8(sp)
+; ZFINX32-NEXT:    sw s1, 12(sp)
+; ZFINX32-NEXT:    lw a0, 104(sp) # 4-byte Folded Reload
+; ZFINX32-NEXT:    lw a1, 100(sp) # 4-byte Folded Reload
+; ZFINX32-NEXT:    lw a2, 96(sp) # 4-byte Folded Reload
+; ZFINX32-NEXT:    lw a3, 92(sp) # 4-byte Folded Reload
 ; ZFINX32-NEXT:    call callee_half_32
 ; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX32-NEXT:    lui a1, 1048560
@@ -548,67 +548,67 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZFINX64-NEXT:    sd s9, 216(sp) # 8-byte Folded Spill
 ; ZFINX64-NEXT:    sd s10, 208(sp) # 8-byte Folded Spill
 ; ZFINX64-NEXT:    sd s11, 200(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    ld t0, 0(a0)
-; ZFINX64-NEXT:    ld a1, 8(a0)
-; ZFINX64-NEXT:    sd a1, 192(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    ld a1, 16(a0)
-; ZFINX64-NEXT:    sd a1, 184(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    ld a1, 24(a0)
-; ZFINX64-NEXT:    sd a1, 176(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    ld a1, 32(a0)
-; ZFINX64-NEXT:    sd a1, 168(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    ld a5, 40(a0)
-; ZFINX64-NEXT:    ld a6, 48(a0)
-; ZFINX64-NEXT:    ld a7, 56(a0)
-; ZFINX64-NEXT:    ld t3, 64(a0)
-; ZFINX64-NEXT:    ld t4, 72(a0)
-; ZFINX64-NEXT:    ld t5, 80(a0)
-; ZFINX64-NEXT:    ld t6, 88(a0)
-; ZFINX64-NEXT:    ld t1, 96(a0)
-; ZFINX64-NEXT:    ld t2, 104(a0)
-; ZFINX64-NEXT:    ld s0, 112(a0)
-; ZFINX64-NEXT:    ld s1, 120(a0)
-; ZFINX64-NEXT:    ld s2, 128(a0)
-; ZFINX64-NEXT:    ld s3, 136(a0)
-; ZFINX64-NEXT:    ld s4, 144(a0)
-; ZFINX64-NEXT:    ld s5, 152(a0)
-; ZFINX64-NEXT:    ld s6, 160(a0)
-; ZFINX64-NEXT:    ld s7, 168(a0)
-; ZFINX64-NEXT:    ld s8, 176(a0)
-; ZFINX64-NEXT:    ld s9, 184(a0)
-; ZFINX64-NEXT:    ld s10, 192(a0)
-; ZFINX64-NEXT:    ld s11, 200(a0)
-; ZFINX64-NEXT:    ld ra, 208(a0)
-; ZFINX64-NEXT:    ld a4, 216(a0)
-; ZFINX64-NEXT:    ld a3, 224(a0)
-; ZFINX64-NEXT:    ld a2, 232(a0)
-; ZFINX64-NEXT:    ld a1, 240(a0)
-; ZFINX64-NEXT:    ld a0, 248(a0)
-; ZFINX64-NEXT:    sd a0, 152(sp)
-; ZFINX64-NEXT:    sd a1, 144(sp)
-; ZFINX64-NEXT:    sd a2, 136(sp)
-; ZFINX64-NEXT:    sd a3, 128(sp)
-; ZFINX64-NEXT:    sd a4, 120(sp)
-; ZFINX64-NEXT:    sd ra, 112(sp)
-; ZFINX64-NEXT:    sd s11, 104(sp)
+; ZFINX64-NEXT:    mv t0, a0
+; ZFINX64-NEXT:    ld a0, 0(a0)
+; ZFINX64-NEXT:    sd a0, 192(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    ld a0, 8(t0)
+; ZFINX64-NEXT:    sd a0, 184(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    ld a0, 16(t0)
+; ZFINX64-NEXT:    sd a0, 176(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    ld a0, 24(t0)
+; ZFINX64-NEXT:    sd a0, 168(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    ld a4, 32(t0)
+; ZFINX64-NEXT:    ld a5, 40(t0)
+; ZFINX64-NEXT:    ld a6, 48(t0)
+; ZFINX64-NEXT:    ld a7, 56(t0)
+; ZFINX64-NEXT:    ld t3, 64(t0)
+; ZFINX64-NEXT:    ld t4, 72(t0)
+; ZFINX64-NEXT:    ld t5, 80(t0)
+; ZFINX64-NEXT:    ld t6, 88(t0)
+; ZFINX64-NEXT:    ld t1, 96(t0)
+; ZFINX64-NEXT:    ld t2, 104(t0)
+; ZFINX64-NEXT:    ld s0, 112(t0)
+; ZFINX64-NEXT:    ld s1, 120(t0)
+; ZFINX64-NEXT:    ld s2, 128(t0)
+; ZFINX64-NEXT:    ld s3, 136(t0)
+; ZFINX64-NEXT:    ld s4, 144(t0)
+; ZFINX64-NEXT:    ld s5, 152(t0)
+; ZFINX64-NEXT:    ld s6, 160(t0)
+; ZFINX64-NEXT:    ld s7, 168(t0)
+; ZFINX64-NEXT:    ld s8, 176(t0)
+; ZFINX64-NEXT:    ld s9, 184(t0)
+; ZFINX64-NEXT:    ld s10, 192(t0)
+; ZFINX64-NEXT:    ld s11, 200(t0)
+; ZFINX64-NEXT:    ld ra, 208(t0)
+; ZFINX64-NEXT:    ld a3, 216(t0)
+; ZFINX64-NEXT:    ld a0, 224(t0)
+; ZFINX64-NEXT:    ld a1, 232(t0)
+; ZFINX64-NEXT:    ld a2, 240(t0)
+; ZFINX64-NEXT:    ld t0, 248(t0)
+; ZFINX64-NEXT:    sd a0, 128(sp)
+; ZFINX64-NEXT:    sd a1, 136(sp)
+; ZFINX64-NEXT:    sd a2, 144(sp)
+; ZFINX64-NEXT:    sd t0, 152(sp)
 ; ZFINX64-NEXT:    sd s10, 96(sp)
-; ZFINX64-NEXT:    sd s9, 88(sp)
-; ZFINX64-NEXT:    sd s8, 80(sp)
-; ZFINX64-NEXT:    sd s7, 72(sp)
+; ZFINX64-NEXT:    sd s11, 104(sp)
+; ZFINX64-NEXT:    sd ra, 112(sp)
+; ZFINX64-NEXT:    sd a3, 120(sp)
 ; ZFINX64-NEXT:    sd s6, 64(sp)
-; ZFINX64-NEXT:    sd s5, 56(sp)
-; ZFINX64-NEXT:    sd s4, 48(sp)
-; ZFINX64-NEXT:    sd s3, 40(sp)
+; ZFINX64-NEXT:    sd s7, 72(sp)
+; ZFINX64-NEXT:    sd s8, 80(sp)
+; ZFINX64-NEXT:    sd s9, 88(sp)
 ; ZFINX64-NEXT:    sd s2, 32(sp)
-; ZFINX64-NEXT:    sd s1, 24(sp)
-; ZFINX64-NEXT:    sd s0, 16(sp)
-; ZFINX64-NEXT:    sd t2, 8(sp)
+; ZFINX64-NEXT:    sd s3, 40(sp)
+; ZFINX64-NEXT:    sd s4, 48(sp)
+; ZFINX64-NEXT:    sd s5, 56(sp)
 ; ZFINX64-NEXT:    sd t1, 0(sp)
-; ZFINX64-NEXT:    mv a0, t0
-; ZFINX64-NEXT:    ld a1, 192(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld a2, 184(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld a3, 176(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld a4, 168(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    sd t2, 8(sp)
+; ZFINX64-NEXT:    sd s0, 16(sp)
+; ZFINX64-NEXT:    sd s1, 24(sp)
+; ZFINX64-NEXT:    ld a0, 192(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld a1, 184(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld a2, 176(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld a3, 168(sp) # 8-byte Folded Reload
 ; ZFINX64-NEXT:    call callee_half_32
 ; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX64-NEXT:    lui a1, 1048560
@@ -646,67 +646,67 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZDINX32-NEXT:    sw s9, 116(sp) # 4-byte Folded Spill
 ; ZDINX32-NEXT:    sw s10, 112(sp) # 4-byte Folded Spill
 ; ZDINX32-NEXT:    sw s11, 108(sp) # 4-byte Folded Spill
-; ZDINX32-NEXT:    lw t0, 0(a0)
-; ZDINX32-NEXT:    lw a1, 4(a0)
-; ZDINX32-NEXT:    sw a1, 104(sp) # 4-byte Folded Spill
-; ZDINX32-NEXT:    lw a1, 8(a0)
-; ZDINX32-NEXT:    sw a1, 100(sp) # 4-byte Folded Spill
-; ZDINX32-NEXT:    lw a1, 12(a0)
-; ZDINX32-NEXT:    sw a1, 96(sp) # 4-byte Folded Spill
-; ZDINX32-NEXT:    lw a1, 16(a0)
-; ZDINX32-NEXT:    sw a1, 92(sp) # 4-byte Folded Spill
-; ZDINX32-NEXT:    lw a5, 20(a0)
-; ZDINX32-NEXT:    lw a6, 24(a0)
-; ZDINX32-NEXT:    lw a7, 28(a0)
-; ZDINX32-NEXT:    lw t3, 32(a0)
-; ZDINX32-NEXT:    lw t4, 36(a0)
-; ZDINX32-NEXT:    lw t5, 40(a0)
-; ZDINX32-NEXT:    lw t6, 44(a0)
-; ZDINX32-NEXT:    lw t1, 48(a0)
-; ZDINX32-NEXT:    lw t2, 52(a0)
-; ZDINX32-NEXT:    lw s0, 56(a0)
-; ZDINX32-NEXT:    lw s1, 60(a0)
-; ZDINX32-NEXT:    lw s2, 64(a0)
-; ZDINX32-NEXT:    lw s3, 68(a0)
-; ZDINX32-NEXT:    lw s4, 72(a0)
-; ZDINX32-NEXT:    lw s5, 76(a0)
-; ZDINX32-NEXT:    lw s6, 80(a0)
-; ZDINX32-NEXT:    lw s7, 84(a0)
-; ZDINX32-NEXT:    lw s8, 88(a0)
-; ZDINX32-NEXT:    lw s9, 92(a0)
-; ZDINX32-NEXT:    lw s10, 96(a0)
-; ZDINX32-NEXT:    lw s11, 100(a0)
-; ZDINX32-NEXT:    lw ra, 104(a0)
-; ZDINX32-NEXT:    lw a4, 108(a0)
-; ZDINX32-NEXT:    lw a3, 112(a0)
-; ZDINX32-NEXT:    lw a2, 116(a0)
-; ZDINX32-NEXT:    lw a1, 120(a0)
-; ZDINX32-NEXT:    lw a0, 124(a0)
-; ZDINX32-NEXT:    sw a0, 76(sp)
-; ZDINX32-NEXT:    sw a1, 72(sp)
-; ZDINX32-NEXT:    sw a2, 68(sp)
-; ZDINX32-NEXT:    sw a3, 64(sp)
-; ZDINX32-NEXT:    sw a4, 60(sp)
-; ZDINX32-NEXT:    sw ra, 56(sp)
-; ZDINX32-NEXT:    sw s11, 52(sp)
+; ZDINX32-NEXT:    mv t0, a0
+; ZDINX32-NEXT:    lw a0, 0(a0)
+; ZDINX32-NEXT:    sw a0, 104(sp) # 4-byte Folded Spill
+; ZDINX32-NEXT:    lw a0, 4(t0)
+; ZDINX32-NEXT:    sw a0, 100(sp) # 4-byte Folded Spill
+; ZDINX32-NEXT:    lw a0, 8(t0)
+; ZDINX32-NEXT:    sw a0, 96(sp) # 4-byte Folded Spill
+; ZDINX32-NEXT:    lw a0, 12(t0)
+; ZDINX32-NEXT:    sw a0, 92(sp) # 4-byte Folded Spill
+; ZDINX32-NEXT:    lw a4, 16(t0)
+; ZDINX32-NEXT:    lw a5, 20(t0)
+; ZDINX32-NEXT:    lw a6, 24(t0)
+; ZDINX32-NEXT:    lw a7, 28(t0)
+; ZDINX32-NEXT:    lw t3, 32(t0)
+; ZDINX32-NEXT:    lw t4, 36(t0)
+; ZDINX32-NEXT:    lw t5, 40(t0)
+; ZDINX32-NEXT:    lw t6, 44(t0)
+; ZDINX32-NEXT:    lw t1, 48(t0)
+; ZDINX32-NEXT:    lw t2, 52(t0)
+; ZDINX32-NEXT:    lw s0, 56(t0)
+; ZDINX32-NEXT:    lw s1, 60(t0)
+; ZDINX32-NEXT:    lw s2, 64(t0)
+; ZDINX32-NEXT:    lw s3, 68(t0)
+; ZDINX32-NEXT:    lw s4, 72(t0)
+; ZDINX32-NEXT:    lw s5, 76(t0)
+; ZDINX32-NEXT:    lw s6, 80(t0)
+; ZDINX32-NEXT:    lw s7, 84(t0)
+; ZDINX32-NEXT:    lw s8, 88(t0)
+; ZDINX32-NEXT:    lw s9, 92(t0)
+; ZDINX32-NEXT:    lw s10, 96(t0)
+; ZDINX32-NEXT:    lw s11, 100(t0)
+; ZDINX32-NEXT:    lw ra, 104(t0)
+; ZDINX32-NEXT:    lw a3, 108(t0)
+; ZDINX32-NEXT:    lw a0, 112(t0)
+; ZDINX32-NEXT:    lw a1, 116(t0)
+; ZDINX32-NEXT:    lw a2, 120(t0)
+; ZDINX32-NEXT:    lw t0, 124(t0)
+; ZDINX32-NEXT:    sw a0, 64(sp)
+; ZDINX32-NEXT:    sw a1, 68(sp)
+; ZDINX32-NEXT:    sw a2, 72(sp)
+; ZDINX32-NEXT:    sw t0, 76(sp)
 ; ZDINX32-NEXT:    sw s10, 48(sp)
-; ZDINX32-NEXT:    sw s9, 44(sp)
-; ZDINX32-NEXT:    sw s8, 40(sp)
-; ZDINX32-NEXT:    sw s7, 36(sp)
+; ZDINX32-NEXT:    sw s11, 52(sp)
+; ZDINX32-NEXT:    sw ra, 56(sp)
+; ZDINX32-NEXT:    sw a3, 60(sp)
 ; ZDINX32-NEXT:    sw s6, 32(sp)
-; ZDINX32-NEXT:    sw s5, 28(sp)
-; ZDINX32-NEXT:    sw s4, 24(sp)
-; ZDINX32-NEXT:    sw s3, 20(sp)
+; ZDINX32-NEXT:    sw s7, 36(sp)
+; ZDINX32-NEXT:    sw s8, 40(sp)
+; ZDINX32-NEXT:    sw s9, 44(sp)
 ; ZDINX32-NEXT:    sw s2, 16(sp)
-; ZDINX32-NEXT:    sw s1, 12(sp)
-; ZDINX32-NEXT:    sw s0, 8(sp)
-; ZDINX32-NEXT:    sw t2, 4(sp)
+; ZDINX32-NEXT:    sw s3, 20(sp)
+; ZDINX32-NEXT:    sw s4, 24(sp)
+; ZDINX32-NEXT:    sw s5, 28(sp)
 ; ZDINX32-NEXT:    sw t1, 0(sp)
-; ZDINX32-NEXT:    mv a0, t0
-; ZDINX32-NEXT:    lw a1, 104(sp) # 4-byte Folded Reload
-; ZDINX32-NEXT:    lw a2, 100(sp) # 4-byte Folded Reload
-; ZDINX32-NEXT:    lw a3, 96(sp) # 4-byte Folded Reload
-; ZDINX32-NEXT:    lw a4, 92(sp) # 4-byte Folded Reload
+; ZDINX32-NEXT:    sw t2, 4(sp)
+; ZDINX32-NEXT:    sw s0, 8(sp)
+; ZDINX32-NEXT:    sw s1, 12(sp)
+; ZDINX32-NEXT:    lw a0, 104(sp) # 4-byte Folded Reload
+; ZDINX32-NEXT:    lw a1, 100(sp) # 4-byte Folded Reload
+; ZDINX32-NEXT:    lw a2, 96(sp) # 4-byte Folded Reload
+; ZDINX32-NEXT:    lw a3, 92(sp) # 4-byte Folded Reload
 ; ZDINX32-NEXT:    call callee_half_32
 ; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX32-NEXT:    lui a1, 1048560
@@ -744,67 +744,67 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZDINX64-NEXT:    sd s9, 216(sp) # 8-byte Folded Spill
 ; ZDINX64-NEXT:    sd s10, 208(sp) # 8-byte Folded Spill
 ; ZDINX64-NEXT:    sd s11, 200(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    ld t0, 0(a0)
-; ZDINX64-NEXT:    ld a1, 8(a0)
-; ZDINX64-NEXT:    sd a1, 192(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    ld a1, 16(a0)
-; ZDINX64-NEXT:    sd a1, 184(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    ld a1, 24(a0)
-; ZDINX64-NEXT:    sd a1, 176(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    ld a1, 32(a0)
-; ZDINX64-NEXT:    sd a1, 168(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    ld a5, 40(a0)
-; ZDINX64-NEXT:    ld a6, 48(a0)
-; ZDINX64-NEXT:    ld a7, 56(a0)
-; ZDINX64-NEXT:    ld t3, 64(a0)
-; ZDINX64-NEXT:    ld t4, 72(a0)
-; ZDINX64-NEXT:    ld t5, 80(a0)
-; ZDINX64-NEXT:    ld t6, 88(a0)
-; ZDINX64-NEXT:    ld t1, 96(a0)
-; ZDINX64-NEXT:    ld t2, 104(a0)
-; ZDINX64-NEXT:    ld s0, 112(a0)
-; ZDINX64-NEXT:    ld s1, 120(a0)
-; ZDINX64-NEXT:    ld s2, 128(a0)
-; ZDINX64-NEXT:    ld s3, 136(a0)
-; ZDINX64-NEXT:    ld s4, 144(a0)
-; ZDINX64-NEXT:    ld s5, 152(a0)
-; ZDINX64-NEXT:    ld s6, 160(a0)
-; ZDINX64-NEXT:    ld s7, 168(a0)
-; ZDINX64-NEXT:    ld s8, 176(a0)
-; ZDINX64-NEXT:    ld s9, 184(a0)
-; ZDINX64-NEXT:    ld s10, 192(a0)
-; ZDINX64-NEXT:    ld s11, 200(a0)
-; ZDINX64-NEXT:    ld ra, 208(a0)
-; ZDINX64-NEXT:    ld a4, 216(a0)
-; ZDINX64-NEXT:    ld a3, 224(a0)
-; ZDINX64-NEXT:    ld a2, 232(a0)
-; ZDINX64-NEXT:    ld a1, 240(a0)
-; ZDINX64-NEXT:    ld a0, 248(a0)
-; ZDINX64-NEXT:    sd a0, 152(sp)
-; ZDINX64-NEXT:    sd a1, 144(sp)
-; ZDINX64-NEXT:    sd a2, 136(sp)
-; ZDINX64-NEXT:    sd a3, 128(sp)
-; ZDINX64-NEXT:    sd a4, 120(sp)
-; ZDINX64-NEXT:    sd ra, 112(sp)
-; ZDINX64-NEXT:    sd s11, 104(sp)
+; ZDINX64-NEXT:    mv t0, a0
+; ZDINX64-NEXT:    ld a0, 0(a0)
+; ZDINX64-NEXT:    sd a0, 192(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    ld a0, 8(t0)
+; ZDINX64-NEXT:    sd a0, 184(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    ld a0, 16(t0)
+; ZDINX64-NEXT:    sd a0, 176(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    ld a0, 24(t0)
+; ZDINX64-NEXT:    sd a0, 168(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    ld a4, 32(t0)
+; ZDINX64-NEXT:    ld a5, 40(t0)
+; ZDINX64-NEXT:    ld a6, 48(t0)
+; ZDINX64-NEXT:    ld a7, 56(t0)
+; ZDINX64-NEXT:    ld t3, 64(t0)
+; ZDINX64-NEXT:    ld t4, 72(t0)
+; ZDINX64-NEXT:    ld t5, 80(t0)
+; ZDINX64-NEXT:    ld t6, 88(t0)
+; ZDINX64-NEXT:    ld t1, 96(t0)
+; ZDINX64-NEXT:    ld t2, 104(t0)
+; ZDINX64-NEXT:    ld s0, 112(t0)
+; ZDINX64-NEXT:    ld s1, 120(t0)
+; ZDINX64-NEXT:    ld s2, 128(t0)
+; ZDINX64-NEXT:    ld s3, 136(t0)
+; ZDINX64-NEXT:    ld s4, 144(t0)
+; ZDINX64-NEXT:    ld s5, 152(t0)
+; ZDINX64-NEXT:    ld s6, 160(t0)
+; ZDINX64-NEXT:    ld s7, 168(t0)
+; ZDINX64-NEXT:    ld s8, 176(t0)
+; ZDINX64-NEXT:    ld s9, 184(t0)
+; ZDINX64-NEXT:    ld s10, 192(t0)
+; ZDINX64-NEXT:    ld s11, 200(t0)
+; ZDINX64-NEXT:    ld ra, 208(t0)
+; ZDINX64-NEXT:    ld a3, 216(t0)
+; ZDINX64-NEXT:    ld a0, 224(t0)
+; ZDINX64-NEXT:    ld a1, 232(t0)
+; ZDINX64-NEXT:    ld a2, 240(t0)
+; ZDINX64-NEXT:    ld t0, 248(t0)
+; ZDINX64-NEXT:    sd a0, 128(sp)
+; ZDINX64-NEXT:    sd a1, 136(sp)
+; ZDINX64-NEXT:    sd a2, 144(sp)
+; ZDINX64-NEXT:    sd t0, 152(sp)
 ; ZDINX64-NEXT:    sd s10, 96(sp)
-; ZDINX64-NEXT:    sd s9, 88(sp)
-; ZDINX64-NEXT:    sd s8, 80(sp)
-; ZDINX64-NEXT:    sd s7, 72(sp)
+; ZDINX64-NEXT:    sd s11, 104(sp)
+; ZDINX64-NEXT:    sd ra, 112(sp)
+; ZDINX64-NEXT:    sd a3, 120(sp)
 ; ZDINX64-NEXT:    sd s6, 64(sp)
-; ZDINX64-NEXT:    sd s5, 56(sp)
-; ZDINX64-NEXT:    sd s4, 48(sp)
-; ZDINX64-NEXT:    sd s3, 40(sp)
+; ZDINX64-NEXT:    sd s7, 72(sp)
+; ZDINX64-NEXT:    sd s8, 80(sp)
+; ZDINX64-NEXT:    sd s9, 88(sp)
 ; ZDINX64-NEXT:    sd s2, 32(sp)
-; ZDINX64-NEXT:    sd s1, 24(sp)
-; ZDINX64-NEXT:    sd s0, 16(sp)
-; ZDINX64-NEXT:    sd t2, 8(sp)
+; ZDINX64-NEXT:    sd s3, 40(sp)
+; ZDINX64-NEXT:    sd s4, 48(sp)
+; ZDINX64-NEXT:    sd s5, 56(sp)
 ; ZDINX64-NEXT:    sd t1, 0(sp)
-; ZDINX64-NEXT:    mv a0, t0
-; ZDINX64-NEXT:    ld a1, 192(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld a2, 184(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld a3, 176(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld a4, 168(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    sd t2, 8(sp)
+; ZDINX64-NEXT:    sd s0, 16(sp)
+; ZDINX64-NEXT:    sd s1, 24(sp)
+; ZDINX64-NEXT:    ld a0, 192(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld a1, 184(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld a2, 176(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld a3, 168(sp) # 8-byte Folded Reload
 ; ZDINX64-NEXT:    call callee_half_32
 ; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX64-NEXT:    lui a1, 1048560
@@ -883,8 +883,8 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZHINX32-NEXT:    lw t0, 172(sp)
 ; ZHINX32-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
 ; ZHINX32-NEXT:    lw t6, 176(sp)
-; ZHINX32-NEXT:    lw t5, 180(sp)
-; ZHINX32-NEXT:    lw t4, 184(sp)
+; ZHINX32-NEXT:    lw t4, 180(sp)
+; ZHINX32-NEXT:    lw t5, 184(sp)
 ; ZHINX32-NEXT:    lw s0, 188(sp)
 ; ZHINX32-NEXT:    lw s1, 192(sp)
 ; ZHINX32-NEXT:    lw s2, 196(sp)
@@ -898,30 +898,30 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZHINX32-NEXT:    lw s10, 228(sp)
 ; ZHINX32-NEXT:    lw s11, 232(sp)
 ; ZHINX32-NEXT:    lw ra, 236(sp)
-; ZHINX32-NEXT:    lw t3, 240(sp)
-; ZHINX32-NEXT:    lw t2, 244(sp)
-; ZHINX32-NEXT:    lw t1, 248(sp)
-; ZHINX32-NEXT:    lw t0, 252(sp)
-; ZHINX32-NEXT:    sw t0, 76(sp)
-; ZHINX32-NEXT:    sw t1, 72(sp)
-; ZHINX32-NEXT:    sw t2, 68(sp)
-; ZHINX32-NEXT:    sw t3, 64(sp)
-; ZHINX32-NEXT:    sw ra, 60(sp)
-; ZHINX32-NEXT:    sw s11, 56(sp)
-; ZHINX32-NEXT:    sw s10, 52(sp)
+; ZHINX32-NEXT:    lw t0, 240(sp)
+; ZHINX32-NEXT:    lw t1, 244(sp)
+; ZHINX32-NEXT:    lw t2, 248(sp)
+; ZHINX32-NEXT:    lw t3, 252(sp)
+; ZHINX32-NEXT:    sw t0, 64(sp)
+; ZHINX32-NEXT:    sw t1, 68(sp)
+; ZHINX32-NEXT:    sw t2, 72(sp)
+; ZHINX32-NEXT:    sw t3, 76(sp)
 ; ZHINX32-NEXT:    sw s9, 48(sp)
-; ZHINX32-NEXT:    sw s8, 44(sp)
-; ZHINX32-NEXT:    sw s7, 40(sp)
-; ZHINX32-NEXT:    sw s6, 36(sp)
+; ZHINX32-NEXT:    sw s10, 52(sp)
+; ZHINX32-NEXT:    sw s11, 56(sp)
+; ZHINX32-NEXT:    sw ra, 60(sp)
 ; ZHINX32-NEXT:    sw s5, 32(sp)
-; ZHINX32-NEXT:    sw s4, 28(sp)
-; ZHINX32-NEXT:    sw s3, 24(sp)
-; ZHINX32-NEXT:    sw s2, 20(sp)
+; ZHINX32-NEXT:    sw s6, 36(sp)
+; ZHINX32-NEXT:    sw s7, 40(sp)
+; ZHINX32-NEXT:    sw s8, 44(sp)
 ; ZHINX32-NEXT:    sw s1, 16(sp)
-; ZHINX32-NEXT:    sw s0, 12(sp)
-; ZHINX32-NEXT:    sw t4, 8(sp)
-; ZHINX32-NEXT:    sw t5, 4(sp)
+; ZHINX32-NEXT:    sw s2, 20(sp)
+; ZHINX32-NEXT:    sw s3, 24(sp)
+; ZHINX32-NEXT:    sw s4, 28(sp)
 ; ZHINX32-NEXT:    sw t6, 0(sp)
+; ZHINX32-NEXT:    sw t4, 4(sp)
+; ZHINX32-NEXT:    sw t5, 8(sp)
+; ZHINX32-NEXT:    sw s0, 12(sp)
 ; ZHINX32-NEXT:    lw t3, 104(sp) # 4-byte Folded Reload
 ; ZHINX32-NEXT:    lw t4, 100(sp) # 4-byte Folded Reload
 ; ZHINX32-NEXT:    lw t5, 96(sp) # 4-byte Folded Reload
@@ -968,8 +968,8 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZHINX64-NEXT:    lw t0, 232(sp)
 ; ZHINX64-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
 ; ZHINX64-NEXT:    lw t6, 240(sp)
-; ZHINX64-NEXT:    lw t5, 248(sp)
-; ZHINX64-NEXT:    lw t4, 256(sp)
+; ZHINX64-NEXT:    lw t4, 248(sp)
+; ZHINX64-NEXT:    lw t5, 256(sp)
 ; ZHINX64-NEXT:    lw s0, 264(sp)
 ; ZHINX64-NEXT:    lw s1, 272(sp)
 ; ZHINX64-NEXT:    lw s2, 280(sp)
@@ -983,30 +983,30 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZHINX64-NEXT:    lw s10, 344(sp)
 ; ZHINX64-NEXT:    lw s11, 352(sp)
 ; ZHINX64-NEXT:    lw ra, 360(sp)
-; ZHINX64-NEXT:    lw t3, 368(sp)
-; ZHINX64-NEXT:    lw t2, 376(sp)
-; ZHINX64-NEXT:    lw t1, 384(sp)
-; ZHINX64-NEXT:    lw t0, 392(sp)
-; ZHINX64-NEXT:    sw t0, 76(sp)
-; ZHINX64-NEXT:    sw t1, 72(sp)
-; ZHINX64-NEXT:    sw t2, 68(sp)
-; ZHINX64-NEXT:    sw t3, 64(sp)
-; ZHINX64-NEXT:    sw ra, 60(sp)
-; ZHINX64-NEXT:    sw s11, 56(sp)
-; ZHINX64-NEXT:    sw s10, 52(sp)
+; ZHINX64-NEXT:    lw t0, 368(sp)
+; ZHINX64-NEXT:    lw t1, 376(sp)
+; ZHINX64-NEXT:    lw t2, 384(sp)
+; ZHINX64-NEXT:    lw t3, 392(sp)
+; ZHINX64-NEXT:    sw t0, 64(sp)
+; ZHINX64-NEXT:    sw t1, 68(sp)
+; ZHINX64-NEXT:    sw t2, 72(sp)
+; ZHINX64-NEXT:    sw t3, 76(sp)
 ; ZHINX64-NEXT:    sw s9, 48(sp)
-; ZHINX64-NEXT:    sw s8, 44(sp)
-; ZHINX64-NEXT:    sw s7, 40(sp)
-; ZHINX64-NEXT:    sw s6, 36(sp)
+; ZHINX64-NEXT:    sw s10, 52(sp)
+; ZHINX64-NEXT:    sw s11, 56(sp)
+; ZHINX64-NEXT:    sw ra, 60(sp)
 ; ZHINX64-NEXT:    sw s5, 32(sp)
-; ZHINX64-NEXT:    sw s4, 28(sp)
-; ZHINX64-NEXT:    sw s3, 24(sp)
-; ZHINX64-NEXT:    sw s2, 20(sp)
+; ZHINX64-NEXT:    sw s6, 36(sp)
+; ZHINX64-NEXT:    sw s7, 40(sp)
+; ZHINX64-NEXT:    sw s8, 44(sp)
 ; ZHINX64-NEXT:    sw s1, 16(sp)
-; ZHINX64-NEXT:    sw s0, 12(sp)
-; ZHINX64-NEXT:    sw t4, 8(sp)
-; ZHINX64-NEXT:    sw t5, 4(sp)
+; ZHINX64-NEXT:    sw s2, 20(sp)
+; ZHINX64-NEXT:    sw s3, 24(sp)
+; ZHINX64-NEXT:    sw s4, 28(sp)
 ; ZHINX64-NEXT:    sw t6, 0(sp)
+; ZHINX64-NEXT:    sw t4, 4(sp)
+; ZHINX64-NEXT:    sw t5, 8(sp)
+; ZHINX64-NEXT:    sw s0, 12(sp)
 ; ZHINX64-NEXT:    lw t3, 100(sp) # 4-byte Folded Reload
 ; ZHINX64-NEXT:    lw t4, 96(sp) # 4-byte Folded Reload
 ; ZHINX64-NEXT:    lw t5, 92(sp) # 4-byte Folded Reload
@@ -1053,8 +1053,8 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZFINX32-NEXT:    lw t0, 172(sp)
 ; ZFINX32-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
 ; ZFINX32-NEXT:    lw t6, 176(sp)
-; ZFINX32-NEXT:    lw t5, 180(sp)
-; ZFINX32-NEXT:    lw t4, 184(sp)
+; ZFINX32-NEXT:    lw t4, 180(sp)
+; ZFINX32-NEXT:    lw t5, 184(sp)
 ; ZFINX32-NEXT:    lw s0, 188(sp)
 ; ZFINX32-NEXT:    lw s1, 192(sp)
 ; ZFINX32-NEXT:    lw s2, 196(sp)
@@ -1068,30 +1068,30 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZFINX32-NEXT:    lw s10, 228(sp)
 ; ZFINX32-NEXT:    lw s11, 232(sp)
 ; ZFINX32-NEXT:    lw ra, 236(sp)
-; ZFINX32-NEXT:    lw t3, 240(sp)
-; ZFINX32-NEXT:    lw t2, 244(sp)
-; ZFINX32-NEXT:    lw t1, 248(sp)
-; ZFINX32-NEXT:    lw t0, 252(sp)
-; ZFINX32-NEXT:    sw t0, 76(sp)
-; ZFINX32-NEXT:    sw t1, 72(sp)
-; ZFINX32-NEXT:    sw t2, 68(sp)
-; ZFINX32-NEXT:    sw t3, 64(sp)
-; ZFINX32-NEXT:    sw ra, 60(sp)
-; ZFINX32-NEXT:    sw s11, 56(sp)
-; ZFINX32-NEXT:    sw s10, 52(sp)
+; ZFINX32-NEXT:    lw t0, 240(sp)
+; ZFINX32-NEXT:    lw t1, 244(sp)
+; ZFINX32-NEXT:    lw t2, 248(sp)
+; ZFINX32-NEXT:    lw t3, 252(sp)
+; ZFINX32-NEXT:    sw t0, 64(sp)
+; ZFINX32-NEXT:    sw t1, 68(sp)
+; ZFINX32-NEXT:    sw t2, 72(sp)
+; ZFINX32-NEXT:    sw t3, 76(sp)
 ; ZFINX32-NEXT:    sw s9, 48(sp)
-; ZFINX32-NEXT:    sw s8, 44(sp)
-; ZFINX32-NEXT:    sw s7, 40(sp)
-; ZFINX32-NEXT:    sw s6, 36(sp)
+; ZFINX32-NEXT:    sw s10, 52(sp)
+; ZFINX32-NEXT:    sw s11, 56(sp)
+; ZFINX32-NEXT:    sw ra, 60(sp)
 ; ZFINX32-NEXT:    sw s5, 32(sp)
-; ZFINX32-NEXT:    sw s4, 28(sp)
-; ZFINX32-NEXT:    sw s3, 24(sp)
-; ZFINX32-NEXT:    sw s2, 20(sp)
+; ZFINX32-NEXT:    sw s6, 36(sp)
+; ZFINX32-NEXT:    sw s7, 40(sp)
+; ZFINX32-NEXT:    sw s8, 44(sp)
 ; ZFINX32-NEXT:    sw s1, 16(sp)
-; ZFINX32-NEXT:    sw s0, 12(sp)
-; ZFINX32-NEXT:    sw t4, 8(sp)
-; ZFINX32-NEXT:    sw t5, 4(sp)
+; ZFINX32-NEXT:    sw s2, 20(sp)
+; ZFINX32-NEXT:    sw s3, 24(sp)
+; ZFINX32-NEXT:    sw s4, 28(sp)
 ; ZFINX32-NEXT:    sw t6, 0(sp)
+; ZFINX32-NEXT:    sw t4, 4(sp)
+; ZFINX32-NEXT:    sw t5, 8(sp)
+; ZFINX32-NEXT:    sw s0, 12(sp)
 ; ZFINX32-NEXT:    lw t3, 104(sp) # 4-byte Folded Reload
 ; ZFINX32-NEXT:    lw t4, 100(sp) # 4-byte Folded Reload
 ; ZFINX32-NEXT:    lw t5, 96(sp) # 4-byte Folded Reload
@@ -1138,8 +1138,8 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZFINX64-NEXT:    lw t0, 232(sp)
 ; ZFINX64-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
 ; ZFINX64-NEXT:    lw t6, 240(sp)
-; ZFINX64-NEXT:    lw t5, 248(sp)
-; ZFINX64-NEXT:    lw t4, 256(sp)
+; ZFINX64-NEXT:    lw t4, 248(sp)
+; ZFINX64-NEXT:    lw t5, 256(sp)
 ; ZFINX64-NEXT:    lw s0, 264(sp)
 ; ZFINX64-NEXT:    lw s1, 272(sp)
 ; ZFINX64-NEXT:    lw s2, 280(sp)
@@ -1153,30 +1153,30 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZFINX64-NEXT:    lw s10, 344(sp)
 ; ZFINX64-NEXT:    lw s11, 352(sp)
 ; ZFINX64-NEXT:    lw ra, 360(sp)
-; ZFINX64-NEXT:    lw t3, 368(sp)
-; ZFINX64-NEXT:    lw t2, 376(sp)
-; ZFINX64-NEXT:    lw t1, 384(sp)
-; ZFINX64-NEXT:    lw t0, 392(sp)
-; ZFINX64-NEXT:    sw t0, 76(sp)
-; ZFINX64-NEXT:    sw t1, 72(sp)
-; ZFINX64-NEXT:    sw t2, 68(sp)
-; ZFINX64-NEXT:    sw t3, 64(sp)
-; ZFINX64-NEXT:    sw ra, 60(sp)
-; ZFINX64-NEXT:    sw s11, 56(sp)
-; ZFINX64-NEXT:    sw s10, 52(sp)
+; ZFINX64-NEXT:    lw t0, 368(sp)
+; ZFINX64-NEXT:    lw t1, 376(sp)
+; ZFINX64-NEXT:    lw t2, 384(sp)
+; ZFINX64-NEXT:    lw t3, 392(sp)
+; ZFINX64-NEXT:    sw t0, 64(sp)
+; ZFINX64-NEXT:    sw t1, 68(sp)
+; ZFINX64-NEXT:    sw t2, 72(sp)
+; ZFINX64-NEXT:    sw t3, 76(sp)
 ; ZFINX64-NEXT:    sw s9, 48(sp)
-; ZFINX64-NEXT:    sw s8, 44(sp)
-; ZFINX64-NEXT:    sw s7, 40(sp)
-; ZFINX64-NEXT:    sw s6, 36(sp)
+; ZFINX64-NEXT:    sw s10, 52(sp)
+; ZFINX64-NEXT:    sw s11, 56(sp)
+; ZFINX64-NEXT:    sw ra, 60(sp)
 ; ZFINX64-NEXT:    sw s5, 32(sp)
-; ZFINX64-NEXT:    sw s4, 28(sp)
-; ZFINX64-NEXT:    sw s3, 24(sp)
-; ZFINX64-NEXT:    sw s2, 20(sp)
+; ZFINX64-NEXT:    sw s6, 36(sp)
+; ZFINX64-NEXT:    sw s7, 40(sp)
+; ZFINX64-NEXT:    sw s8, 44(sp)
 ; ZFINX64-NEXT:    sw s1, 16(sp)
-; ZFINX64-NEXT:    sw s0, 12(sp)
-; ZFINX64-NEXT:    sw t4, 8(sp)
-; ZFINX64-NEXT:    sw t5, 4(sp)
+; ZFINX64-NEXT:    sw s2, 20(sp)
+; ZFINX64-NEXT:    sw s3, 24(sp)
+; ZFINX64-NEXT:    sw s4, 28(sp)
 ; ZFINX64-NEXT:    sw t6, 0(sp)
+; ZFINX64-NEXT:    sw t4, 4(sp)
+; ZFINX64-NEXT:    sw t5, 8(sp)
+; ZFINX64-NEXT:    sw s0, 12(sp)
 ; ZFINX64-NEXT:    lw t3, 100(sp) # 4-byte Folded Reload
 ; ZFINX64-NEXT:    lw t4, 96(sp) # 4-byte Folded Reload
 ; ZFINX64-NEXT:    lw t5, 92(sp) # 4-byte Folded Reload
@@ -1223,8 +1223,8 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZDINX32-NEXT:    lw t0, 172(sp)
 ; ZDINX32-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
 ; ZDINX32-NEXT:    lw t6, 176(sp)
-; ZDINX32-NEXT:    lw t5, 180(sp)
-; ZDINX32-NEXT:    lw t4, 184(sp)
+; ZDINX32-NEXT:    lw t4, 180(sp)
+; ZDINX32-NEXT:    lw t5, 184(sp)
 ; ZDINX32-NEXT:    lw s0, 188(sp)
 ; ZDINX32-NEXT:    lw s1, 192(sp)
 ; ZDINX32-NEXT:    lw s2, 196(sp)
@@ -1238,30 +1238,30 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZDINX32-NEXT:    lw s10, 228(sp)
 ; ZDINX32-NEXT:    lw s11, 232(sp)
 ; ZDINX32-NEXT:    lw ra, 236(sp)
-; ZDINX32-NEXT:    lw t3, 240(sp)
-; ZDINX32-NEXT:    lw t2, 244(sp)
-; ZDINX32-NEXT:    lw t1, 248(sp)
-; ZDINX32-NEXT:    lw t0, 252(sp)
-; ZDINX32-NEXT:    sw t0, 76(sp)
-; ZDINX32-NEXT:    sw t1, 72(sp)
-; ZDINX32-NEXT:    sw t2, 68(sp)
-; ZDINX32-NEXT:    sw t3, 64(sp)
-; ZDINX32-NEXT:    sw ra, 60(sp)
-; ZDINX32-NEXT:    sw s11, 56(sp)
-; ZDINX32-NEXT:    sw s10, 52(sp)
+; ZDINX32-NEXT:    lw t0, 240(sp)
+; ZDINX32-NEXT:    lw t1, 244(sp)
+; ZDINX32-NEXT:    lw t2, 248(sp)
+; ZDINX32-NEXT:    lw t3, 252(sp)
+; ZDINX32-NEXT:    sw t0, 64(sp)
+; ZDINX32-NEXT:    sw t1, 68(sp)
+; ZDINX32-NEXT:    sw t2, 72(sp)
+; ZDINX32-NEXT:    sw t3, 76(sp)
 ; ZDINX32-NEXT:    sw s9, 48(sp)
-; ZDINX32-NEXT:    sw s8, 44(sp)
-; ZDINX32-NEXT:    sw s7, 40(sp)
-; ZDINX32-NEXT:    sw s6, 36(sp)
+; ZDINX32-NEXT:    sw s10, 52(sp)
+; ZDINX32-NEXT:    sw s11, 56(sp)
+; ZDINX32-NEXT:    sw ra, 60(sp)
 ; ZDINX32-NEXT:    sw s5, 32(sp)
-; ZDINX32-NEXT:    sw s4, 28(sp)
-; ZDINX32-NEXT:    sw s3, 24(sp)
-; ZDINX32-NEXT:    sw s2, 20(sp)
+; ZDINX32-NEXT:    sw s6, 36(sp)
+; ZDINX32-NEXT:    sw s7, 40(sp)
+; ZDINX32-NEXT:    sw s8, 44(sp)
 ; ZDINX32-NEXT:    sw s1, 16(sp)
-; ZDINX32-NEXT:    sw s0, 12(sp)
-; ZDINX32-NEXT:    sw t4, 8(sp)
-; ZDINX32-NEXT:    sw t5, 4(sp)
+; ZDINX32-NEXT:    sw s2, 20(sp)
+; ZDINX32-NEXT:    sw s3, 24(sp)
+; ZDINX32-NEXT:    sw s4, 28(sp)
 ; ZDINX32-NEXT:    sw t6, 0(sp)
+; ZDINX32-NEXT:    sw t4, 4(sp)
+; ZDINX32-NEXT:    sw t5, 8(sp)
+; ZDINX32-NEXT:    sw s0, 12(sp)
 ; ZDINX32-NEXT:    lw t3, 104(sp) # 4-byte Folded Reload
 ; ZDINX32-NEXT:    lw t4, 100(sp) # 4-byte Folded Reload
 ; ZDINX32-NEXT:    lw t5, 96(sp) # 4-byte Folded Reload
@@ -1308,8 +1308,8 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZDINX64-NEXT:    lw t0, 232(sp)
 ; ZDINX64-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
 ; ZDINX64-NEXT:    lw t6, 240(sp)
-; ZDINX64-NEXT:    lw t5, 248(sp)
-; ZDINX64-NEXT:    lw t4, 256(sp)
+; ZDINX64-NEXT:    lw t4, 248(sp)
+; ZDINX64-NEXT:    lw t5, 256(sp)
 ; ZDINX64-NEXT:    lw s0, 264(sp)
 ; ZDINX64-NEXT:    lw s1, 272(sp)
 ; ZDINX64-NEXT:    lw s2, 280(sp)
@@ -1323,30 +1323,30 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ; ZDINX64-NEXT:    lw s10, 344(sp)
 ; ZDINX64-NEXT:    lw s11, 352(sp)
 ; ZDINX64-NEXT:    lw ra, 360(sp)
-; ZDINX64-NEXT:    lw t3, 368(sp)
-; ZDINX64-NEXT:    lw t2, 376(sp)
-; ZDINX64-NEXT:    lw t1, 384(sp)
-; ZDINX64-NEXT:    lw t0, 392(sp)
-; ZDINX64-NEXT:    sw t0, 76(sp)
-; ZDINX64-NEXT:    sw t1, 72(sp)
-; ZDINX64-NEXT:    sw t2, 68(sp)
-; ZDINX64-NEXT:    sw t3, 64(sp)
-; ZDINX64-NEXT:    sw ra, 60(sp)
-; ZDINX64-NEXT:    sw s11, 56(sp)
-; ZDINX64-NEXT:    sw s10, 52(sp)
+; ZDINX64-NEXT:    lw t0, 368(sp)
+; ZDINX64-NEXT:    lw t1, 376(sp)
+; ZDINX64-NEXT:    lw t2, 384(sp)
+; ZDINX64-NEXT:    lw t3, 392(sp)
+; ZDINX64-NEXT:    sw t0, 64(sp)
+; ZDINX64-NEXT:    sw t1, 68(sp)
+; ZDINX64-NEXT:    sw t2, 72(sp)
+; ZDINX64-NEXT:    sw t3, 76(sp)
 ; ZDINX64-NEXT:    sw s9, 48(sp)
-; ZDINX64-NEXT:    sw s8, 44(sp)
-; ZDINX64-NEXT:    sw s7, 40(sp)
-; ZDINX64-NEXT:    sw s6, 36(sp)
+; ZDINX64-NEXT:    sw s10, 52(sp)
+; ZDINX64-NEXT:    sw s11, 56(sp)
+; ZDINX64-NEXT:    sw ra, 60(sp)
 ; ZDINX64-NEXT:    sw s5, 32(sp)
-; ZDINX64-NEXT:    sw s4, 28(sp)
-; ZDINX64-NEXT:    sw s3, 24(sp)
-; ZDINX64-NEXT:    sw s2, 20(sp)
+; ZDINX64-NEXT:    sw s6, 36(sp)
+; ZDINX64-NEXT:    sw s7, 40(sp)
+; ZDINX64-NEXT:    sw s8, 44(sp)
 ; ZDINX64-NEXT:    sw s1, 16(sp)
-; ZDINX64-NEXT:    sw s0, 12(sp)
-; ZDINX64-NEXT:    sw t4, 8(sp)
-; ZDINX64-NEXT:    sw t5, 4(sp)
+; ZDINX64-NEXT:    sw s2, 20(sp)
+; ZDINX64-NEXT:    sw s3, 24(sp)
+; ZDINX64-NEXT:    sw s4, 28(sp)
 ; ZDINX64-NEXT:    sw t6, 0(sp)
+; ZDINX64-NEXT:    sw t4, 4(sp)
+; ZDINX64-NEXT:    sw t5, 8(sp)
+; ZDINX64-NEXT:    sw s0, 12(sp)
 ; ZDINX64-NEXT:    lw t3, 100(sp) # 4-byte Folded Reload
 ; ZDINX64-NEXT:    lw t4, 96(sp) # 4-byte Folded Reload
 ; ZDINX64-NEXT:    lw t5, 92(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll b/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
index 4d4580188096f0508d79ceda796968d6acc8cc8b..a204b928304123ff1b2cf5f621177faa77faa725 100644
--- a/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
+++ b/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
@@ -293,8 +293,8 @@ define dso_local void @store_g_4() nounwind {
 ; RV32I-MEDIUM-NEXT:  .Lpcrel_hi6:
 ; RV32I-MEDIUM-NEXT:    auipc a0, %pcrel_hi(g_4)
 ; RV32I-MEDIUM-NEXT:    addi a0, a0, %pcrel_lo(.Lpcrel_hi6)
-; RV32I-MEDIUM-NEXT:    sw zero, 4(a0)
 ; RV32I-MEDIUM-NEXT:    sw zero, 0(a0)
+; RV32I-MEDIUM-NEXT:    sw zero, 4(a0)
 ; RV32I-MEDIUM-NEXT:    ret
 ;
 ; RV64I-LABEL: store_g_4:
@@ -335,8 +335,8 @@ define dso_local void @store_g_8() nounwind {
 ; RV32I-MEDIUM-NEXT:  .Lpcrel_hi7:
 ; RV32I-MEDIUM-NEXT:    auipc a0, %pcrel_hi(g_8)
 ; RV32I-MEDIUM-NEXT:    addi a0, a0, %pcrel_lo(.Lpcrel_hi7)
-; RV32I-MEDIUM-NEXT:    sw zero, 4(a0)
 ; RV32I-MEDIUM-NEXT:    sw zero, 0(a0)
+; RV32I-MEDIUM-NEXT:    sw zero, 4(a0)
 ; RV32I-MEDIUM-NEXT:    ret
 ;
 ; RV64I-LABEL: store_g_8:
diff --git a/llvm/test/CodeGen/RISCV/forced-atomics.ll b/llvm/test/CodeGen/RISCV/forced-atomics.ll
index 603491bf3d3003d7b45cfc1a963156ca3b504452..133d735a46ed7f53c992be500d3bc473e15da8ba 100644
--- a/llvm/test/CodeGen/RISCV/forced-atomics.ll
+++ b/llvm/test/CodeGen/RISCV/forced-atomics.ll
@@ -4338,8 +4338,8 @@ define i64 @cmpxchg64_monotonic(ptr %p) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw zero, 4(sp)
 ; RV32-NEXT:    sw zero, 0(sp)
+; RV32-NEXT:    sw zero, 4(sp)
 ; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    li a2, 1
 ; RV32-NEXT:    li a3, 0
@@ -4398,8 +4398,8 @@ define i64 @cmpxchg64_seq_cst(ptr %p) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw zero, 4(sp)
 ; RV32-NEXT:    sw zero, 0(sp)
+; RV32-NEXT:    sw zero, 4(sp)
 ; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    li a2, 1
 ; RV32-NEXT:    li a4, 5
@@ -4464,14 +4464,14 @@ define i128 @load128(ptr %p) nounwind {
 ; RV32-NEXT:    addi a2, sp, 8
 ; RV32-NEXT:    li a3, 5
 ; RV32-NEXT:    call __atomic_load
-; RV32-NEXT:    lw a0, 20(sp)
-; RV32-NEXT:    lw a1, 16(sp)
-; RV32-NEXT:    lw a2, 12(sp)
-; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    sw a0, 12(s0)
-; RV32-NEXT:    sw a1, 8(s0)
-; RV32-NEXT:    sw a2, 4(s0)
-; RV32-NEXT:    sw a3, 0(s0)
+; RV32-NEXT:    lw a0, 8(sp)
+; RV32-NEXT:    lw a1, 12(sp)
+; RV32-NEXT:    lw a2, 16(sp)
+; RV32-NEXT:    lw a3, 20(sp)
+; RV32-NEXT:    sw a0, 0(s0)
+; RV32-NEXT:    sw a1, 4(s0)
+; RV32-NEXT:    sw a2, 8(s0)
+; RV32-NEXT:    sw a3, 12(s0)
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 32
@@ -4496,10 +4496,10 @@ define void @store128(ptr %p) nounwind {
 ; RV32-NEXT:    addi sp, sp, -32
 ; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    mv a1, a0
-; RV32-NEXT:    sw zero, 20(sp)
-; RV32-NEXT:    sw zero, 16(sp)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw zero, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
+; RV32-NEXT:    sw zero, 16(sp)
+; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    li a0, 16
 ; RV32-NEXT:    addi a2, sp, 8
 ; RV32-NEXT:    li a3, 5
@@ -4550,8 +4550,8 @@ define i128 @rmw128(ptr %p) nounwind {
 ; RV32-NEXT:    sw a2, 20(sp)
 ; RV32-NEXT:    sw a3, 24(sp)
 ; RV32-NEXT:    sw a4, 28(sp)
-; RV32-NEXT:    sw a5, 4(sp)
 ; RV32-NEXT:    sw a0, 0(sp)
+; RV32-NEXT:    sw a5, 4(sp)
 ; RV32-NEXT:    sw a6, 8(sp)
 ; RV32-NEXT:    sw a7, 12(sp)
 ; RV32-NEXT:    li a0, 16
@@ -4599,29 +4599,29 @@ define i128 @cmpxchg128(ptr %p) nounwind {
 ; RV32-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    mv s0, a0
-; RV32-NEXT:    sw zero, 36(sp)
-; RV32-NEXT:    sw zero, 32(sp)
-; RV32-NEXT:    sw zero, 28(sp)
 ; RV32-NEXT:    sw zero, 24(sp)
-; RV32-NEXT:    sw zero, 20(sp)
-; RV32-NEXT:    sw zero, 16(sp)
-; RV32-NEXT:    sw zero, 12(sp)
+; RV32-NEXT:    sw zero, 28(sp)
+; RV32-NEXT:    sw zero, 32(sp)
+; RV32-NEXT:    sw zero, 36(sp)
 ; RV32-NEXT:    li a0, 1
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
+; RV32-NEXT:    sw zero, 16(sp)
+; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    li a0, 16
 ; RV32-NEXT:    addi a2, sp, 24
 ; RV32-NEXT:    addi a3, sp, 8
 ; RV32-NEXT:    li a4, 5
 ; RV32-NEXT:    li a5, 5
 ; RV32-NEXT:    call __atomic_compare_exchange
-; RV32-NEXT:    lw a0, 36(sp)
-; RV32-NEXT:    lw a1, 32(sp)
-; RV32-NEXT:    lw a2, 28(sp)
-; RV32-NEXT:    lw a3, 24(sp)
-; RV32-NEXT:    sw a0, 12(s0)
-; RV32-NEXT:    sw a1, 8(s0)
-; RV32-NEXT:    sw a2, 4(s0)
-; RV32-NEXT:    sw a3, 0(s0)
+; RV32-NEXT:    lw a0, 24(sp)
+; RV32-NEXT:    lw a1, 28(sp)
+; RV32-NEXT:    lw a2, 32(sp)
+; RV32-NEXT:    lw a3, 36(sp)
+; RV32-NEXT:    sw a0, 0(s0)
+; RV32-NEXT:    sw a1, 4(s0)
+; RV32-NEXT:    sw a2, 8(s0)
+; RV32-NEXT:    sw a3, 12(s0)
 ; RV32-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 48
@@ -4631,8 +4631,8 @@ define i128 @cmpxchg128(ptr %p) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi sp, sp, -32
 ; RV64-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd zero, 8(sp)
 ; RV64-NEXT:    sd zero, 0(sp)
+; RV64-NEXT:    sd zero, 8(sp)
 ; RV64-NEXT:    mv a1, sp
 ; RV64-NEXT:    li a2, 1
 ; RV64-NEXT:    li a4, 5
diff --git a/llvm/test/CodeGen/RISCV/fp-fcanonicalize.ll b/llvm/test/CodeGen/RISCV/fp-fcanonicalize.ll
new file mode 100644
index 0000000000000000000000000000000000000000..94954d77fd0c466d79b0772a516fead38038b60f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/fp-fcanonicalize.ll
@@ -0,0 +1,1262 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; FIXME: @llvm.canonicalize doesn't support soft-float abi yet.
+; RUN: llc --mtriple=riscv64 --mattr=+d,+zfh < %s | FileCheck %s --check-prefixes=CHECK,CHECK-FP16-RV64
+; RUN: llc --mtriple=riscv64 --mattr=+d,-zfh < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16-RV64
+; RUN: llc --mtriple=riscv32 --mattr=+d,+zfh < %s | FileCheck %s --check-prefixes=CHECK,CHECK-FP16-RV32
+; RUN: llc --mtriple=riscv32 --mattr=+d,-zfh < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16-RV32
+
+declare half @llvm.fcanonicalize.f16(half)
+declare float @llvm.fcanonicalize.f32(float)
+declare double @llvm.fcanonicalize.f64(double)
+
+define half @fcanonicalize_f16(half %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_f16:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_f16:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV64-NEXT:    lui a1, 1048560
+; CHECK-NOFP16-RV64-NEXT:    or a0, a0, a1
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 16
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_f16:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_f16:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV32-NEXT:    lui a1, 1048560
+; CHECK-NOFP16-RV32-NEXT:    or a0, a0, a1
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 16
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call half @llvm.canonicalize.f16(half %x)
+  ret half %z
+}
+
+define half @fcanonicalize_f16_nnan(half %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_f16_nnan:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_f16_nnan:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV64-NEXT:    lui a1, 1048560
+; CHECK-NOFP16-RV64-NEXT:    or a0, a0, a1
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 16
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_f16_nnan:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_f16_nnan:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV32-NEXT:    lui a1, 1048560
+; CHECK-NOFP16-RV32-NEXT:    or a0, a0, a1
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 16
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call nnan half @llvm.canonicalize.f16(half %x)
+  ret half %z
+}
+
+define <2 x half> @fcanonicalize_v2f16(<2 x half> %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_v2f16:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_v2f16:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -32
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s0, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s1, -24
+; CHECK-NOFP16-RV64-NEXT:    mv s0, a1
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s0
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a1, fa0
+; CHECK-NOFP16-RV64-NEXT:    mv a0, s1
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 32
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_v2f16:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_v2f16:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s0, -8
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs0, -16
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs0, a1
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s0, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a1, fa0
+; CHECK-NOFP16-RV32-NEXT:    mv a0, s0
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 16
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %x)
+  ret <2 x half> %z
+}
+
+define <2 x half> @fcanonicalize_v2f16_nnan(<2 x half> %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_v2f16_nnan:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_v2f16_nnan:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -32
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s0, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s1, -24
+; CHECK-NOFP16-RV64-NEXT:    mv s0, a1
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s0
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a1, fa0
+; CHECK-NOFP16-RV64-NEXT:    mv a0, s1
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 32
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_v2f16_nnan:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_v2f16_nnan:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s0, -8
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs0, -16
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs0, a1
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s0, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a1, fa0
+; CHECK-NOFP16-RV32-NEXT:    mv a0, s0
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 16
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call nnan <2 x half> @llvm.canonicalize.v2f16(<2 x half> %x)
+  ret <2 x half> %z
+}
+
+define <4 x half> @fcanonicalize_v4f16(<4 x half> %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_v4f16:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa5, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    fmin.h fa4, fa1, fa1
+; CHECK-FP16-RV64-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV64-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV64-NEXT:    fsh fa5, 0(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa4, 2(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_v4f16:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -64
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs0, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs2, 0(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s0, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s1, -24
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s2, -32
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s3, -40
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs0, -48
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs1, -56
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs2, -64
+; CHECK-NOFP16-RV64-NEXT:    lhu s1, 0(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s2, 8(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s3, 16(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu a1, 24(a1)
+; CHECK-NOFP16-RV64-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs0, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s3
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s2
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s2, fs2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fs0, fs0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s3, fs1
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV64-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh a0, 6(s0)
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs0, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs2, 0(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 64
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_v4f16:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa5, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    fmin.h fa4, fa1, fa1
+; CHECK-FP16-RV32-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV32-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV32-NEXT:    fsh fa5, 0(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa4, 2(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_v4f16:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -64
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s1, 52(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s2, 48(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s3, 44(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs0, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs1, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs2, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs3, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s0, -8
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s1, -12
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s2, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s3, -20
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs0, -32
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs1, -40
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs2, -48
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs3, -56
+; CHECK-NOFP16-RV32-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV32-NEXT:    lhu a0, 0(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a2, 4(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a3, 8(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a1, 12(a1)
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs0, a0
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs1, a2
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs2, a3
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs3, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs2
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs1
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s2, fs1
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fs3, fs3
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s3, fs2
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV32-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh a0, 6(s0)
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s1, 52(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s2, 48(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s3, 44(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs0, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs1, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs2, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs3, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 64
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call <4 x half> @llvm.canonicalize.v4f16(<4 x half> %x)
+  ret <4 x half> %z
+}
+
+define <4 x half> @fcanonicalize_v4f16_nnan(<4 x half> %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_v4f16_nnan:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa5, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    fmin.h fa4, fa1, fa1
+; CHECK-FP16-RV64-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV64-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV64-NEXT:    fsh fa5, 0(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa4, 2(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_v4f16_nnan:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -64
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs0, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs2, 0(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s0, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s1, -24
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s2, -32
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s3, -40
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs0, -48
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs1, -56
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs2, -64
+; CHECK-NOFP16-RV64-NEXT:    lhu s1, 0(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s2, 8(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s3, 16(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu a1, 24(a1)
+; CHECK-NOFP16-RV64-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs0, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s3
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s2
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s2, fs2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fs0, fs0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s3, fs1
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV64-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh a0, 6(s0)
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs0, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs2, 0(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 64
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_v4f16_nnan:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa5, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    fmin.h fa4, fa1, fa1
+; CHECK-FP16-RV32-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV32-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV32-NEXT:    fsh fa5, 0(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa4, 2(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_v4f16_nnan:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -64
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s1, 52(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s2, 48(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s3, 44(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs0, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs1, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs2, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs3, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s0, -8
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s1, -12
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s2, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s3, -20
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs0, -32
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs1, -40
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs2, -48
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs3, -56
+; CHECK-NOFP16-RV32-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV32-NEXT:    lhu a0, 0(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a2, 4(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a3, 8(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a1, 12(a1)
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs0, a0
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs1, a2
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs2, a3
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs3, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs2
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs1
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s2, fs1
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fs3, fs3
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s3, fs2
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV32-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh a0, 6(s0)
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s1, 52(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s2, 48(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s3, 44(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs0, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs1, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs2, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs3, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 64
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call nnan <4 x half> @llvm.canonicalize.v4f16(<4 x half> %x)
+  ret <4 x half> %z
+}
+
+define <8 x half> @fcanonicalize_v8f16(<8 x half> %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_v8f16:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV64-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV64-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV64-NEXT:    fmin.h fa4, fa4, fa4
+; CHECK-FP16-RV64-NEXT:    fmin.h fa5, fa5, fa5
+; CHECK-FP16-RV64-NEXT:    fmin.h ft0, fa6, fa6
+; CHECK-FP16-RV64-NEXT:    fmin.h ft1, fa7, fa7
+; CHECK-FP16-RV64-NEXT:    fsh fa4, 8(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa5, 10(a0)
+; CHECK-FP16-RV64-NEXT:    fsh ft0, 12(a0)
+; CHECK-FP16-RV64-NEXT:    fsh ft1, 14(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa0, 0(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa1, 2(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_v8f16:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -128
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 128
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s1, 104(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s2, 96(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s3, 88(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s4, 80(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s5, 72(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s6, 64(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s7, 56(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs0, 48(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs1, 40(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs2, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs3, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs4, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs5, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs6, 0(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s0, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s1, -24
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s2, -32
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s3, -40
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s4, -48
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s5, -56
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s6, -64
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s7, -72
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs0, -80
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs1, -88
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs2, -96
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs3, -104
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs4, -112
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs5, -120
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs6, -128
+; CHECK-NOFP16-RV64-NEXT:    lhu s1, 0(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s2, 8(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s3, 16(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s4, 24(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s5, 32(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s6, 40(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s7, 48(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu a1, 56(a1)
+; CHECK-NOFP16-RV64-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs0, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s7
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s6
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s5
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs3, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s4
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs4, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s3
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs5, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s2
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs6, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s2, fs6
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s3, fs5
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s4, fs4
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s5, fs3
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s6, fs2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fs0, fs0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s7, fs1
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV64-NEXT:    sh s5, 8(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s6, 10(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s7, 12(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh a0, 14(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s4, 6(s0)
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s2, 96(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s3, 88(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s4, 80(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s5, 72(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s6, 64(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s7, 56(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs0, 48(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs1, 40(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs2, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs3, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs4, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs5, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs6, 0(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 128
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_v8f16:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV32-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV32-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV32-NEXT:    fmin.h fa4, fa4, fa4
+; CHECK-FP16-RV32-NEXT:    fmin.h fa5, fa5, fa5
+; CHECK-FP16-RV32-NEXT:    fmin.h ft0, fa6, fa6
+; CHECK-FP16-RV32-NEXT:    fmin.h ft1, fa7, fa7
+; CHECK-FP16-RV32-NEXT:    fsh fa4, 8(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa5, 10(a0)
+; CHECK-FP16-RV32-NEXT:    fsh ft0, 12(a0)
+; CHECK-FP16-RV32-NEXT:    fsh ft1, 14(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa0, 0(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa1, 2(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_v8f16:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -112
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 108(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s0, 104(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s1, 100(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s2, 96(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s3, 92(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s4, 88(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s5, 84(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s6, 80(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s7, 76(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs0, 64(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs1, 56(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs2, 48(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs3, 40(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs4, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs5, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs6, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs7, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s0, -8
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s1, -12
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s2, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s3, -20
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s4, -24
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s5, -28
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s6, -32
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s7, -36
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs0, -48
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs1, -56
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs2, -64
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs3, -72
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs4, -80
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs5, -88
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs6, -96
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs7, -104
+; CHECK-NOFP16-RV32-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV32-NEXT:    lhu a0, 0(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a2, 4(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a3, 8(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a4, 12(a1)
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs0, a0
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs2, a2
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs4, a3
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs5, a4
+; CHECK-NOFP16-RV32-NEXT:    lhu a0, 16(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a2, 20(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a3, 24(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a1, 28(a1)
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs6, a0
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs7, a2
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs3, a3
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs3
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs3, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs7
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs7, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs6
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs6, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs5
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs5, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs4
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs4, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs2
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s2, fs2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s3, fs4
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s4, fs5
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s5, fs6
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s6, fs7
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fs1, fs1
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s7, fs3
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV32-NEXT:    sh s5, 8(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s6, 10(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s7, 12(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh a0, 14(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s4, 6(s0)
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 108(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s0, 104(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s1, 100(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s2, 96(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s3, 92(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s4, 88(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s5, 84(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s6, 80(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s7, 76(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs0, 64(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs1, 56(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs2, 48(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs3, 40(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs4, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs5, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs6, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs7, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 112
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call <8 x half> @llvm.canonicalize.v8f16(<8 x half> %x)
+  ret <8 x half> %z
+}
+
+define <8 x half> @fcanonicalize_v8f16_nnan(<8 x half> %x) {
+; CHECK-FP16-RV64-LABEL: fcanonicalize_v8f16_nnan:
+; CHECK-FP16-RV64:       # %bb.0:
+; CHECK-FP16-RV64-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV64-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV64-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV64-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV64-NEXT:    fmin.h fa4, fa4, fa4
+; CHECK-FP16-RV64-NEXT:    fmin.h fa5, fa5, fa5
+; CHECK-FP16-RV64-NEXT:    fmin.h ft0, fa6, fa6
+; CHECK-FP16-RV64-NEXT:    fmin.h ft1, fa7, fa7
+; CHECK-FP16-RV64-NEXT:    fsh fa4, 8(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa5, 10(a0)
+; CHECK-FP16-RV64-NEXT:    fsh ft0, 12(a0)
+; CHECK-FP16-RV64-NEXT:    fsh ft1, 14(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa0, 0(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa1, 2(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV64-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV64-NEXT:    ret
+;
+; CHECK-NOFP16-RV64-LABEL: fcanonicalize_v8f16_nnan:
+; CHECK-NOFP16-RV64:       # %bb.0:
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, -128
+; CHECK-NOFP16-RV64-NEXT:    .cfi_def_cfa_offset 128
+; CHECK-NOFP16-RV64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s1, 104(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s2, 96(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s3, 88(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s4, 80(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s5, 72(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s6, 64(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    sd s7, 56(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs0, 48(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs1, 40(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs2, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs3, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs4, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs5, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    fsd fs6, 0(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset ra, -8
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s0, -16
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s1, -24
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s2, -32
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s3, -40
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s4, -48
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s5, -56
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s6, -64
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset s7, -72
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs0, -80
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs1, -88
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs2, -96
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs3, -104
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs4, -112
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs5, -120
+; CHECK-NOFP16-RV64-NEXT:    .cfi_offset fs6, -128
+; CHECK-NOFP16-RV64-NEXT:    lhu s1, 0(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s2, 8(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s3, 16(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s4, 24(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s5, 32(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s6, 40(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu s7, 48(a1)
+; CHECK-NOFP16-RV64-NEXT:    lhu a1, 56(a1)
+; CHECK-NOFP16-RV64-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs0, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s7
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s6
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s5
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs3, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s4
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs4, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s3
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs5, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s2
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.s fs6, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.w.x fa0, s1
+; CHECK-NOFP16-RV64-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s2, fs6
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s3, fs5
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s4, fs4
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s5, fs3
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s6, fs2
+; CHECK-NOFP16-RV64-NEXT:    fmin.s fa0, fs0, fs0
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w s7, fs1
+; CHECK-NOFP16-RV64-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV64-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV64-NEXT:    sh s5, 8(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s6, 10(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s7, 12(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh a0, 14(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV64-NEXT:    sh s4, 6(s0)
+; CHECK-NOFP16-RV64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s2, 96(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s3, 88(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s4, 80(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s5, 72(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s6, 64(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    ld s7, 56(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs0, 48(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs1, 40(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs2, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs3, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs4, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs5, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    fld fs6, 0(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV64-NEXT:    addi sp, sp, 128
+; CHECK-NOFP16-RV64-NEXT:    ret
+;
+; CHECK-FP16-RV32-LABEL: fcanonicalize_v8f16_nnan:
+; CHECK-FP16-RV32:       # %bb.0:
+; CHECK-FP16-RV32-NEXT:    fmin.h fa0, fa0, fa0
+; CHECK-FP16-RV32-NEXT:    fmin.h fa1, fa1, fa1
+; CHECK-FP16-RV32-NEXT:    fmin.h fa2, fa2, fa2
+; CHECK-FP16-RV32-NEXT:    fmin.h fa3, fa3, fa3
+; CHECK-FP16-RV32-NEXT:    fmin.h fa4, fa4, fa4
+; CHECK-FP16-RV32-NEXT:    fmin.h fa5, fa5, fa5
+; CHECK-FP16-RV32-NEXT:    fmin.h ft0, fa6, fa6
+; CHECK-FP16-RV32-NEXT:    fmin.h ft1, fa7, fa7
+; CHECK-FP16-RV32-NEXT:    fsh fa4, 8(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa5, 10(a0)
+; CHECK-FP16-RV32-NEXT:    fsh ft0, 12(a0)
+; CHECK-FP16-RV32-NEXT:    fsh ft1, 14(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa0, 0(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa1, 2(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa2, 4(a0)
+; CHECK-FP16-RV32-NEXT:    fsh fa3, 6(a0)
+; CHECK-FP16-RV32-NEXT:    ret
+;
+; CHECK-NOFP16-RV32-LABEL: fcanonicalize_v8f16_nnan:
+; CHECK-NOFP16-RV32:       # %bb.0:
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, -112
+; CHECK-NOFP16-RV32-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NOFP16-RV32-NEXT:    sw ra, 108(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s0, 104(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s1, 100(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s2, 96(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s3, 92(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s4, 88(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s5, 84(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s6, 80(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    sw s7, 76(sp) # 4-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs0, 64(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs1, 56(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs2, 48(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs3, 40(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs4, 32(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs5, 24(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs6, 16(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    fsd fs7, 8(sp) # 8-byte Folded Spill
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset ra, -4
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s0, -8
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s1, -12
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s2, -16
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s3, -20
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s4, -24
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s5, -28
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s6, -32
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset s7, -36
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs0, -48
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs1, -56
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs2, -64
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs3, -72
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs4, -80
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs5, -88
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs6, -96
+; CHECK-NOFP16-RV32-NEXT:    .cfi_offset fs7, -104
+; CHECK-NOFP16-RV32-NEXT:    mv s0, a0
+; CHECK-NOFP16-RV32-NEXT:    lhu a0, 0(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a2, 4(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a3, 8(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a4, 12(a1)
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs0, a0
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs2, a2
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs4, a3
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs5, a4
+; CHECK-NOFP16-RV32-NEXT:    lhu a0, 16(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a2, 20(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a3, 24(a1)
+; CHECK-NOFP16-RV32-NEXT:    lhu a1, 28(a1)
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs6, a0
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs7, a2
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fs3, a3
+; CHECK-NOFP16-RV32-NEXT:    fmv.w.x fa0, a1
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs3
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs3, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs7
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs7, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs6
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs6, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs5
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs5, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs4
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs4, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs2
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fs2, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.s fa0, fs0
+; CHECK-NOFP16-RV32-NEXT:    call __extendhfsf2
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s1, fa0
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s2, fs2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s3, fs4
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s4, fs5
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s5, fs6
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s6, fs7
+; CHECK-NOFP16-RV32-NEXT:    fmin.s fa0, fs1, fs1
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w s7, fs3
+; CHECK-NOFP16-RV32-NEXT:    call __truncsfhf2
+; CHECK-NOFP16-RV32-NEXT:    fmv.x.w a0, fa0
+; CHECK-NOFP16-RV32-NEXT:    sh s5, 8(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s6, 10(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s7, 12(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh a0, 14(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s1, 0(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s2, 2(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s3, 4(s0)
+; CHECK-NOFP16-RV32-NEXT:    sh s4, 6(s0)
+; CHECK-NOFP16-RV32-NEXT:    lw ra, 108(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s0, 104(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s1, 100(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s2, 96(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s3, 92(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s4, 88(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s5, 84(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s6, 80(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    lw s7, 76(sp) # 4-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs0, 64(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs1, 56(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs2, 48(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs3, 40(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs4, 32(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs5, 24(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs6, 16(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    fld fs7, 8(sp) # 8-byte Folded Reload
+; CHECK-NOFP16-RV32-NEXT:    addi sp, sp, 112
+; CHECK-NOFP16-RV32-NEXT:    ret
+  %z = call nnan <8 x half> @llvm.canonicalize.v8f16(<8 x half> %x)
+  ret <8 x half> %z
+}
+
+define float @fcanonicalize_f32(float %x) {
+; CHECK-LABEL: fcanonicalize_f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NEXT:    ret
+  %z = call float @llvm.canonicalize.f32(float %x)
+  ret float %z
+}
+
+define float @fcanonicalize_f32_nnan(float %x) {
+; CHECK-LABEL: fcanonicalize_f32_nnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NEXT:    ret
+  %z = call nnan float @llvm.canonicalize.f32(float %x)
+  ret float %z
+}
+
+define <2 x float> @fcanonicalize_v2f32(<2 x float> %x) {
+; CHECK-LABEL: fcanonicalize_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NEXT:    fmin.s fa1, fa1, fa1
+; CHECK-NEXT:    ret
+  %z = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> %x)
+  ret <2 x float> %z
+}
+
+define <2 x float> @fcanonicalize_v2f32_nnan(<2 x float> %x) {
+; CHECK-LABEL: fcanonicalize_v2f32_nnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.s fa0, fa0, fa0
+; CHECK-NEXT:    fmin.s fa1, fa1, fa1
+; CHECK-NEXT:    ret
+  %z = call nnan <2 x float> @llvm.canonicalize.v2f32(<2 x float> %x)
+  ret <2 x float> %z
+}
+
+define <4 x float> @fcanonicalize_v4f32(<4 x float> %x) {
+; CHECK-LABEL: fcanonicalize_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.s fa5, fa0, fa0
+; CHECK-NEXT:    fmin.s fa4, fa1, fa1
+; CHECK-NEXT:    fmin.s fa2, fa2, fa2
+; CHECK-NEXT:    fmin.s fa3, fa3, fa3
+; CHECK-NEXT:    fsw fa5, 0(a0)
+; CHECK-NEXT:    fsw fa4, 4(a0)
+; CHECK-NEXT:    fsw fa2, 8(a0)
+; CHECK-NEXT:    fsw fa3, 12(a0)
+; CHECK-NEXT:    ret
+  %z = call <4 x float> @llvm.canonicalize.v4f32(<4 x float> %x)
+  ret <4 x float> %z
+}
+
+define <4 x float> @fcanonicalize_v4f32_nnan(<4 x float> %x) {
+; CHECK-LABEL: fcanonicalize_v4f32_nnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.s fa5, fa0, fa0
+; CHECK-NEXT:    fmin.s fa4, fa1, fa1
+; CHECK-NEXT:    fmin.s fa2, fa2, fa2
+; CHECK-NEXT:    fmin.s fa3, fa3, fa3
+; CHECK-NEXT:    fsw fa5, 0(a0)
+; CHECK-NEXT:    fsw fa4, 4(a0)
+; CHECK-NEXT:    fsw fa2, 8(a0)
+; CHECK-NEXT:    fsw fa3, 12(a0)
+; CHECK-NEXT:    ret
+  %z = call nnan <4 x float> @llvm.canonicalize.v4f32(<4 x float> %x)
+  ret <4 x float> %z
+}
+
+define double @fcanonicalize_f64(double %x) {
+; CHECK-LABEL: fcanonicalize_f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.d fa0, fa0, fa0
+; CHECK-NEXT:    ret
+  %z = call double @llvm.canonicalize.f64(double %x)
+  ret double %z
+}
+
+define double @fcanonicalize_f64_nnan(double %x) {
+; CHECK-LABEL: fcanonicalize_f64_nnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.d fa0, fa0, fa0
+; CHECK-NEXT:    ret
+  %z = call nnan double @llvm.canonicalize.f64(double %x)
+  ret double %z
+}
+
+define <2 x double> @fcanonicalize_v2f64(<2 x double> %x) {
+; CHECK-LABEL: fcanonicalize_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.d fa0, fa0, fa0
+; CHECK-NEXT:    fmin.d fa1, fa1, fa1
+; CHECK-NEXT:    ret
+  %z = call <2 x double> @llvm.canonicalize.v2f64(<2 x double> %x)
+  ret <2 x double> %z
+}
+
+define <2 x double> @fcanonicalize_v2f64_nnan(<2 x double> %x) {
+; CHECK-LABEL: fcanonicalize_v2f64_nnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmin.d fa0, fa0, fa0
+; CHECK-NEXT:    fmin.d fa1, fa1, fa1
+; CHECK-NEXT:    ret
+  %z = call nnan <2 x double> @llvm.canonicalize.v2f64(<2 x double> %x)
+  ret <2 x double> %z
+}
diff --git a/llvm/test/CodeGen/RISCV/fp128.ll b/llvm/test/CodeGen/RISCV/fp128.ll
index 611a70c0ad8b939ca5b9333fb25a085c971a5e85..0bde85b54e5d1591021d3e9a02d7a4a827b83b4f 100644
--- a/llvm/test/CodeGen/RISCV/fp128.ll
+++ b/llvm/test/CodeGen/RISCV/fp128.ll
@@ -15,24 +15,24 @@ define i32 @test_load_and_cmp() nounwind {
 ; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a0, %hi(x)
 ; RV32I-NEXT:    lw a2, %lo(x)(a0)
-; RV32I-NEXT:    lw a1, %lo(x+4)(a0)
-; RV32I-NEXT:    lw a3, %lo(x+8)(a0)
-; RV32I-NEXT:    lw a0, %lo(x+12)(a0)
-; RV32I-NEXT:    lui a4, %hi(y)
-; RV32I-NEXT:    lw a5, %lo(y)(a4)
-; RV32I-NEXT:    lw a6, %lo(y+4)(a4)
-; RV32I-NEXT:    lw a7, %lo(y+8)(a4)
-; RV32I-NEXT:    lw a4, %lo(y+12)(a4)
-; RV32I-NEXT:    sw a4, 20(sp)
-; RV32I-NEXT:    sw a7, 16(sp)
+; RV32I-NEXT:    lw a3, %lo(x+4)(a0)
+; RV32I-NEXT:    lw a4, %lo(x+8)(a0)
+; RV32I-NEXT:    lw a5, %lo(x+12)(a0)
+; RV32I-NEXT:    lui a0, %hi(y)
+; RV32I-NEXT:    lw a1, %lo(y)(a0)
+; RV32I-NEXT:    lw a6, %lo(y+4)(a0)
+; RV32I-NEXT:    lw a7, %lo(y+8)(a0)
+; RV32I-NEXT:    lw a0, %lo(y+12)(a0)
+; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a0, 36(sp)
-; RV32I-NEXT:    sw a3, 32(sp)
-; RV32I-NEXT:    sw a1, 28(sp)
+; RV32I-NEXT:    sw a7, 16(sp)
+; RV32I-NEXT:    sw a0, 20(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    sw a2, 24(sp)
+; RV32I-NEXT:    sw a3, 28(sp)
+; RV32I-NEXT:    sw a4, 32(sp)
+; RV32I-NEXT:    sw a5, 36(sp)
 ; RV32I-NEXT:    call __netf2
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
@@ -52,35 +52,35 @@ define i32 @test_add_and_fptosi() nounwind {
 ; RV32I-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a0, %hi(x)
 ; RV32I-NEXT:    lw a3, %lo(x)(a0)
-; RV32I-NEXT:    lw a1, %lo(x+4)(a0)
-; RV32I-NEXT:    lw a2, %lo(x+8)(a0)
-; RV32I-NEXT:    lw a0, %lo(x+12)(a0)
-; RV32I-NEXT:    lui a4, %hi(y)
-; RV32I-NEXT:    lw a5, %lo(y)(a4)
-; RV32I-NEXT:    lw a6, %lo(y+4)(a4)
-; RV32I-NEXT:    lw a7, %lo(y+8)(a4)
-; RV32I-NEXT:    lw a4, %lo(y+12)(a4)
-; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    lw a4, %lo(x+4)(a0)
+; RV32I-NEXT:    lw a5, %lo(x+8)(a0)
+; RV32I-NEXT:    lw a6, %lo(x+12)(a0)
+; RV32I-NEXT:    lui a0, %hi(y)
+; RV32I-NEXT:    lw a1, %lo(y)(a0)
+; RV32I-NEXT:    lw a2, %lo(y+4)(a0)
+; RV32I-NEXT:    lw a7, %lo(y+8)(a0)
+; RV32I-NEXT:    lw a0, %lo(y+12)(a0)
+; RV32I-NEXT:    sw a1, 24(sp)
+; RV32I-NEXT:    sw a2, 28(sp)
 ; RV32I-NEXT:    sw a7, 32(sp)
-; RV32I-NEXT:    sw a6, 28(sp)
-; RV32I-NEXT:    sw a5, 24(sp)
-; RV32I-NEXT:    sw a0, 52(sp)
-; RV32I-NEXT:    sw a2, 48(sp)
-; RV32I-NEXT:    sw a1, 44(sp)
+; RV32I-NEXT:    sw a0, 36(sp)
 ; RV32I-NEXT:    addi a0, sp, 56
 ; RV32I-NEXT:    addi a1, sp, 40
 ; RV32I-NEXT:    addi a2, sp, 24
 ; RV32I-NEXT:    sw a3, 40(sp)
+; RV32I-NEXT:    sw a4, 44(sp)
+; RV32I-NEXT:    sw a5, 48(sp)
+; RV32I-NEXT:    sw a6, 52(sp)
 ; RV32I-NEXT:    call __addtf3
 ; RV32I-NEXT:    lw a1, 56(sp)
-; RV32I-NEXT:    lw a0, 60(sp)
-; RV32I-NEXT:    lw a2, 64(sp)
-; RV32I-NEXT:    lw a3, 68(sp)
-; RV32I-NEXT:    sw a3, 20(sp)
-; RV32I-NEXT:    sw a2, 16(sp)
-; RV32I-NEXT:    sw a0, 12(sp)
+; RV32I-NEXT:    lw a2, 60(sp)
+; RV32I-NEXT:    lw a3, 64(sp)
+; RV32I-NEXT:    lw a4, 68(sp)
 ; RV32I-NEXT:    addi a0, sp, 8
 ; RV32I-NEXT:    sw a1, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
+; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a4, 20(sp)
 ; RV32I-NEXT:    call __fixtfsi
 ; RV32I-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 80
diff --git a/llvm/test/CodeGen/RISCV/frame.ll b/llvm/test/CodeGen/RISCV/frame.ll
index d50f1e55417a7254a1c7134f82632e66b042be4f..10d542496e0f7f0142e62373319859645adbfe76 100644
--- a/llvm/test/CodeGen/RISCV/frame.ll
+++ b/llvm/test/CodeGen/RISCV/frame.ll
@@ -12,10 +12,10 @@ define i32 @test() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -32
 ; RV32I-FPELIM-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    sw zero, 24(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 20(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 16(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 12(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 8(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 12(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 16(sp)
+; RV32I-FPELIM-NEXT:    sw zero, 20(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 12
 ; RV32I-FPELIM-NEXT:    call test1
 ; RV32I-FPELIM-NEXT:    li a0, 0
@@ -30,10 +30,10 @@ define i32 @test() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -24(s0)
-; RV32I-WITHFP-NEXT:    sw zero, -28(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -32(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -28(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -24(s0)
+; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -28
 ; RV32I-WITHFP-NEXT:    call test1
 ; RV32I-WITHFP-NEXT:    li a0, 0
diff --git a/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll b/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
index f532f3c9f33df03c21faf0c1f1c07c1a0bc996b6..9322abcfbbdcefe64705897e5e5dc05c0fd97adb 100644
--- a/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
+++ b/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
@@ -17,10 +17,10 @@ define void @getSetCCResultType(ptr %p, ptr %q) nounwind {
 ; RV32I-NEXT:    addi a3, a3, -1
 ; RV32I-NEXT:    addi a2, a2, -1
 ; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    sw a2, 8(a0)
-; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a4, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
+; RV32I-NEXT:    sw a2, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
 entry:
   %0 = load <4 x i32>, ptr %p, align 16
diff --git a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
index 5f9866f08c821db478e0d856a22de74fa9038acc..77efffb6358af7ea73fd1873753aa068b58b327b 100644
--- a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
+++ b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
@@ -17,9 +17,9 @@ define dso_local void @multiple_stores() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    lui a0, %hi(s)
 ; CHECK-NEXT:    addi a0, a0, %lo(s)
 ; CHECK-NEXT:    li a1, 10
+; CHECK-NEXT:    li a2, 20
 ; CHECK-NEXT:    sw a1, 160(a0)
-; CHECK-NEXT:    li a1, 20
-; CHECK-NEXT:    sw a1, 164(a0)
+; CHECK-NEXT:    sw a2, 164(a0)
 ; CHECK-NEXT:    ret
 entry:
   store i32 10, ptr getelementptr inbounds (%struct.S, ptr @s, i32 0, i32 1), align 4
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-v-constraint.ll b/llvm/test/CodeGen/RISCV/inline-asm-v-constraint.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c04e4fea7b2c29fa45136852876acde582381adc
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/inline-asm-v-constraint.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+define <vscale x 1 x i8> @constraint_vr(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1) nounwind {
+; RV32I-LABEL: constraint_vr:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    vadd.vv v8, v8, v9
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: constraint_vr:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    vadd.vv v8, v8, v9
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %a = tail call <vscale x 1 x i8> asm "vadd.vv $0, $1, $2", "=^vr,^vr,^vr"(
+    <vscale x 1 x i8> %0, <vscale x 1 x i8> %1)
+  ret <vscale x 1 x i8> %a
+}
+
+define <vscale x 1 x i8> @constraint_vd(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1) nounwind {
+; RV32I-LABEL: constraint_vd:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    vadd.vv v8, v8, v9
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: constraint_vd:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    vadd.vv v8, v8, v9
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %a = tail call <vscale x 1 x i8> asm "vadd.vv $0, $1, $2", "=^vd,^vr,^vr"(
+    <vscale x 1 x i8> %0, <vscale x 1 x i8> %1)
+  ret <vscale x 1 x i8> %a
+}
+
+define <vscale x 1 x i1> @constraint_vm(<vscale x 1 x i1> %0, <vscale x 1 x i1> %1) nounwind {
+; RV32I-LABEL: constraint_vm:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    vmv1r.v v9, v0
+; RV32I-NEXT:    vmv1r.v v0, v8
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    vadd.vv v0, v9, v0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: constraint_vm:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    vmv1r.v v9, v0
+; RV64I-NEXT:    vmv1r.v v0, v8
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    vadd.vv v0, v9, v0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %a = tail call <vscale x 1 x i1> asm "vadd.vv $0, $1, $2", "=^vr,^vr,^vm"(
+    <vscale x 1 x i1> %0, <vscale x 1 x i1> %1)
+  ret <vscale x 1 x i1> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
index 6009a6c7e138aeee739102fecc4f767bcb50ac9b..8116d138d288e2f58472588aa06831c431207a1d 100644
--- a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
+++ b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
@@ -71,8 +71,8 @@ define i64 @ctz_nxv8i1_no_range(<vscale x 8 x i16> %a) {
 ; RV32-NEXT:    li a1, 0
 ; RV32-NEXT:    li a3, 0
 ; RV32-NEXT:    call __muldi3
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a0, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    addi a2, sp, 16
 ; RV32-NEXT:    vsetvli a3, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a2), zero
diff --git a/llvm/test/CodeGen/RISCV/legalize-fneg.ll b/llvm/test/CodeGen/RISCV/legalize-fneg.ll
index dfd62e8d5f9f562b6332d744477eac2a79f1b73a..f60b77b92c09ea53fe829d41a172edd23da4a40a 100644
--- a/llvm/test/CodeGen/RISCV/legalize-fneg.ll
+++ b/llvm/test/CodeGen/RISCV/legalize-fneg.ll
@@ -57,14 +57,14 @@ define void @test3(ptr %a, ptr %b) nounwind {
 ; RV32-LABEL: test3:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    lw a2, 12(a1)
-; RV32-NEXT:    lw a3, 4(a1)
-; RV32-NEXT:    lw a4, 8(a1)
-; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lw a3, 0(a1)
+; RV32-NEXT:    lw a4, 4(a1)
+; RV32-NEXT:    lw a1, 8(a1)
 ; RV32-NEXT:    lui a5, 524288
 ; RV32-NEXT:    xor a2, a2, a5
-; RV32-NEXT:    sw a4, 8(a0)
-; RV32-NEXT:    sw a1, 0(a0)
-; RV32-NEXT:    sw a3, 4(a0)
+; RV32-NEXT:    sw a3, 0(a0)
+; RV32-NEXT:    sw a4, 4(a0)
+; RV32-NEXT:    sw a1, 8(a0)
 ; RV32-NEXT:    sw a2, 12(a0)
 ; RV32-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/llvm.exp10.ll b/llvm/test/CodeGen/RISCV/llvm.exp10.ll
index 0941f6a73da2800c95f243a9d26ff15a33ec9690..cc07449c4e620075032ac71b810effbedb7c9ae4 100644
--- a/llvm/test/CodeGen/RISCV/llvm.exp10.ll
+++ b/llvm/test/CodeGen/RISCV/llvm.exp10.ll
@@ -197,8 +197,8 @@ define <3 x half> @exp10_v3f16(<3 x half> %x) {
 ; RV32IFD-NEXT:    call exp10f
 ; RV32IFD-NEXT:    call __truncsfhf2
 ; RV32IFD-NEXT:    fmv.x.w a0, fa0
-; RV32IFD-NEXT:    sh a0, 4(s0)
 ; RV32IFD-NEXT:    sw s1, 0(s0)
+; RV32IFD-NEXT:    sh a0, 4(s0)
 ; RV32IFD-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -246,8 +246,8 @@ define <3 x half> @exp10_v3f16(<3 x half> %x) {
 ; RV64IFD-NEXT:    call exp10f
 ; RV64IFD-NEXT:    call __truncsfhf2
 ; RV64IFD-NEXT:    fmv.x.w a0, fa0
-; RV64IFD-NEXT:    sh a0, 4(s0)
 ; RV64IFD-NEXT:    sw s1, 0(s0)
+; RV64IFD-NEXT:    sh a0, 4(s0)
 ; RV64IFD-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -313,10 +313,10 @@ define <4 x half> @exp10_v4f16(<4 x half> %x) {
 ; RV32IFD-NEXT:    fmv.x.w s3, fs3
 ; RV32IFD-NEXT:    call __truncsfhf2
 ; RV32IFD-NEXT:    fmv.x.w a0, fa0
-; RV32IFD-NEXT:    sh a0, 6(s0)
-; RV32IFD-NEXT:    sh s3, 4(s0)
-; RV32IFD-NEXT:    sh s2, 2(s0)
 ; RV32IFD-NEXT:    sh s1, 0(s0)
+; RV32IFD-NEXT:    sh s2, 2(s0)
+; RV32IFD-NEXT:    sh s3, 4(s0)
+; RV32IFD-NEXT:    sh a0, 6(s0)
 ; RV32IFD-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s1, 52(sp) # 4-byte Folded Reload
@@ -377,10 +377,10 @@ define <4 x half> @exp10_v4f16(<4 x half> %x) {
 ; RV64IFD-NEXT:    fmv.x.w s3, fs0
 ; RV64IFD-NEXT:    call __truncsfhf2
 ; RV64IFD-NEXT:    fmv.x.w a0, fa0
-; RV64IFD-NEXT:    sh a0, 6(s0)
-; RV64IFD-NEXT:    sh s3, 4(s0)
-; RV64IFD-NEXT:    sh s2, 2(s0)
 ; RV64IFD-NEXT:    sh s1, 0(s0)
+; RV64IFD-NEXT:    sh s2, 2(s0)
+; RV64IFD-NEXT:    sh s3, 4(s0)
+; RV64IFD-NEXT:    sh a0, 6(s0)
 ; RV64IFD-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -504,9 +504,9 @@ define <3 x float> @exp10_v3f32(<3 x float> %x) {
 ; RV32IFD-NEXT:    fmv.s fs1, fa0
 ; RV32IFD-NEXT:    fmv.s fa0, fs0
 ; RV32IFD-NEXT:    call exp10f
-; RV32IFD-NEXT:    fsw fa0, 8(s0)
-; RV32IFD-NEXT:    fsw fs1, 4(s0)
 ; RV32IFD-NEXT:    fsw fs2, 0(s0)
+; RV32IFD-NEXT:    fsw fs1, 4(s0)
+; RV32IFD-NEXT:    fsw fa0, 8(s0)
 ; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 16(sp) # 8-byte Folded Reload
@@ -544,8 +544,8 @@ define <3 x float> @exp10_v3f32(<3 x float> %x) {
 ; RV64IFD-NEXT:    or s1, a0, s1
 ; RV64IFD-NEXT:    fmv.s fa0, fs0
 ; RV64IFD-NEXT:    call exp10f
-; RV64IFD-NEXT:    fsw fa0, 8(s0)
 ; RV64IFD-NEXT:    sd s1, 0(s0)
+; RV64IFD-NEXT:    fsw fa0, 8(s0)
 ; RV64IFD-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -588,10 +588,10 @@ define <4 x float> @exp10_v4f32(<4 x float> %x) {
 ; RV32IFD-NEXT:    fmv.s fs1, fa0
 ; RV32IFD-NEXT:    fmv.s fa0, fs0
 ; RV32IFD-NEXT:    call exp10f
-; RV32IFD-NEXT:    fsw fa0, 12(s0)
-; RV32IFD-NEXT:    fsw fs1, 8(s0)
-; RV32IFD-NEXT:    fsw fs2, 4(s0)
 ; RV32IFD-NEXT:    fsw fs3, 0(s0)
+; RV32IFD-NEXT:    fsw fs2, 4(s0)
+; RV32IFD-NEXT:    fsw fs1, 8(s0)
+; RV32IFD-NEXT:    fsw fa0, 12(s0)
 ; RV32IFD-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 32(sp) # 8-byte Folded Reload
@@ -631,10 +631,10 @@ define <4 x float> @exp10_v4f32(<4 x float> %x) {
 ; RV64IFD-NEXT:    fmv.s fs1, fa0
 ; RV64IFD-NEXT:    fmv.s fa0, fs0
 ; RV64IFD-NEXT:    call exp10f
-; RV64IFD-NEXT:    fsw fa0, 12(s0)
-; RV64IFD-NEXT:    fsw fs1, 8(s0)
-; RV64IFD-NEXT:    fsw fs2, 4(s0)
 ; RV64IFD-NEXT:    fsw fs3, 0(s0)
+; RV64IFD-NEXT:    fsw fs2, 4(s0)
+; RV64IFD-NEXT:    fsw fs1, 8(s0)
+; RV64IFD-NEXT:    fsw fa0, 12(s0)
 ; RV64IFD-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fld fs0, 24(sp) # 8-byte Folded Reload
@@ -736,9 +736,9 @@ define <3 x double> @exp10_v3f64(<3 x double> %x) {
 ; RV32IFD-NEXT:    fmv.d fs1, fa0
 ; RV32IFD-NEXT:    fmv.d fa0, fs0
 ; RV32IFD-NEXT:    call exp10
-; RV32IFD-NEXT:    fsd fa0, 16(s0)
-; RV32IFD-NEXT:    fsd fs1, 8(s0)
 ; RV32IFD-NEXT:    fsd fs2, 0(s0)
+; RV32IFD-NEXT:    fsd fs1, 8(s0)
+; RV32IFD-NEXT:    fsd fa0, 16(s0)
 ; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 16(sp) # 8-byte Folded Reload
@@ -771,9 +771,9 @@ define <3 x double> @exp10_v3f64(<3 x double> %x) {
 ; RV64IFD-NEXT:    fmv.d fs1, fa0
 ; RV64IFD-NEXT:    fmv.d fa0, fs0
 ; RV64IFD-NEXT:    call exp10
-; RV64IFD-NEXT:    fsd fa0, 16(s0)
-; RV64IFD-NEXT:    fsd fs1, 8(s0)
 ; RV64IFD-NEXT:    fsd fs2, 0(s0)
+; RV64IFD-NEXT:    fsd fs1, 8(s0)
+; RV64IFD-NEXT:    fsd fa0, 16(s0)
 ; RV64IFD-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fld fs0, 24(sp) # 8-byte Folded Reload
@@ -816,10 +816,10 @@ define <4 x double> @exp10_v4f64(<4 x double> %x) {
 ; RV32IFD-NEXT:    fmv.d fs1, fa0
 ; RV32IFD-NEXT:    fmv.d fa0, fs0
 ; RV32IFD-NEXT:    call exp10
-; RV32IFD-NEXT:    fsd fa0, 24(s0)
-; RV32IFD-NEXT:    fsd fs1, 16(s0)
-; RV32IFD-NEXT:    fsd fs2, 8(s0)
 ; RV32IFD-NEXT:    fsd fs3, 0(s0)
+; RV32IFD-NEXT:    fsd fs2, 8(s0)
+; RV32IFD-NEXT:    fsd fs1, 16(s0)
+; RV32IFD-NEXT:    fsd fa0, 24(s0)
 ; RV32IFD-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 32(sp) # 8-byte Folded Reload
@@ -859,10 +859,10 @@ define <4 x double> @exp10_v4f64(<4 x double> %x) {
 ; RV64IFD-NEXT:    fmv.d fs1, fa0
 ; RV64IFD-NEXT:    fmv.d fa0, fs0
 ; RV64IFD-NEXT:    call exp10
-; RV64IFD-NEXT:    fsd fa0, 24(s0)
-; RV64IFD-NEXT:    fsd fs1, 16(s0)
-; RV64IFD-NEXT:    fsd fs2, 8(s0)
 ; RV64IFD-NEXT:    fsd fs3, 0(s0)
+; RV64IFD-NEXT:    fsd fs2, 8(s0)
+; RV64IFD-NEXT:    fsd fs1, 16(s0)
+; RV64IFD-NEXT:    fsd fa0, 24(s0)
 ; RV64IFD-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fld fs0, 24(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/llvm.frexp.ll b/llvm/test/CodeGen/RISCV/llvm.frexp.ll
index 2c9d640e03a634a4536f530e9019dfb8f6ba0f25..e85a7118f5ff830f20768d511c68824fbf50fa30 100644
--- a/llvm/test/CodeGen/RISCV/llvm.frexp.ll
+++ b/llvm/test/CodeGen/RISCV/llvm.frexp.ll
@@ -568,18 +568,18 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi
 ; RV32IFD-NEXT:    addi a0, sp, 20
 ; RV32IFD-NEXT:    fmv.s fa0, fs0
 ; RV32IFD-NEXT:    call frexpf
-; RV32IFD-NEXT:    lw a0, 20(sp)
-; RV32IFD-NEXT:    lw a1, 16(sp)
-; RV32IFD-NEXT:    lw a2, 12(sp)
-; RV32IFD-NEXT:    lw a3, 8(sp)
-; RV32IFD-NEXT:    sw a0, 28(s0)
-; RV32IFD-NEXT:    sw a1, 24(s0)
-; RV32IFD-NEXT:    sw a2, 20(s0)
-; RV32IFD-NEXT:    sw a3, 16(s0)
-; RV32IFD-NEXT:    fsw fa0, 12(s0)
-; RV32IFD-NEXT:    fsw fs1, 8(s0)
-; RV32IFD-NEXT:    fsw fs2, 4(s0)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
+; RV32IFD-NEXT:    lw a2, 16(sp)
+; RV32IFD-NEXT:    lw a3, 20(sp)
+; RV32IFD-NEXT:    sw a0, 16(s0)
+; RV32IFD-NEXT:    sw a1, 20(s0)
+; RV32IFD-NEXT:    sw a2, 24(s0)
+; RV32IFD-NEXT:    sw a3, 28(s0)
 ; RV32IFD-NEXT:    fsw fs3, 0(s0)
+; RV32IFD-NEXT:    fsw fs2, 4(s0)
+; RV32IFD-NEXT:    fsw fs1, 8(s0)
+; RV32IFD-NEXT:    fsw fa0, 12(s0)
 ; RV32IFD-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 48(sp) # 8-byte Folded Reload
@@ -616,18 +616,18 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi
 ; RV64IFD-NEXT:    addi a0, sp, 24
 ; RV64IFD-NEXT:    fmv.s fa0, fs0
 ; RV64IFD-NEXT:    call frexpf
-; RV64IFD-NEXT:    ld a0, 24(sp)
-; RV64IFD-NEXT:    ld a1, 16(sp)
-; RV64IFD-NEXT:    ld a2, 8(sp)
-; RV64IFD-NEXT:    ld a3, 0(sp)
-; RV64IFD-NEXT:    sw a0, 28(s0)
-; RV64IFD-NEXT:    sw a1, 24(s0)
-; RV64IFD-NEXT:    sw a2, 20(s0)
-; RV64IFD-NEXT:    sw a3, 16(s0)
-; RV64IFD-NEXT:    fsw fa0, 12(s0)
-; RV64IFD-NEXT:    fsw fs1, 8(s0)
-; RV64IFD-NEXT:    fsw fs2, 4(s0)
+; RV64IFD-NEXT:    ld a0, 0(sp)
+; RV64IFD-NEXT:    ld a1, 8(sp)
+; RV64IFD-NEXT:    ld a2, 16(sp)
+; RV64IFD-NEXT:    ld a3, 24(sp)
+; RV64IFD-NEXT:    sw a0, 16(s0)
+; RV64IFD-NEXT:    sw a1, 20(s0)
+; RV64IFD-NEXT:    sw a2, 24(s0)
+; RV64IFD-NEXT:    sw a3, 28(s0)
 ; RV64IFD-NEXT:    fsw fs3, 0(s0)
+; RV64IFD-NEXT:    fsw fs2, 4(s0)
+; RV64IFD-NEXT:    fsw fs1, 8(s0)
+; RV64IFD-NEXT:    fsw fa0, 12(s0)
 ; RV64IFD-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fld fs0, 56(sp) # 8-byte Folded Reload
@@ -666,18 +666,18 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi
 ; RV32IZFINXZDINX-NEXT:    addi a1, sp, 20
 ; RV32IZFINXZDINX-NEXT:    mv a0, s0
 ; RV32IZFINXZDINX-NEXT:    call frexpf
-; RV32IZFINXZDINX-NEXT:    lw a1, 20(sp)
-; RV32IZFINXZDINX-NEXT:    lw a2, 16(sp)
-; RV32IZFINXZDINX-NEXT:    lw a3, 12(sp)
-; RV32IZFINXZDINX-NEXT:    lw a4, 8(sp)
-; RV32IZFINXZDINX-NEXT:    sw a1, 28(s3)
-; RV32IZFINXZDINX-NEXT:    sw a2, 24(s3)
-; RV32IZFINXZDINX-NEXT:    sw a3, 20(s3)
-; RV32IZFINXZDINX-NEXT:    sw a4, 16(s3)
-; RV32IZFINXZDINX-NEXT:    sw a0, 12(s3)
-; RV32IZFINXZDINX-NEXT:    sw s1, 8(s3)
-; RV32IZFINXZDINX-NEXT:    sw s2, 4(s3)
+; RV32IZFINXZDINX-NEXT:    lw a1, 8(sp)
+; RV32IZFINXZDINX-NEXT:    lw a2, 12(sp)
+; RV32IZFINXZDINX-NEXT:    lw a3, 16(sp)
+; RV32IZFINXZDINX-NEXT:    lw a4, 20(sp)
+; RV32IZFINXZDINX-NEXT:    sw a1, 16(s3)
+; RV32IZFINXZDINX-NEXT:    sw a2, 20(s3)
+; RV32IZFINXZDINX-NEXT:    sw a3, 24(s3)
+; RV32IZFINXZDINX-NEXT:    sw a4, 28(s3)
 ; RV32IZFINXZDINX-NEXT:    sw s4, 0(s3)
+; RV32IZFINXZDINX-NEXT:    sw s2, 4(s3)
+; RV32IZFINXZDINX-NEXT:    sw s1, 8(s3)
+; RV32IZFINXZDINX-NEXT:    sw a0, 12(s3)
 ; RV32IZFINXZDINX-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -716,18 +716,18 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi
 ; RV64IZFINXZDINX-NEXT:    addi a1, sp, 24
 ; RV64IZFINXZDINX-NEXT:    mv a0, s0
 ; RV64IZFINXZDINX-NEXT:    call frexpf
-; RV64IZFINXZDINX-NEXT:    ld a1, 24(sp)
-; RV64IZFINXZDINX-NEXT:    ld a2, 16(sp)
-; RV64IZFINXZDINX-NEXT:    ld a3, 8(sp)
-; RV64IZFINXZDINX-NEXT:    ld a4, 0(sp)
-; RV64IZFINXZDINX-NEXT:    sw a1, 28(s3)
-; RV64IZFINXZDINX-NEXT:    sw a2, 24(s3)
-; RV64IZFINXZDINX-NEXT:    sw a3, 20(s3)
-; RV64IZFINXZDINX-NEXT:    sw a4, 16(s3)
-; RV64IZFINXZDINX-NEXT:    sw a0, 12(s3)
-; RV64IZFINXZDINX-NEXT:    sw s1, 8(s3)
-; RV64IZFINXZDINX-NEXT:    sw s2, 4(s3)
+; RV64IZFINXZDINX-NEXT:    ld a1, 0(sp)
+; RV64IZFINXZDINX-NEXT:    ld a2, 8(sp)
+; RV64IZFINXZDINX-NEXT:    ld a3, 16(sp)
+; RV64IZFINXZDINX-NEXT:    ld a4, 24(sp)
+; RV64IZFINXZDINX-NEXT:    sw a1, 16(s3)
+; RV64IZFINXZDINX-NEXT:    sw a2, 20(s3)
+; RV64IZFINXZDINX-NEXT:    sw a3, 24(s3)
+; RV64IZFINXZDINX-NEXT:    sw a4, 28(s3)
 ; RV64IZFINXZDINX-NEXT:    sw s4, 0(s3)
+; RV64IZFINXZDINX-NEXT:    sw s2, 4(s3)
+; RV64IZFINXZDINX-NEXT:    sw s1, 8(s3)
+; RV64IZFINXZDINX-NEXT:    sw a0, 12(s3)
 ; RV64IZFINXZDINX-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    ld s1, 56(sp) # 8-byte Folded Reload
@@ -770,14 +770,14 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi
 ; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    lw a3, 16(sp)
 ; RV32I-NEXT:    lw a4, 20(sp)
-; RV32I-NEXT:    sw a0, 12(s3)
-; RV32I-NEXT:    sw s1, 8(s3)
-; RV32I-NEXT:    sw s0, 4(s3)
 ; RV32I-NEXT:    sw s4, 0(s3)
-; RV32I-NEXT:    sw a4, 28(s3)
-; RV32I-NEXT:    sw a3, 24(s3)
-; RV32I-NEXT:    sw a2, 20(s3)
+; RV32I-NEXT:    sw s0, 4(s3)
+; RV32I-NEXT:    sw s1, 8(s3)
+; RV32I-NEXT:    sw a0, 12(s3)
 ; RV32I-NEXT:    sw a1, 16(s3)
+; RV32I-NEXT:    sw a2, 20(s3)
+; RV32I-NEXT:    sw a3, 24(s3)
+; RV32I-NEXT:    sw a4, 28(s3)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -820,14 +820,14 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi
 ; RV64I-NEXT:    lw a2, 4(sp)
 ; RV64I-NEXT:    lw a3, 8(sp)
 ; RV64I-NEXT:    lw a4, 12(sp)
-; RV64I-NEXT:    sw a0, 12(s3)
-; RV64I-NEXT:    sw s1, 8(s3)
-; RV64I-NEXT:    sw s0, 4(s3)
 ; RV64I-NEXT:    sw s4, 0(s3)
-; RV64I-NEXT:    sw a4, 28(s3)
-; RV64I-NEXT:    sw a3, 24(s3)
-; RV64I-NEXT:    sw a2, 20(s3)
+; RV64I-NEXT:    sw s0, 4(s3)
+; RV64I-NEXT:    sw s1, 8(s3)
+; RV64I-NEXT:    sw a0, 12(s3)
 ; RV64I-NEXT:    sw a1, 16(s3)
+; RV64I-NEXT:    sw a2, 20(s3)
+; RV64I-NEXT:    sw a3, 24(s3)
+; RV64I-NEXT:    sw a4, 28(s3)
 ; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -868,10 +868,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) nounwi
 ; RV32IFD-NEXT:    addi a0, sp, 20
 ; RV32IFD-NEXT:    fmv.s fa0, fs0
 ; RV32IFD-NEXT:    call frexpf
-; RV32IFD-NEXT:    fsw fa0, 12(s0)
-; RV32IFD-NEXT:    fsw fs1, 8(s0)
-; RV32IFD-NEXT:    fsw fs2, 4(s0)
 ; RV32IFD-NEXT:    fsw fs3, 0(s0)
+; RV32IFD-NEXT:    fsw fs2, 4(s0)
+; RV32IFD-NEXT:    fsw fs1, 8(s0)
+; RV32IFD-NEXT:    fsw fa0, 12(s0)
 ; RV32IFD-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 48(sp) # 8-byte Folded Reload
@@ -908,10 +908,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) nounwi
 ; RV64IFD-NEXT:    addi a0, sp, 24
 ; RV64IFD-NEXT:    fmv.s fa0, fs0
 ; RV64IFD-NEXT:    call frexpf
-; RV64IFD-NEXT:    fsw fa0, 12(s0)
-; RV64IFD-NEXT:    fsw fs1, 8(s0)
-; RV64IFD-NEXT:    fsw fs2, 4(s0)
 ; RV64IFD-NEXT:    fsw fs3, 0(s0)
+; RV64IFD-NEXT:    fsw fs2, 4(s0)
+; RV64IFD-NEXT:    fsw fs1, 8(s0)
+; RV64IFD-NEXT:    fsw fa0, 12(s0)
 ; RV64IFD-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fld fs0, 56(sp) # 8-byte Folded Reload
@@ -950,10 +950,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) nounwi
 ; RV32IZFINXZDINX-NEXT:    addi a1, sp, 20
 ; RV32IZFINXZDINX-NEXT:    mv a0, s0
 ; RV32IZFINXZDINX-NEXT:    call frexpf
-; RV32IZFINXZDINX-NEXT:    sw a0, 12(s3)
-; RV32IZFINXZDINX-NEXT:    sw s1, 8(s3)
-; RV32IZFINXZDINX-NEXT:    sw s2, 4(s3)
 ; RV32IZFINXZDINX-NEXT:    sw s4, 0(s3)
+; RV32IZFINXZDINX-NEXT:    sw s2, 4(s3)
+; RV32IZFINXZDINX-NEXT:    sw s1, 8(s3)
+; RV32IZFINXZDINX-NEXT:    sw a0, 12(s3)
 ; RV32IZFINXZDINX-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -992,10 +992,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) nounwi
 ; RV64IZFINXZDINX-NEXT:    addi a1, sp, 24
 ; RV64IZFINXZDINX-NEXT:    mv a0, s0
 ; RV64IZFINXZDINX-NEXT:    call frexpf
-; RV64IZFINXZDINX-NEXT:    sw a0, 12(s3)
-; RV64IZFINXZDINX-NEXT:    sw s1, 8(s3)
-; RV64IZFINXZDINX-NEXT:    sw s2, 4(s3)
 ; RV64IZFINXZDINX-NEXT:    sw s4, 0(s3)
+; RV64IZFINXZDINX-NEXT:    sw s2, 4(s3)
+; RV64IZFINXZDINX-NEXT:    sw s1, 8(s3)
+; RV64IZFINXZDINX-NEXT:    sw a0, 12(s3)
 ; RV64IZFINXZDINX-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    ld s1, 56(sp) # 8-byte Folded Reload
@@ -1034,10 +1034,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) nounwi
 ; RV32I-NEXT:    addi a1, sp, 20
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    call frexpf
-; RV32I-NEXT:    sw a0, 12(s3)
-; RV32I-NEXT:    sw s1, 8(s3)
-; RV32I-NEXT:    sw s0, 4(s3)
 ; RV32I-NEXT:    sw s4, 0(s3)
+; RV32I-NEXT:    sw s0, 4(s3)
+; RV32I-NEXT:    sw s1, 8(s3)
+; RV32I-NEXT:    sw a0, 12(s3)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -1076,10 +1076,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) nounwi
 ; RV64I-NEXT:    addi a1, sp, 12
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call frexpf
-; RV64I-NEXT:    sw a0, 12(s3)
-; RV64I-NEXT:    sw s1, 8(s3)
-; RV64I-NEXT:    sw s0, 4(s3)
 ; RV64I-NEXT:    sw s4, 0(s3)
+; RV64I-NEXT:    sw s0, 4(s3)
+; RV64I-NEXT:    sw s1, 8(s3)
+; RV64I-NEXT:    sw a0, 12(s3)
 ; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -1117,14 +1117,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind {
 ; RV32IFD-NEXT:    addi a0, sp, 12
 ; RV32IFD-NEXT:    fmv.s fa0, fs0
 ; RV32IFD-NEXT:    call frexpf
-; RV32IFD-NEXT:    lw a0, 12(sp)
-; RV32IFD-NEXT:    lw a1, 8(sp)
-; RV32IFD-NEXT:    lw a2, 4(sp)
-; RV32IFD-NEXT:    lw a3, 0(sp)
-; RV32IFD-NEXT:    sw a0, 12(s0)
-; RV32IFD-NEXT:    sw a1, 8(s0)
-; RV32IFD-NEXT:    sw a2, 4(s0)
-; RV32IFD-NEXT:    sw a3, 0(s0)
+; RV32IFD-NEXT:    lw a0, 0(sp)
+; RV32IFD-NEXT:    lw a1, 4(sp)
+; RV32IFD-NEXT:    lw a2, 8(sp)
+; RV32IFD-NEXT:    lw a3, 12(sp)
+; RV32IFD-NEXT:    sw a0, 0(s0)
+; RV32IFD-NEXT:    sw a1, 4(s0)
+; RV32IFD-NEXT:    sw a2, 8(s0)
+; RV32IFD-NEXT:    sw a3, 12(s0)
 ; RV32IFD-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 32(sp) # 8-byte Folded Reload
@@ -1156,14 +1156,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind {
 ; RV64IFD-NEXT:    addi a0, sp, 32
 ; RV64IFD-NEXT:    fmv.s fa0, fs0
 ; RV64IFD-NEXT:    call frexpf
-; RV64IFD-NEXT:    ld a0, 32(sp)
-; RV64IFD-NEXT:    ld a1, 24(sp)
-; RV64IFD-NEXT:    ld a2, 16(sp)
-; RV64IFD-NEXT:    ld a3, 8(sp)
-; RV64IFD-NEXT:    sw a0, 12(s0)
-; RV64IFD-NEXT:    sw a1, 8(s0)
-; RV64IFD-NEXT:    sw a2, 4(s0)
-; RV64IFD-NEXT:    sw a3, 0(s0)
+; RV64IFD-NEXT:    ld a0, 8(sp)
+; RV64IFD-NEXT:    ld a1, 16(sp)
+; RV64IFD-NEXT:    ld a2, 24(sp)
+; RV64IFD-NEXT:    ld a3, 32(sp)
+; RV64IFD-NEXT:    sw a0, 0(s0)
+; RV64IFD-NEXT:    sw a1, 4(s0)
+; RV64IFD-NEXT:    sw a2, 8(s0)
+; RV64IFD-NEXT:    sw a3, 12(s0)
 ; RV64IFD-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fld fs0, 56(sp) # 8-byte Folded Reload
@@ -1197,14 +1197,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind {
 ; RV32IZFINXZDINX-NEXT:    addi a1, sp, 24
 ; RV32IZFINXZDINX-NEXT:    mv a0, s0
 ; RV32IZFINXZDINX-NEXT:    call frexpf
-; RV32IZFINXZDINX-NEXT:    lw a0, 24(sp)
-; RV32IZFINXZDINX-NEXT:    lw a1, 20(sp)
-; RV32IZFINXZDINX-NEXT:    lw a2, 16(sp)
-; RV32IZFINXZDINX-NEXT:    lw a3, 12(sp)
-; RV32IZFINXZDINX-NEXT:    sw a0, 12(s3)
-; RV32IZFINXZDINX-NEXT:    sw a1, 8(s3)
-; RV32IZFINXZDINX-NEXT:    sw a2, 4(s3)
-; RV32IZFINXZDINX-NEXT:    sw a3, 0(s3)
+; RV32IZFINXZDINX-NEXT:    lw a0, 12(sp)
+; RV32IZFINXZDINX-NEXT:    lw a1, 16(sp)
+; RV32IZFINXZDINX-NEXT:    lw a2, 20(sp)
+; RV32IZFINXZDINX-NEXT:    lw a3, 24(sp)
+; RV32IZFINXZDINX-NEXT:    sw a0, 0(s3)
+; RV32IZFINXZDINX-NEXT:    sw a1, 4(s3)
+; RV32IZFINXZDINX-NEXT:    sw a2, 8(s3)
+; RV32IZFINXZDINX-NEXT:    sw a3, 12(s3)
 ; RV32IZFINXZDINX-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -1238,14 +1238,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    addi a1, sp, 32
 ; RV64IZFINXZDINX-NEXT:    mv a0, s0
 ; RV64IZFINXZDINX-NEXT:    call frexpf
-; RV64IZFINXZDINX-NEXT:    ld a0, 32(sp)
-; RV64IZFINXZDINX-NEXT:    ld a1, 24(sp)
-; RV64IZFINXZDINX-NEXT:    ld a2, 16(sp)
-; RV64IZFINXZDINX-NEXT:    ld a3, 8(sp)
-; RV64IZFINXZDINX-NEXT:    sw a0, 12(s3)
-; RV64IZFINXZDINX-NEXT:    sw a1, 8(s3)
-; RV64IZFINXZDINX-NEXT:    sw a2, 4(s3)
-; RV64IZFINXZDINX-NEXT:    sw a3, 0(s3)
+; RV64IZFINXZDINX-NEXT:    ld a0, 8(sp)
+; RV64IZFINXZDINX-NEXT:    ld a1, 16(sp)
+; RV64IZFINXZDINX-NEXT:    ld a2, 24(sp)
+; RV64IZFINXZDINX-NEXT:    ld a3, 32(sp)
+; RV64IZFINXZDINX-NEXT:    sw a0, 0(s3)
+; RV64IZFINXZDINX-NEXT:    sw a1, 4(s3)
+; RV64IZFINXZDINX-NEXT:    sw a2, 8(s3)
+; RV64IZFINXZDINX-NEXT:    sw a3, 12(s3)
 ; RV64IZFINXZDINX-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    ld s1, 56(sp) # 8-byte Folded Reload
@@ -1279,14 +1279,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind {
 ; RV32I-NEXT:    addi a1, sp, 24
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    call frexpf
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    lw a1, 20(sp)
-; RV32I-NEXT:    lw a2, 16(sp)
-; RV32I-NEXT:    lw a3, 12(sp)
-; RV32I-NEXT:    sw a0, 12(s3)
-; RV32I-NEXT:    sw a1, 8(s3)
-; RV32I-NEXT:    sw a2, 4(s3)
-; RV32I-NEXT:    sw a3, 0(s3)
+; RV32I-NEXT:    lw a0, 12(sp)
+; RV32I-NEXT:    lw a1, 16(sp)
+; RV32I-NEXT:    lw a2, 20(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
+; RV32I-NEXT:    sw a0, 0(s3)
+; RV32I-NEXT:    sw a1, 4(s3)
+; RV32I-NEXT:    sw a2, 8(s3)
+; RV32I-NEXT:    sw a3, 12(s3)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -1320,14 +1320,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind {
 ; RV64I-NEXT:    addi a1, sp, 20
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call frexpf
-; RV64I-NEXT:    lw a0, 20(sp)
-; RV64I-NEXT:    lw a1, 16(sp)
-; RV64I-NEXT:    lw a2, 12(sp)
-; RV64I-NEXT:    lw a3, 8(sp)
-; RV64I-NEXT:    sw a0, 12(s3)
-; RV64I-NEXT:    sw a1, 8(s3)
-; RV64I-NEXT:    sw a2, 4(s3)
-; RV64I-NEXT:    sw a3, 0(s3)
+; RV64I-NEXT:    lw a0, 8(sp)
+; RV64I-NEXT:    lw a1, 12(sp)
+; RV64I-NEXT:    lw a2, 16(sp)
+; RV64I-NEXT:    lw a3, 20(sp)
+; RV64I-NEXT:    sw a0, 0(s3)
+; RV64I-NEXT:    sw a1, 4(s3)
+; RV64I-NEXT:    sw a2, 8(s3)
+; RV64I-NEXT:    sw a3, 12(s3)
 ; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -1397,8 +1397,8 @@ define { double, i32 } @test_frexp_f64_i32(double %a) nounwind {
 ; RV32I-NEXT:    mv a1, a3
 ; RV32I-NEXT:    call frexp
 ; RV32I-NEXT:    lw a2, 4(sp)
-; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
 ; RV32I-NEXT:    sw a2, 8(s0)
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -1580,28 +1580,28 @@ define { fp128, i32 } @test_frexp_f128_i32(fp128 %a) nounwind {
 ; RV32IFD-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lw a3, 0(a1)
-; RV32IFD-NEXT:    lw a2, 4(a1)
-; RV32IFD-NEXT:    lw a4, 8(a1)
-; RV32IFD-NEXT:    lw a1, 12(a1)
+; RV32IFD-NEXT:    lw a4, 4(a1)
+; RV32IFD-NEXT:    lw a5, 8(a1)
+; RV32IFD-NEXT:    lw a6, 12(a1)
 ; RV32IFD-NEXT:    mv s0, a0
-; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    sw a4, 8(sp)
-; RV32IFD-NEXT:    sw a2, 4(sp)
 ; RV32IFD-NEXT:    addi a0, sp, 16
 ; RV32IFD-NEXT:    mv a1, sp
 ; RV32IFD-NEXT:    addi a2, sp, 36
 ; RV32IFD-NEXT:    sw a3, 0(sp)
+; RV32IFD-NEXT:    sw a4, 4(sp)
+; RV32IFD-NEXT:    sw a5, 8(sp)
+; RV32IFD-NEXT:    sw a6, 12(sp)
 ; RV32IFD-NEXT:    call frexpl
-; RV32IFD-NEXT:    lw a0, 24(sp)
-; RV32IFD-NEXT:    lw a1, 28(sp)
-; RV32IFD-NEXT:    lw a2, 16(sp)
-; RV32IFD-NEXT:    lw a3, 20(sp)
-; RV32IFD-NEXT:    lw a4, 36(sp)
-; RV32IFD-NEXT:    sw a1, 12(s0)
-; RV32IFD-NEXT:    sw a0, 8(s0)
-; RV32IFD-NEXT:    sw a3, 4(s0)
-; RV32IFD-NEXT:    sw a2, 0(s0)
-; RV32IFD-NEXT:    sw a4, 16(s0)
+; RV32IFD-NEXT:    lw a0, 36(sp)
+; RV32IFD-NEXT:    lw a1, 16(sp)
+; RV32IFD-NEXT:    lw a2, 20(sp)
+; RV32IFD-NEXT:    lw a3, 24(sp)
+; RV32IFD-NEXT:    lw a4, 28(sp)
+; RV32IFD-NEXT:    sw a1, 0(s0)
+; RV32IFD-NEXT:    sw a2, 4(s0)
+; RV32IFD-NEXT:    sw a3, 8(s0)
+; RV32IFD-NEXT:    sw a4, 12(s0)
+; RV32IFD-NEXT:    sw a0, 16(s0)
 ; RV32IFD-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 48
@@ -1619,8 +1619,8 @@ define { fp128, i32 } @test_frexp_f128_i32(fp128 %a) nounwind {
 ; RV64IFD-NEXT:    mv a1, a3
 ; RV64IFD-NEXT:    call frexpl
 ; RV64IFD-NEXT:    lw a2, 12(sp)
-; RV64IFD-NEXT:    sd a1, 8(s0)
 ; RV64IFD-NEXT:    sd a0, 0(s0)
+; RV64IFD-NEXT:    sd a1, 8(s0)
 ; RV64IFD-NEXT:    sw a2, 16(s0)
 ; RV64IFD-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -1633,28 +1633,28 @@ define { fp128, i32 } @test_frexp_f128_i32(fp128 %a) nounwind {
 ; RV32IZFINXZDINX-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    lw a3, 0(a1)
-; RV32IZFINXZDINX-NEXT:    lw a2, 4(a1)
-; RV32IZFINXZDINX-NEXT:    lw a4, 8(a1)
-; RV32IZFINXZDINX-NEXT:    lw a1, 12(a1)
+; RV32IZFINXZDINX-NEXT:    lw a4, 4(a1)
+; RV32IZFINXZDINX-NEXT:    lw a5, 8(a1)
+; RV32IZFINXZDINX-NEXT:    lw a6, 12(a1)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
-; RV32IZFINXZDINX-NEXT:    sw a1, 12(sp)
-; RV32IZFINXZDINX-NEXT:    sw a4, 8(sp)
-; RV32IZFINXZDINX-NEXT:    sw a2, 4(sp)
 ; RV32IZFINXZDINX-NEXT:    addi a0, sp, 16
 ; RV32IZFINXZDINX-NEXT:    mv a1, sp
 ; RV32IZFINXZDINX-NEXT:    addi a2, sp, 36
 ; RV32IZFINXZDINX-NEXT:    sw a3, 0(sp)
+; RV32IZFINXZDINX-NEXT:    sw a4, 4(sp)
+; RV32IZFINXZDINX-NEXT:    sw a5, 8(sp)
+; RV32IZFINXZDINX-NEXT:    sw a6, 12(sp)
 ; RV32IZFINXZDINX-NEXT:    call frexpl
-; RV32IZFINXZDINX-NEXT:    lw a0, 24(sp)
-; RV32IZFINXZDINX-NEXT:    lw a1, 28(sp)
-; RV32IZFINXZDINX-NEXT:    lw a2, 16(sp)
-; RV32IZFINXZDINX-NEXT:    lw a3, 20(sp)
-; RV32IZFINXZDINX-NEXT:    lw a4, 36(sp)
-; RV32IZFINXZDINX-NEXT:    sw a1, 12(s0)
-; RV32IZFINXZDINX-NEXT:    sw a0, 8(s0)
-; RV32IZFINXZDINX-NEXT:    sw a3, 4(s0)
-; RV32IZFINXZDINX-NEXT:    sw a2, 0(s0)
-; RV32IZFINXZDINX-NEXT:    sw a4, 16(s0)
+; RV32IZFINXZDINX-NEXT:    lw a0, 36(sp)
+; RV32IZFINXZDINX-NEXT:    lw a1, 16(sp)
+; RV32IZFINXZDINX-NEXT:    lw a2, 20(sp)
+; RV32IZFINXZDINX-NEXT:    lw a3, 24(sp)
+; RV32IZFINXZDINX-NEXT:    lw a4, 28(sp)
+; RV32IZFINXZDINX-NEXT:    sw a1, 0(s0)
+; RV32IZFINXZDINX-NEXT:    sw a2, 4(s0)
+; RV32IZFINXZDINX-NEXT:    sw a3, 8(s0)
+; RV32IZFINXZDINX-NEXT:    sw a4, 12(s0)
+; RV32IZFINXZDINX-NEXT:    sw a0, 16(s0)
 ; RV32IZFINXZDINX-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    addi sp, sp, 48
@@ -1672,8 +1672,8 @@ define { fp128, i32 } @test_frexp_f128_i32(fp128 %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    mv a1, a3
 ; RV64IZFINXZDINX-NEXT:    call frexpl
 ; RV64IZFINXZDINX-NEXT:    lw a2, 12(sp)
-; RV64IZFINXZDINX-NEXT:    sd a1, 8(s0)
 ; RV64IZFINXZDINX-NEXT:    sd a0, 0(s0)
+; RV64IZFINXZDINX-NEXT:    sd a1, 8(s0)
 ; RV64IZFINXZDINX-NEXT:    sw a2, 16(s0)
 ; RV64IZFINXZDINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -1686,28 +1686,28 @@ define { fp128, i32 } @test_frexp_f128_i32(fp128 %a) nounwind {
 ; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a3, 0(a1)
-; RV32I-NEXT:    lw a2, 4(a1)
-; RV32I-NEXT:    lw a4, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a6, 12(a1)
 ; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    sw a1, 12(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    addi a0, sp, 16
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    addi a2, sp, 36
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    call frexpl
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    lw a1, 28(sp)
-; RV32I-NEXT:    lw a2, 16(sp)
-; RV32I-NEXT:    lw a3, 20(sp)
-; RV32I-NEXT:    lw a4, 36(sp)
-; RV32I-NEXT:    sw a1, 12(s0)
-; RV32I-NEXT:    sw a0, 8(s0)
-; RV32I-NEXT:    sw a3, 4(s0)
-; RV32I-NEXT:    sw a2, 0(s0)
-; RV32I-NEXT:    sw a4, 16(s0)
+; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a1, 16(sp)
+; RV32I-NEXT:    lw a2, 20(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
+; RV32I-NEXT:    lw a4, 28(sp)
+; RV32I-NEXT:    sw a1, 0(s0)
+; RV32I-NEXT:    sw a2, 4(s0)
+; RV32I-NEXT:    sw a3, 8(s0)
+; RV32I-NEXT:    sw a4, 12(s0)
+; RV32I-NEXT:    sw a0, 16(s0)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -1725,8 +1725,8 @@ define { fp128, i32 } @test_frexp_f128_i32(fp128 %a) nounwind {
 ; RV64I-NEXT:    mv a1, a3
 ; RV64I-NEXT:    call frexpl
 ; RV64I-NEXT:    lw a2, 12(sp)
-; RV64I-NEXT:    sd a1, 8(s0)
 ; RV64I-NEXT:    sd a0, 0(s0)
+; RV64I-NEXT:    sd a1, 8(s0)
 ; RV64I-NEXT:    sw a2, 16(s0)
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -1743,26 +1743,26 @@ define fp128 @test_frexp_f128_i32_only_use_fract(fp128 %a) nounwind {
 ; RV32IFD-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lw a3, 0(a1)
-; RV32IFD-NEXT:    lw a2, 4(a1)
-; RV32IFD-NEXT:    lw a4, 8(a1)
-; RV32IFD-NEXT:    lw a1, 12(a1)
+; RV32IFD-NEXT:    lw a4, 4(a1)
+; RV32IFD-NEXT:    lw a5, 8(a1)
+; RV32IFD-NEXT:    lw a6, 12(a1)
 ; RV32IFD-NEXT:    mv s0, a0
-; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    sw a4, 8(sp)
-; RV32IFD-NEXT:    sw a2, 4(sp)
 ; RV32IFD-NEXT:    addi a0, sp, 16
 ; RV32IFD-NEXT:    mv a1, sp
 ; RV32IFD-NEXT:    addi a2, sp, 36
 ; RV32IFD-NEXT:    sw a3, 0(sp)
+; RV32IFD-NEXT:    sw a4, 4(sp)
+; RV32IFD-NEXT:    sw a5, 8(sp)
+; RV32IFD-NEXT:    sw a6, 12(sp)
 ; RV32IFD-NEXT:    call frexpl
-; RV32IFD-NEXT:    lw a0, 28(sp)
-; RV32IFD-NEXT:    lw a1, 24(sp)
-; RV32IFD-NEXT:    lw a2, 20(sp)
-; RV32IFD-NEXT:    lw a3, 16(sp)
-; RV32IFD-NEXT:    sw a0, 12(s0)
-; RV32IFD-NEXT:    sw a1, 8(s0)
-; RV32IFD-NEXT:    sw a2, 4(s0)
-; RV32IFD-NEXT:    sw a3, 0(s0)
+; RV32IFD-NEXT:    lw a0, 16(sp)
+; RV32IFD-NEXT:    lw a1, 20(sp)
+; RV32IFD-NEXT:    lw a2, 24(sp)
+; RV32IFD-NEXT:    lw a3, 28(sp)
+; RV32IFD-NEXT:    sw a0, 0(s0)
+; RV32IFD-NEXT:    sw a1, 4(s0)
+; RV32IFD-NEXT:    sw a2, 8(s0)
+; RV32IFD-NEXT:    sw a3, 12(s0)
 ; RV32IFD-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 48
@@ -1784,26 +1784,26 @@ define fp128 @test_frexp_f128_i32_only_use_fract(fp128 %a) nounwind {
 ; RV32IZFINXZDINX-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    lw a3, 0(a1)
-; RV32IZFINXZDINX-NEXT:    lw a2, 4(a1)
-; RV32IZFINXZDINX-NEXT:    lw a4, 8(a1)
-; RV32IZFINXZDINX-NEXT:    lw a1, 12(a1)
+; RV32IZFINXZDINX-NEXT:    lw a4, 4(a1)
+; RV32IZFINXZDINX-NEXT:    lw a5, 8(a1)
+; RV32IZFINXZDINX-NEXT:    lw a6, 12(a1)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
-; RV32IZFINXZDINX-NEXT:    sw a1, 12(sp)
-; RV32IZFINXZDINX-NEXT:    sw a4, 8(sp)
-; RV32IZFINXZDINX-NEXT:    sw a2, 4(sp)
 ; RV32IZFINXZDINX-NEXT:    addi a0, sp, 16
 ; RV32IZFINXZDINX-NEXT:    mv a1, sp
 ; RV32IZFINXZDINX-NEXT:    addi a2, sp, 36
 ; RV32IZFINXZDINX-NEXT:    sw a3, 0(sp)
+; RV32IZFINXZDINX-NEXT:    sw a4, 4(sp)
+; RV32IZFINXZDINX-NEXT:    sw a5, 8(sp)
+; RV32IZFINXZDINX-NEXT:    sw a6, 12(sp)
 ; RV32IZFINXZDINX-NEXT:    call frexpl
-; RV32IZFINXZDINX-NEXT:    lw a0, 28(sp)
-; RV32IZFINXZDINX-NEXT:    lw a1, 24(sp)
-; RV32IZFINXZDINX-NEXT:    lw a2, 20(sp)
-; RV32IZFINXZDINX-NEXT:    lw a3, 16(sp)
-; RV32IZFINXZDINX-NEXT:    sw a0, 12(s0)
-; RV32IZFINXZDINX-NEXT:    sw a1, 8(s0)
-; RV32IZFINXZDINX-NEXT:    sw a2, 4(s0)
-; RV32IZFINXZDINX-NEXT:    sw a3, 0(s0)
+; RV32IZFINXZDINX-NEXT:    lw a0, 16(sp)
+; RV32IZFINXZDINX-NEXT:    lw a1, 20(sp)
+; RV32IZFINXZDINX-NEXT:    lw a2, 24(sp)
+; RV32IZFINXZDINX-NEXT:    lw a3, 28(sp)
+; RV32IZFINXZDINX-NEXT:    sw a0, 0(s0)
+; RV32IZFINXZDINX-NEXT:    sw a1, 4(s0)
+; RV32IZFINXZDINX-NEXT:    sw a2, 8(s0)
+; RV32IZFINXZDINX-NEXT:    sw a3, 12(s0)
 ; RV32IZFINXZDINX-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    addi sp, sp, 48
@@ -1825,26 +1825,26 @@ define fp128 @test_frexp_f128_i32_only_use_fract(fp128 %a) nounwind {
 ; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a3, 0(a1)
-; RV32I-NEXT:    lw a2, 4(a1)
-; RV32I-NEXT:    lw a4, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a6, 12(a1)
 ; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    sw a1, 12(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    addi a0, sp, 16
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    addi a2, sp, 36
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    call frexpl
-; RV32I-NEXT:    lw a0, 28(sp)
-; RV32I-NEXT:    lw a1, 24(sp)
-; RV32I-NEXT:    lw a2, 20(sp)
-; RV32I-NEXT:    lw a3, 16(sp)
-; RV32I-NEXT:    sw a0, 12(s0)
-; RV32I-NEXT:    sw a1, 8(s0)
-; RV32I-NEXT:    sw a2, 4(s0)
-; RV32I-NEXT:    sw a3, 0(s0)
+; RV32I-NEXT:    lw a0, 16(sp)
+; RV32I-NEXT:    lw a1, 20(sp)
+; RV32I-NEXT:    lw a2, 24(sp)
+; RV32I-NEXT:    lw a3, 28(sp)
+; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 4(s0)
+; RV32I-NEXT:    sw a2, 8(s0)
+; RV32I-NEXT:    sw a3, 12(s0)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -1870,16 +1870,16 @@ define i32 @test_frexp_f128_i32_only_use_exp(fp128 %a) nounwind {
 ; RV32IFD-NEXT:    addi sp, sp, -48
 ; RV32IFD-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lw a3, 0(a0)
-; RV32IFD-NEXT:    lw a1, 4(a0)
-; RV32IFD-NEXT:    lw a2, 8(a0)
-; RV32IFD-NEXT:    lw a0, 12(a0)
-; RV32IFD-NEXT:    sw a0, 20(sp)
-; RV32IFD-NEXT:    sw a2, 16(sp)
-; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    lw a4, 4(a0)
+; RV32IFD-NEXT:    lw a5, 8(a0)
+; RV32IFD-NEXT:    lw a6, 12(a0)
 ; RV32IFD-NEXT:    addi a0, sp, 24
 ; RV32IFD-NEXT:    addi a1, sp, 8
 ; RV32IFD-NEXT:    addi a2, sp, 40
 ; RV32IFD-NEXT:    sw a3, 8(sp)
+; RV32IFD-NEXT:    sw a4, 12(sp)
+; RV32IFD-NEXT:    sw a5, 16(sp)
+; RV32IFD-NEXT:    sw a6, 20(sp)
 ; RV32IFD-NEXT:    call frexpl
 ; RV32IFD-NEXT:    lw a0, 40(sp)
 ; RV32IFD-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
@@ -1902,16 +1902,16 @@ define i32 @test_frexp_f128_i32_only_use_exp(fp128 %a) nounwind {
 ; RV32IZFINXZDINX-NEXT:    addi sp, sp, -48
 ; RV32IZFINXZDINX-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    lw a3, 0(a0)
-; RV32IZFINXZDINX-NEXT:    lw a1, 4(a0)
-; RV32IZFINXZDINX-NEXT:    lw a2, 8(a0)
-; RV32IZFINXZDINX-NEXT:    lw a0, 12(a0)
-; RV32IZFINXZDINX-NEXT:    sw a0, 20(sp)
-; RV32IZFINXZDINX-NEXT:    sw a2, 16(sp)
-; RV32IZFINXZDINX-NEXT:    sw a1, 12(sp)
+; RV32IZFINXZDINX-NEXT:    lw a4, 4(a0)
+; RV32IZFINXZDINX-NEXT:    lw a5, 8(a0)
+; RV32IZFINXZDINX-NEXT:    lw a6, 12(a0)
 ; RV32IZFINXZDINX-NEXT:    addi a0, sp, 24
 ; RV32IZFINXZDINX-NEXT:    addi a1, sp, 8
 ; RV32IZFINXZDINX-NEXT:    addi a2, sp, 40
 ; RV32IZFINXZDINX-NEXT:    sw a3, 8(sp)
+; RV32IZFINXZDINX-NEXT:    sw a4, 12(sp)
+; RV32IZFINXZDINX-NEXT:    sw a5, 16(sp)
+; RV32IZFINXZDINX-NEXT:    sw a6, 20(sp)
 ; RV32IZFINXZDINX-NEXT:    call frexpl
 ; RV32IZFINXZDINX-NEXT:    lw a0, 40(sp)
 ; RV32IZFINXZDINX-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
@@ -1934,16 +1934,16 @@ define i32 @test_frexp_f128_i32_only_use_exp(fp128 %a) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a3, 0(a0)
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 8(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    sw a2, 16(sp)
-; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    lw a4, 4(a0)
+; RV32I-NEXT:    lw a5, 8(a0)
+; RV32I-NEXT:    lw a6, 12(a0)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a2, sp, 40
 ; RV32I-NEXT:    sw a3, 8(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a6, 20(sp)
 ; RV32I-NEXT:    call frexpl
 ; RV32I-NEXT:    lw a0, 40(sp)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/memcpy.ll b/llvm/test/CodeGen/RISCV/memcpy.ll
index 41c27d83defe618f4df24fb8a22757d849730634..5b8955ee0e0a0c7614551d24999b9881db25ba17 100644
--- a/llvm/test/CodeGen/RISCV/memcpy.ll
+++ b/llvm/test/CodeGen/RISCV/memcpy.ll
@@ -25,16 +25,16 @@ define i32 @t0() {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    lui a0, %hi(src)
 ; RV32-NEXT:    lw a1, %lo(src)(a0)
+; RV32-NEXT:    lui a2, %hi(dst)
+; RV32-NEXT:    sw a1, %lo(dst)(a2)
 ; RV32-NEXT:    addi a0, a0, %lo(src)
-; RV32-NEXT:    lw a2, 4(a0)
+; RV32-NEXT:    lw a1, 4(a0)
 ; RV32-NEXT:    lh a3, 8(a0)
 ; RV32-NEXT:    lbu a0, 10(a0)
-; RV32-NEXT:    lui a4, %hi(dst)
-; RV32-NEXT:    sw a1, %lo(dst)(a4)
-; RV32-NEXT:    addi a1, a4, %lo(dst)
-; RV32-NEXT:    sb a0, 10(a1)
-; RV32-NEXT:    sh a3, 8(a1)
-; RV32-NEXT:    sw a2, 4(a1)
+; RV32-NEXT:    addi a2, a2, %lo(dst)
+; RV32-NEXT:    sw a1, 4(a2)
+; RV32-NEXT:    sh a3, 8(a2)
+; RV32-NEXT:    sb a0, 10(a2)
 ; RV32-NEXT:    li a0, 0
 ; RV32-NEXT:    ret
 ;
@@ -42,14 +42,14 @@ define i32 @t0() {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    lui a0, %hi(src)
 ; RV64-NEXT:    ld a1, %lo(src)(a0)
+; RV64-NEXT:    lui a2, %hi(dst)
 ; RV64-NEXT:    addi a0, a0, %lo(src)
-; RV64-NEXT:    lh a2, 8(a0)
+; RV64-NEXT:    lh a3, 8(a0)
 ; RV64-NEXT:    lbu a0, 10(a0)
-; RV64-NEXT:    lui a3, %hi(dst)
-; RV64-NEXT:    sd a1, %lo(dst)(a3)
-; RV64-NEXT:    addi a1, a3, %lo(dst)
+; RV64-NEXT:    sd a1, %lo(dst)(a2)
+; RV64-NEXT:    addi a1, a2, %lo(dst)
+; RV64-NEXT:    sh a3, 8(a1)
 ; RV64-NEXT:    sb a0, 10(a1)
-; RV64-NEXT:    sh a2, 8(a1)
 ; RV64-NEXT:    li a0, 0
 ; RV64-NEXT:    ret
 ;
@@ -107,39 +107,39 @@ define void @t1(ptr nocapture %C) nounwind {
 ; RV32-FAST-NEXT:    sw a1, 27(a0)
 ; RV32-FAST-NEXT:    lui a1, 300325
 ; RV32-FAST-NEXT:    addi a1, a1, 1107
+; RV32-FAST-NEXT:    lui a2, 132181
+; RV32-FAST-NEXT:    addi a2, a2, -689
+; RV32-FAST-NEXT:    lui a3, 340483
+; RV32-FAST-NEXT:    addi a3, a3, -947
+; RV32-FAST-NEXT:    sw a3, 16(a0)
+; RV32-FAST-NEXT:    sw a2, 20(a0)
 ; RV32-FAST-NEXT:    sw a1, 24(a0)
-; RV32-FAST-NEXT:    lui a1, 132181
-; RV32-FAST-NEXT:    addi a1, a1, -689
-; RV32-FAST-NEXT:    sw a1, 20(a0)
-; RV32-FAST-NEXT:    lui a1, 340483
-; RV32-FAST-NEXT:    addi a1, a1, -947
-; RV32-FAST-NEXT:    sw a1, 16(a0)
 ; RV32-FAST-NEXT:    lui a1, 267556
 ; RV32-FAST-NEXT:    addi a1, a1, 1871
+; RV32-FAST-NEXT:    lui a2, 337154
+; RV32-FAST-NEXT:    addi a2, a2, 69
+; RV32-FAST-NEXT:    lui a3, 320757
+; RV32-FAST-NEXT:    addi a3, a3, 1107
+; RV32-FAST-NEXT:    lui a4, 365861
+; RV32-FAST-NEXT:    addi a4, a4, -1980
+; RV32-FAST-NEXT:    sw a4, 0(a0)
+; RV32-FAST-NEXT:    sw a3, 4(a0)
+; RV32-FAST-NEXT:    sw a2, 8(a0)
 ; RV32-FAST-NEXT:    sw a1, 12(a0)
-; RV32-FAST-NEXT:    lui a1, 337154
-; RV32-FAST-NEXT:    addi a1, a1, 69
-; RV32-FAST-NEXT:    sw a1, 8(a0)
-; RV32-FAST-NEXT:    lui a1, 320757
-; RV32-FAST-NEXT:    addi a1, a1, 1107
-; RV32-FAST-NEXT:    sw a1, 4(a0)
-; RV32-FAST-NEXT:    lui a1, 365861
-; RV32-FAST-NEXT:    addi a1, a1, -1980
-; RV32-FAST-NEXT:    sw a1, 0(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: t1:
 ; RV64-FAST:       # %bb.0: # %entry
 ; RV64-FAST-NEXT:    lui a1, %hi(.L.str1)
-; RV64-FAST-NEXT:    ld a2, %lo(.L.str1)(a1)
-; RV64-FAST-NEXT:    addi a1, a1, %lo(.L.str1)
-; RV64-FAST-NEXT:    ld a3, 23(a1)
-; RV64-FAST-NEXT:    ld a4, 16(a1)
-; RV64-FAST-NEXT:    ld a1, 8(a1)
-; RV64-FAST-NEXT:    sd a2, 0(a0)
+; RV64-FAST-NEXT:    addi a2, a1, %lo(.L.str1)
+; RV64-FAST-NEXT:    ld a3, 23(a2)
+; RV64-FAST-NEXT:    ld a1, %lo(.L.str1)(a1)
+; RV64-FAST-NEXT:    ld a4, 8(a2)
+; RV64-FAST-NEXT:    ld a2, 16(a2)
 ; RV64-FAST-NEXT:    sd a3, 23(a0)
-; RV64-FAST-NEXT:    sd a4, 16(a0)
-; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    sd a1, 0(a0)
+; RV64-FAST-NEXT:    sd a4, 8(a0)
+; RV64-FAST-NEXT:    sd a2, 16(a0)
 ; RV64-FAST-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %C, ptr @.str1, i64 31, i1 false)
@@ -165,17 +165,17 @@ define void @t2(ptr nocapture %C) nounwind {
 ; RV64-FAST:       # %bb.0: # %entry
 ; RV64-FAST-NEXT:    lui a1, %hi(.L.str2)
 ; RV64-FAST-NEXT:    ld a2, %lo(.L.str2)(a1)
-; RV64-FAST-NEXT:    sd a2, 0(a0)
+; RV64-FAST-NEXT:    lui a3, 1156
+; RV64-FAST-NEXT:    addi a3, a3, 332
+; RV64-FAST-NEXT:    sw a3, 32(a0)
 ; RV64-FAST-NEXT:    addi a1, a1, %lo(.L.str2)
-; RV64-FAST-NEXT:    ld a2, 8(a1)
-; RV64-FAST-NEXT:    ld a3, 16(a1)
+; RV64-FAST-NEXT:    ld a3, 8(a1)
+; RV64-FAST-NEXT:    ld a4, 16(a1)
 ; RV64-FAST-NEXT:    ld a1, 24(a1)
-; RV64-FAST-NEXT:    lui a4, 1156
-; RV64-FAST-NEXT:    addi a4, a4, 332
-; RV64-FAST-NEXT:    sw a4, 32(a0)
+; RV64-FAST-NEXT:    sd a2, 0(a0)
+; RV64-FAST-NEXT:    sd a3, 8(a0)
+; RV64-FAST-NEXT:    sd a4, 16(a0)
 ; RV64-FAST-NEXT:    sd a1, 24(a0)
-; RV64-FAST-NEXT:    sd a3, 16(a0)
-; RV64-FAST-NEXT:    sd a2, 8(a0)
 ; RV64-FAST-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %C, ptr @.str2, i64 36, i1 false)
@@ -201,22 +201,22 @@ define void @t3(ptr nocapture %C) nounwind {
 ; RV32-FAST:       # %bb.0: # %entry
 ; RV32-FAST-NEXT:    lui a1, 1109
 ; RV32-FAST-NEXT:    addi a1, a1, -689
+; RV32-FAST-NEXT:    lui a2, 340483
+; RV32-FAST-NEXT:    addi a2, a2, -947
+; RV32-FAST-NEXT:    sw a2, 16(a0)
 ; RV32-FAST-NEXT:    sw a1, 20(a0)
-; RV32-FAST-NEXT:    lui a1, 340483
-; RV32-FAST-NEXT:    addi a1, a1, -947
-; RV32-FAST-NEXT:    sw a1, 16(a0)
 ; RV32-FAST-NEXT:    lui a1, 267556
 ; RV32-FAST-NEXT:    addi a1, a1, 1871
+; RV32-FAST-NEXT:    lui a2, 337154
+; RV32-FAST-NEXT:    addi a2, a2, 69
+; RV32-FAST-NEXT:    lui a3, 320757
+; RV32-FAST-NEXT:    addi a3, a3, 1107
+; RV32-FAST-NEXT:    lui a4, 365861
+; RV32-FAST-NEXT:    addi a4, a4, -1980
+; RV32-FAST-NEXT:    sw a4, 0(a0)
+; RV32-FAST-NEXT:    sw a3, 4(a0)
+; RV32-FAST-NEXT:    sw a2, 8(a0)
 ; RV32-FAST-NEXT:    sw a1, 12(a0)
-; RV32-FAST-NEXT:    lui a1, 337154
-; RV32-FAST-NEXT:    addi a1, a1, 69
-; RV32-FAST-NEXT:    sw a1, 8(a0)
-; RV32-FAST-NEXT:    lui a1, 320757
-; RV32-FAST-NEXT:    addi a1, a1, 1107
-; RV32-FAST-NEXT:    sw a1, 4(a0)
-; RV32-FAST-NEXT:    lui a1, 365861
-; RV32-FAST-NEXT:    addi a1, a1, -1980
-; RV32-FAST-NEXT:    sw a1, 0(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: t3:
@@ -224,11 +224,11 @@ define void @t3(ptr nocapture %C) nounwind {
 ; RV64-FAST-NEXT:    lui a1, %hi(.L.str3)
 ; RV64-FAST-NEXT:    ld a2, %lo(.L.str3)(a1)
 ; RV64-FAST-NEXT:    addi a1, a1, %lo(.L.str3)
-; RV64-FAST-NEXT:    ld a3, 16(a1)
-; RV64-FAST-NEXT:    ld a1, 8(a1)
+; RV64-FAST-NEXT:    ld a3, 8(a1)
+; RV64-FAST-NEXT:    ld a1, 16(a1)
 ; RV64-FAST-NEXT:    sd a2, 0(a0)
-; RV64-FAST-NEXT:    sd a3, 16(a0)
-; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    sd a3, 8(a0)
+; RV64-FAST-NEXT:    sd a1, 16(a0)
 ; RV64-FAST-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %C, ptr @.str3, i64 24, i1 false)
@@ -256,16 +256,16 @@ define void @t4(ptr nocapture %C) nounwind {
 ; RV32-FAST-NEXT:    sh a1, 16(a0)
 ; RV32-FAST-NEXT:    lui a1, 132388
 ; RV32-FAST-NEXT:    addi a1, a1, 1871
+; RV32-FAST-NEXT:    lui a2, 337154
+; RV32-FAST-NEXT:    addi a2, a2, 69
+; RV32-FAST-NEXT:    lui a3, 320757
+; RV32-FAST-NEXT:    addi a3, a3, 1107
+; RV32-FAST-NEXT:    lui a4, 365861
+; RV32-FAST-NEXT:    addi a4, a4, -1980
+; RV32-FAST-NEXT:    sw a4, 0(a0)
+; RV32-FAST-NEXT:    sw a3, 4(a0)
+; RV32-FAST-NEXT:    sw a2, 8(a0)
 ; RV32-FAST-NEXT:    sw a1, 12(a0)
-; RV32-FAST-NEXT:    lui a1, 337154
-; RV32-FAST-NEXT:    addi a1, a1, 69
-; RV32-FAST-NEXT:    sw a1, 8(a0)
-; RV32-FAST-NEXT:    lui a1, 320757
-; RV32-FAST-NEXT:    addi a1, a1, 1107
-; RV32-FAST-NEXT:    sw a1, 4(a0)
-; RV32-FAST-NEXT:    lui a1, 365861
-; RV32-FAST-NEXT:    addi a1, a1, -1980
-; RV32-FAST-NEXT:    sw a1, 0(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: t4:
@@ -275,9 +275,9 @@ define void @t4(ptr nocapture %C) nounwind {
 ; RV64-FAST-NEXT:    addi a1, a1, %lo(.L.str4)
 ; RV64-FAST-NEXT:    ld a1, 8(a1)
 ; RV64-FAST-NEXT:    li a3, 32
-; RV64-FAST-NEXT:    sh a3, 16(a0)
 ; RV64-FAST-NEXT:    sd a2, 0(a0)
 ; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    sh a3, 16(a0)
 ; RV64-FAST-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %C, ptr @.str4, i64 18, i1 false)
@@ -287,36 +287,36 @@ entry:
 define void @t5(ptr nocapture %C) nounwind {
 ; RV32-LABEL: t5:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    sb zero, 6(a0)
 ; RV32-NEXT:    li a1, 84
+; RV32-NEXT:    li a2, 83
+; RV32-NEXT:    sb a2, 4(a0)
 ; RV32-NEXT:    sb a1, 5(a0)
-; RV32-NEXT:    li a1, 83
-; RV32-NEXT:    sb a1, 4(a0)
+; RV32-NEXT:    sb zero, 6(a0)
 ; RV32-NEXT:    li a1, 89
+; RV32-NEXT:    li a2, 82
+; RV32-NEXT:    li a3, 72
+; RV32-NEXT:    li a4, 68
+; RV32-NEXT:    sb a4, 0(a0)
+; RV32-NEXT:    sb a3, 1(a0)
+; RV32-NEXT:    sb a2, 2(a0)
 ; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    li a1, 82
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    li a1, 72
-; RV32-NEXT:    sb a1, 1(a0)
-; RV32-NEXT:    li a1, 68
-; RV32-NEXT:    sb a1, 0(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: t5:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    sb zero, 6(a0)
 ; RV64-NEXT:    li a1, 84
+; RV64-NEXT:    li a2, 83
+; RV64-NEXT:    sb a2, 4(a0)
 ; RV64-NEXT:    sb a1, 5(a0)
-; RV64-NEXT:    li a1, 83
-; RV64-NEXT:    sb a1, 4(a0)
+; RV64-NEXT:    sb zero, 6(a0)
 ; RV64-NEXT:    li a1, 89
+; RV64-NEXT:    li a2, 82
+; RV64-NEXT:    li a3, 72
+; RV64-NEXT:    li a4, 68
+; RV64-NEXT:    sb a4, 0(a0)
+; RV64-NEXT:    sb a3, 1(a0)
+; RV64-NEXT:    sb a2, 2(a0)
 ; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    li a1, 82
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    li a1, 72
-; RV64-NEXT:    sb a1, 1(a0)
-; RV64-NEXT:    li a1, 68
-; RV64-NEXT:    sb a1, 0(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: t5:
diff --git a/llvm/test/CodeGen/RISCV/memset-inline.ll b/llvm/test/CodeGen/RISCV/memset-inline.ll
index 55fe81a58805edc5147e5085ac4ff99c816fcccc..6ee6e1261e7e916ec419d5c39b1ae90d1c110c5b 100644
--- a/llvm/test/CodeGen/RISCV/memset-inline.ll
+++ b/llvm/test/CodeGen/RISCV/memset-inline.ll
@@ -31,14 +31,14 @@ define void @memset_1(ptr %a, i8 %value) nounwind {
 define void @memset_2(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_2:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 1(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_2:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 1(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_2:
@@ -63,18 +63,18 @@ define void @memset_2(ptr %a, i8 %value) nounwind {
 define void @memset_4(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_4:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    sb a1, 1(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_4:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    sb a1, 1(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_4:
@@ -102,26 +102,26 @@ define void @memset_4(ptr %a, i8 %value) nounwind {
 define void @memset_8(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 7(a0)
-; RV32-NEXT:    sb a1, 6(a0)
-; RV32-NEXT:    sb a1, 5(a0)
 ; RV32-NEXT:    sb a1, 4(a0)
-; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 6(a0)
+; RV32-NEXT:    sb a1, 7(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_8:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 7(a0)
-; RV64-NEXT:    sb a1, 6(a0)
-; RV64-NEXT:    sb a1, 5(a0)
 ; RV64-NEXT:    sb a1, 4(a0)
-; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 6(a0)
+; RV64-NEXT:    sb a1, 7(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_8:
@@ -130,8 +130,8 @@ define void @memset_8(ptr %a, i8 %value) nounwind {
 ; RV32-FAST-NEXT:    lui a2, 4112
 ; RV32-FAST-NEXT:    addi a2, a2, 257
 ; RV32-FAST-NEXT:    mul a1, a1, a2
-; RV32-FAST-NEXT:    sw a1, 4(a0)
 ; RV32-FAST-NEXT:    sw a1, 0(a0)
+; RV32-FAST-NEXT:    sw a1, 4(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: memset_8:
@@ -151,42 +151,42 @@ define void @memset_8(ptr %a, i8 %value) nounwind {
 define void @memset_16(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 15(a0)
-; RV32-NEXT:    sb a1, 14(a0)
-; RV32-NEXT:    sb a1, 13(a0)
 ; RV32-NEXT:    sb a1, 12(a0)
-; RV32-NEXT:    sb a1, 11(a0)
-; RV32-NEXT:    sb a1, 10(a0)
-; RV32-NEXT:    sb a1, 9(a0)
+; RV32-NEXT:    sb a1, 13(a0)
+; RV32-NEXT:    sb a1, 14(a0)
+; RV32-NEXT:    sb a1, 15(a0)
 ; RV32-NEXT:    sb a1, 8(a0)
-; RV32-NEXT:    sb a1, 7(a0)
-; RV32-NEXT:    sb a1, 6(a0)
-; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 9(a0)
+; RV32-NEXT:    sb a1, 10(a0)
+; RV32-NEXT:    sb a1, 11(a0)
 ; RV32-NEXT:    sb a1, 4(a0)
-; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 6(a0)
+; RV32-NEXT:    sb a1, 7(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_16:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 15(a0)
-; RV64-NEXT:    sb a1, 14(a0)
-; RV64-NEXT:    sb a1, 13(a0)
 ; RV64-NEXT:    sb a1, 12(a0)
-; RV64-NEXT:    sb a1, 11(a0)
-; RV64-NEXT:    sb a1, 10(a0)
-; RV64-NEXT:    sb a1, 9(a0)
+; RV64-NEXT:    sb a1, 13(a0)
+; RV64-NEXT:    sb a1, 14(a0)
+; RV64-NEXT:    sb a1, 15(a0)
 ; RV64-NEXT:    sb a1, 8(a0)
-; RV64-NEXT:    sb a1, 7(a0)
-; RV64-NEXT:    sb a1, 6(a0)
-; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 9(a0)
+; RV64-NEXT:    sb a1, 10(a0)
+; RV64-NEXT:    sb a1, 11(a0)
 ; RV64-NEXT:    sb a1, 4(a0)
-; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 6(a0)
+; RV64-NEXT:    sb a1, 7(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_16:
@@ -195,10 +195,10 @@ define void @memset_16(ptr %a, i8 %value) nounwind {
 ; RV32-FAST-NEXT:    lui a2, 4112
 ; RV32-FAST-NEXT:    addi a2, a2, 257
 ; RV32-FAST-NEXT:    mul a1, a1, a2
-; RV32-FAST-NEXT:    sw a1, 12(a0)
-; RV32-FAST-NEXT:    sw a1, 8(a0)
-; RV32-FAST-NEXT:    sw a1, 4(a0)
 ; RV32-FAST-NEXT:    sw a1, 0(a0)
+; RV32-FAST-NEXT:    sw a1, 4(a0)
+; RV32-FAST-NEXT:    sw a1, 8(a0)
+; RV32-FAST-NEXT:    sw a1, 12(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: memset_16:
@@ -209,8 +209,8 @@ define void @memset_16(ptr %a, i8 %value) nounwind {
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
-; RV64-FAST-NEXT:    sd a1, 8(a0)
 ; RV64-FAST-NEXT:    sd a1, 0(a0)
+; RV64-FAST-NEXT:    sd a1, 8(a0)
 ; RV64-FAST-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 %value, i64 16, i1 0)
   ret void
@@ -219,74 +219,74 @@ define void @memset_16(ptr %a, i8 %value) nounwind {
 define void @memset_32(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 31(a0)
-; RV32-NEXT:    sb a1, 30(a0)
-; RV32-NEXT:    sb a1, 29(a0)
 ; RV32-NEXT:    sb a1, 28(a0)
-; RV32-NEXT:    sb a1, 27(a0)
-; RV32-NEXT:    sb a1, 26(a0)
-; RV32-NEXT:    sb a1, 25(a0)
+; RV32-NEXT:    sb a1, 29(a0)
+; RV32-NEXT:    sb a1, 30(a0)
+; RV32-NEXT:    sb a1, 31(a0)
 ; RV32-NEXT:    sb a1, 24(a0)
-; RV32-NEXT:    sb a1, 23(a0)
-; RV32-NEXT:    sb a1, 22(a0)
-; RV32-NEXT:    sb a1, 21(a0)
+; RV32-NEXT:    sb a1, 25(a0)
+; RV32-NEXT:    sb a1, 26(a0)
+; RV32-NEXT:    sb a1, 27(a0)
 ; RV32-NEXT:    sb a1, 20(a0)
-; RV32-NEXT:    sb a1, 19(a0)
-; RV32-NEXT:    sb a1, 18(a0)
-; RV32-NEXT:    sb a1, 17(a0)
+; RV32-NEXT:    sb a1, 21(a0)
+; RV32-NEXT:    sb a1, 22(a0)
+; RV32-NEXT:    sb a1, 23(a0)
 ; RV32-NEXT:    sb a1, 16(a0)
-; RV32-NEXT:    sb a1, 15(a0)
-; RV32-NEXT:    sb a1, 14(a0)
-; RV32-NEXT:    sb a1, 13(a0)
+; RV32-NEXT:    sb a1, 17(a0)
+; RV32-NEXT:    sb a1, 18(a0)
+; RV32-NEXT:    sb a1, 19(a0)
 ; RV32-NEXT:    sb a1, 12(a0)
-; RV32-NEXT:    sb a1, 11(a0)
-; RV32-NEXT:    sb a1, 10(a0)
-; RV32-NEXT:    sb a1, 9(a0)
+; RV32-NEXT:    sb a1, 13(a0)
+; RV32-NEXT:    sb a1, 14(a0)
+; RV32-NEXT:    sb a1, 15(a0)
 ; RV32-NEXT:    sb a1, 8(a0)
-; RV32-NEXT:    sb a1, 7(a0)
-; RV32-NEXT:    sb a1, 6(a0)
-; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 9(a0)
+; RV32-NEXT:    sb a1, 10(a0)
+; RV32-NEXT:    sb a1, 11(a0)
 ; RV32-NEXT:    sb a1, 4(a0)
-; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 6(a0)
+; RV32-NEXT:    sb a1, 7(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 31(a0)
-; RV64-NEXT:    sb a1, 30(a0)
-; RV64-NEXT:    sb a1, 29(a0)
 ; RV64-NEXT:    sb a1, 28(a0)
-; RV64-NEXT:    sb a1, 27(a0)
-; RV64-NEXT:    sb a1, 26(a0)
-; RV64-NEXT:    sb a1, 25(a0)
+; RV64-NEXT:    sb a1, 29(a0)
+; RV64-NEXT:    sb a1, 30(a0)
+; RV64-NEXT:    sb a1, 31(a0)
 ; RV64-NEXT:    sb a1, 24(a0)
-; RV64-NEXT:    sb a1, 23(a0)
-; RV64-NEXT:    sb a1, 22(a0)
-; RV64-NEXT:    sb a1, 21(a0)
+; RV64-NEXT:    sb a1, 25(a0)
+; RV64-NEXT:    sb a1, 26(a0)
+; RV64-NEXT:    sb a1, 27(a0)
 ; RV64-NEXT:    sb a1, 20(a0)
-; RV64-NEXT:    sb a1, 19(a0)
-; RV64-NEXT:    sb a1, 18(a0)
-; RV64-NEXT:    sb a1, 17(a0)
+; RV64-NEXT:    sb a1, 21(a0)
+; RV64-NEXT:    sb a1, 22(a0)
+; RV64-NEXT:    sb a1, 23(a0)
 ; RV64-NEXT:    sb a1, 16(a0)
-; RV64-NEXT:    sb a1, 15(a0)
-; RV64-NEXT:    sb a1, 14(a0)
-; RV64-NEXT:    sb a1, 13(a0)
+; RV64-NEXT:    sb a1, 17(a0)
+; RV64-NEXT:    sb a1, 18(a0)
+; RV64-NEXT:    sb a1, 19(a0)
 ; RV64-NEXT:    sb a1, 12(a0)
-; RV64-NEXT:    sb a1, 11(a0)
-; RV64-NEXT:    sb a1, 10(a0)
-; RV64-NEXT:    sb a1, 9(a0)
+; RV64-NEXT:    sb a1, 13(a0)
+; RV64-NEXT:    sb a1, 14(a0)
+; RV64-NEXT:    sb a1, 15(a0)
 ; RV64-NEXT:    sb a1, 8(a0)
-; RV64-NEXT:    sb a1, 7(a0)
-; RV64-NEXT:    sb a1, 6(a0)
-; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 9(a0)
+; RV64-NEXT:    sb a1, 10(a0)
+; RV64-NEXT:    sb a1, 11(a0)
 ; RV64-NEXT:    sb a1, 4(a0)
-; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 6(a0)
+; RV64-NEXT:    sb a1, 7(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_32:
@@ -295,14 +295,14 @@ define void @memset_32(ptr %a, i8 %value) nounwind {
 ; RV32-FAST-NEXT:    lui a2, 4112
 ; RV32-FAST-NEXT:    addi a2, a2, 257
 ; RV32-FAST-NEXT:    mul a1, a1, a2
-; RV32-FAST-NEXT:    sw a1, 28(a0)
-; RV32-FAST-NEXT:    sw a1, 24(a0)
-; RV32-FAST-NEXT:    sw a1, 20(a0)
 ; RV32-FAST-NEXT:    sw a1, 16(a0)
-; RV32-FAST-NEXT:    sw a1, 12(a0)
-; RV32-FAST-NEXT:    sw a1, 8(a0)
-; RV32-FAST-NEXT:    sw a1, 4(a0)
+; RV32-FAST-NEXT:    sw a1, 20(a0)
+; RV32-FAST-NEXT:    sw a1, 24(a0)
+; RV32-FAST-NEXT:    sw a1, 28(a0)
 ; RV32-FAST-NEXT:    sw a1, 0(a0)
+; RV32-FAST-NEXT:    sw a1, 4(a0)
+; RV32-FAST-NEXT:    sw a1, 8(a0)
+; RV32-FAST-NEXT:    sw a1, 12(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: memset_32:
@@ -313,10 +313,10 @@ define void @memset_32(ptr %a, i8 %value) nounwind {
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
-; RV64-FAST-NEXT:    sd a1, 24(a0)
-; RV64-FAST-NEXT:    sd a1, 16(a0)
-; RV64-FAST-NEXT:    sd a1, 8(a0)
 ; RV64-FAST-NEXT:    sd a1, 0(a0)
+; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    sd a1, 16(a0)
+; RV64-FAST-NEXT:    sd a1, 24(a0)
 ; RV64-FAST-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 %value, i64 32, i1 0)
   ret void
@@ -325,138 +325,138 @@ define void @memset_32(ptr %a, i8 %value) nounwind {
 define void @memset_64(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 63(a0)
-; RV32-NEXT:    sb a1, 62(a0)
-; RV32-NEXT:    sb a1, 61(a0)
 ; RV32-NEXT:    sb a1, 60(a0)
-; RV32-NEXT:    sb a1, 59(a0)
-; RV32-NEXT:    sb a1, 58(a0)
-; RV32-NEXT:    sb a1, 57(a0)
+; RV32-NEXT:    sb a1, 61(a0)
+; RV32-NEXT:    sb a1, 62(a0)
+; RV32-NEXT:    sb a1, 63(a0)
 ; RV32-NEXT:    sb a1, 56(a0)
-; RV32-NEXT:    sb a1, 55(a0)
-; RV32-NEXT:    sb a1, 54(a0)
-; RV32-NEXT:    sb a1, 53(a0)
+; RV32-NEXT:    sb a1, 57(a0)
+; RV32-NEXT:    sb a1, 58(a0)
+; RV32-NEXT:    sb a1, 59(a0)
 ; RV32-NEXT:    sb a1, 52(a0)
-; RV32-NEXT:    sb a1, 51(a0)
-; RV32-NEXT:    sb a1, 50(a0)
-; RV32-NEXT:    sb a1, 49(a0)
+; RV32-NEXT:    sb a1, 53(a0)
+; RV32-NEXT:    sb a1, 54(a0)
+; RV32-NEXT:    sb a1, 55(a0)
 ; RV32-NEXT:    sb a1, 48(a0)
-; RV32-NEXT:    sb a1, 47(a0)
-; RV32-NEXT:    sb a1, 46(a0)
-; RV32-NEXT:    sb a1, 45(a0)
+; RV32-NEXT:    sb a1, 49(a0)
+; RV32-NEXT:    sb a1, 50(a0)
+; RV32-NEXT:    sb a1, 51(a0)
 ; RV32-NEXT:    sb a1, 44(a0)
-; RV32-NEXT:    sb a1, 43(a0)
-; RV32-NEXT:    sb a1, 42(a0)
-; RV32-NEXT:    sb a1, 41(a0)
+; RV32-NEXT:    sb a1, 45(a0)
+; RV32-NEXT:    sb a1, 46(a0)
+; RV32-NEXT:    sb a1, 47(a0)
 ; RV32-NEXT:    sb a1, 40(a0)
-; RV32-NEXT:    sb a1, 39(a0)
-; RV32-NEXT:    sb a1, 38(a0)
-; RV32-NEXT:    sb a1, 37(a0)
+; RV32-NEXT:    sb a1, 41(a0)
+; RV32-NEXT:    sb a1, 42(a0)
+; RV32-NEXT:    sb a1, 43(a0)
 ; RV32-NEXT:    sb a1, 36(a0)
-; RV32-NEXT:    sb a1, 35(a0)
-; RV32-NEXT:    sb a1, 34(a0)
-; RV32-NEXT:    sb a1, 33(a0)
-; RV32-NEXT:    sb a1, 32(a0)
-; RV32-NEXT:    sb a1, 31(a0)
-; RV32-NEXT:    sb a1, 30(a0)
-; RV32-NEXT:    sb a1, 29(a0)
+; RV32-NEXT:    sb a1, 37(a0)
+; RV32-NEXT:    sb a1, 38(a0)
+; RV32-NEXT:    sb a1, 39(a0)
+; RV32-NEXT:    sb a1, 32(a0)
+; RV32-NEXT:    sb a1, 33(a0)
+; RV32-NEXT:    sb a1, 34(a0)
+; RV32-NEXT:    sb a1, 35(a0)
 ; RV32-NEXT:    sb a1, 28(a0)
-; RV32-NEXT:    sb a1, 27(a0)
-; RV32-NEXT:    sb a1, 26(a0)
-; RV32-NEXT:    sb a1, 25(a0)
+; RV32-NEXT:    sb a1, 29(a0)
+; RV32-NEXT:    sb a1, 30(a0)
+; RV32-NEXT:    sb a1, 31(a0)
 ; RV32-NEXT:    sb a1, 24(a0)
-; RV32-NEXT:    sb a1, 23(a0)
-; RV32-NEXT:    sb a1, 22(a0)
-; RV32-NEXT:    sb a1, 21(a0)
+; RV32-NEXT:    sb a1, 25(a0)
+; RV32-NEXT:    sb a1, 26(a0)
+; RV32-NEXT:    sb a1, 27(a0)
 ; RV32-NEXT:    sb a1, 20(a0)
-; RV32-NEXT:    sb a1, 19(a0)
-; RV32-NEXT:    sb a1, 18(a0)
-; RV32-NEXT:    sb a1, 17(a0)
+; RV32-NEXT:    sb a1, 21(a0)
+; RV32-NEXT:    sb a1, 22(a0)
+; RV32-NEXT:    sb a1, 23(a0)
 ; RV32-NEXT:    sb a1, 16(a0)
-; RV32-NEXT:    sb a1, 15(a0)
-; RV32-NEXT:    sb a1, 14(a0)
-; RV32-NEXT:    sb a1, 13(a0)
+; RV32-NEXT:    sb a1, 17(a0)
+; RV32-NEXT:    sb a1, 18(a0)
+; RV32-NEXT:    sb a1, 19(a0)
 ; RV32-NEXT:    sb a1, 12(a0)
-; RV32-NEXT:    sb a1, 11(a0)
-; RV32-NEXT:    sb a1, 10(a0)
-; RV32-NEXT:    sb a1, 9(a0)
+; RV32-NEXT:    sb a1, 13(a0)
+; RV32-NEXT:    sb a1, 14(a0)
+; RV32-NEXT:    sb a1, 15(a0)
 ; RV32-NEXT:    sb a1, 8(a0)
-; RV32-NEXT:    sb a1, 7(a0)
-; RV32-NEXT:    sb a1, 6(a0)
-; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 9(a0)
+; RV32-NEXT:    sb a1, 10(a0)
+; RV32-NEXT:    sb a1, 11(a0)
 ; RV32-NEXT:    sb a1, 4(a0)
-; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 6(a0)
+; RV32-NEXT:    sb a1, 7(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 63(a0)
-; RV64-NEXT:    sb a1, 62(a0)
-; RV64-NEXT:    sb a1, 61(a0)
 ; RV64-NEXT:    sb a1, 60(a0)
-; RV64-NEXT:    sb a1, 59(a0)
-; RV64-NEXT:    sb a1, 58(a0)
-; RV64-NEXT:    sb a1, 57(a0)
+; RV64-NEXT:    sb a1, 61(a0)
+; RV64-NEXT:    sb a1, 62(a0)
+; RV64-NEXT:    sb a1, 63(a0)
 ; RV64-NEXT:    sb a1, 56(a0)
-; RV64-NEXT:    sb a1, 55(a0)
-; RV64-NEXT:    sb a1, 54(a0)
-; RV64-NEXT:    sb a1, 53(a0)
+; RV64-NEXT:    sb a1, 57(a0)
+; RV64-NEXT:    sb a1, 58(a0)
+; RV64-NEXT:    sb a1, 59(a0)
 ; RV64-NEXT:    sb a1, 52(a0)
-; RV64-NEXT:    sb a1, 51(a0)
-; RV64-NEXT:    sb a1, 50(a0)
-; RV64-NEXT:    sb a1, 49(a0)
+; RV64-NEXT:    sb a1, 53(a0)
+; RV64-NEXT:    sb a1, 54(a0)
+; RV64-NEXT:    sb a1, 55(a0)
 ; RV64-NEXT:    sb a1, 48(a0)
-; RV64-NEXT:    sb a1, 47(a0)
-; RV64-NEXT:    sb a1, 46(a0)
-; RV64-NEXT:    sb a1, 45(a0)
+; RV64-NEXT:    sb a1, 49(a0)
+; RV64-NEXT:    sb a1, 50(a0)
+; RV64-NEXT:    sb a1, 51(a0)
 ; RV64-NEXT:    sb a1, 44(a0)
-; RV64-NEXT:    sb a1, 43(a0)
-; RV64-NEXT:    sb a1, 42(a0)
-; RV64-NEXT:    sb a1, 41(a0)
+; RV64-NEXT:    sb a1, 45(a0)
+; RV64-NEXT:    sb a1, 46(a0)
+; RV64-NEXT:    sb a1, 47(a0)
 ; RV64-NEXT:    sb a1, 40(a0)
-; RV64-NEXT:    sb a1, 39(a0)
-; RV64-NEXT:    sb a1, 38(a0)
-; RV64-NEXT:    sb a1, 37(a0)
+; RV64-NEXT:    sb a1, 41(a0)
+; RV64-NEXT:    sb a1, 42(a0)
+; RV64-NEXT:    sb a1, 43(a0)
 ; RV64-NEXT:    sb a1, 36(a0)
-; RV64-NEXT:    sb a1, 35(a0)
-; RV64-NEXT:    sb a1, 34(a0)
-; RV64-NEXT:    sb a1, 33(a0)
+; RV64-NEXT:    sb a1, 37(a0)
+; RV64-NEXT:    sb a1, 38(a0)
+; RV64-NEXT:    sb a1, 39(a0)
 ; RV64-NEXT:    sb a1, 32(a0)
-; RV64-NEXT:    sb a1, 31(a0)
-; RV64-NEXT:    sb a1, 30(a0)
-; RV64-NEXT:    sb a1, 29(a0)
+; RV64-NEXT:    sb a1, 33(a0)
+; RV64-NEXT:    sb a1, 34(a0)
+; RV64-NEXT:    sb a1, 35(a0)
 ; RV64-NEXT:    sb a1, 28(a0)
-; RV64-NEXT:    sb a1, 27(a0)
-; RV64-NEXT:    sb a1, 26(a0)
-; RV64-NEXT:    sb a1, 25(a0)
+; RV64-NEXT:    sb a1, 29(a0)
+; RV64-NEXT:    sb a1, 30(a0)
+; RV64-NEXT:    sb a1, 31(a0)
 ; RV64-NEXT:    sb a1, 24(a0)
-; RV64-NEXT:    sb a1, 23(a0)
-; RV64-NEXT:    sb a1, 22(a0)
-; RV64-NEXT:    sb a1, 21(a0)
+; RV64-NEXT:    sb a1, 25(a0)
+; RV64-NEXT:    sb a1, 26(a0)
+; RV64-NEXT:    sb a1, 27(a0)
 ; RV64-NEXT:    sb a1, 20(a0)
-; RV64-NEXT:    sb a1, 19(a0)
-; RV64-NEXT:    sb a1, 18(a0)
-; RV64-NEXT:    sb a1, 17(a0)
+; RV64-NEXT:    sb a1, 21(a0)
+; RV64-NEXT:    sb a1, 22(a0)
+; RV64-NEXT:    sb a1, 23(a0)
 ; RV64-NEXT:    sb a1, 16(a0)
-; RV64-NEXT:    sb a1, 15(a0)
-; RV64-NEXT:    sb a1, 14(a0)
-; RV64-NEXT:    sb a1, 13(a0)
+; RV64-NEXT:    sb a1, 17(a0)
+; RV64-NEXT:    sb a1, 18(a0)
+; RV64-NEXT:    sb a1, 19(a0)
 ; RV64-NEXT:    sb a1, 12(a0)
-; RV64-NEXT:    sb a1, 11(a0)
-; RV64-NEXT:    sb a1, 10(a0)
-; RV64-NEXT:    sb a1, 9(a0)
+; RV64-NEXT:    sb a1, 13(a0)
+; RV64-NEXT:    sb a1, 14(a0)
+; RV64-NEXT:    sb a1, 15(a0)
 ; RV64-NEXT:    sb a1, 8(a0)
-; RV64-NEXT:    sb a1, 7(a0)
-; RV64-NEXT:    sb a1, 6(a0)
-; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 9(a0)
+; RV64-NEXT:    sb a1, 10(a0)
+; RV64-NEXT:    sb a1, 11(a0)
 ; RV64-NEXT:    sb a1, 4(a0)
-; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 6(a0)
+; RV64-NEXT:    sb a1, 7(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_64:
@@ -465,22 +465,22 @@ define void @memset_64(ptr %a, i8 %value) nounwind {
 ; RV32-FAST-NEXT:    lui a2, 4112
 ; RV32-FAST-NEXT:    addi a2, a2, 257
 ; RV32-FAST-NEXT:    mul a1, a1, a2
-; RV32-FAST-NEXT:    sw a1, 60(a0)
-; RV32-FAST-NEXT:    sw a1, 56(a0)
-; RV32-FAST-NEXT:    sw a1, 52(a0)
 ; RV32-FAST-NEXT:    sw a1, 48(a0)
-; RV32-FAST-NEXT:    sw a1, 44(a0)
-; RV32-FAST-NEXT:    sw a1, 40(a0)
-; RV32-FAST-NEXT:    sw a1, 36(a0)
+; RV32-FAST-NEXT:    sw a1, 52(a0)
+; RV32-FAST-NEXT:    sw a1, 56(a0)
+; RV32-FAST-NEXT:    sw a1, 60(a0)
 ; RV32-FAST-NEXT:    sw a1, 32(a0)
-; RV32-FAST-NEXT:    sw a1, 28(a0)
-; RV32-FAST-NEXT:    sw a1, 24(a0)
-; RV32-FAST-NEXT:    sw a1, 20(a0)
+; RV32-FAST-NEXT:    sw a1, 36(a0)
+; RV32-FAST-NEXT:    sw a1, 40(a0)
+; RV32-FAST-NEXT:    sw a1, 44(a0)
 ; RV32-FAST-NEXT:    sw a1, 16(a0)
-; RV32-FAST-NEXT:    sw a1, 12(a0)
-; RV32-FAST-NEXT:    sw a1, 8(a0)
-; RV32-FAST-NEXT:    sw a1, 4(a0)
+; RV32-FAST-NEXT:    sw a1, 20(a0)
+; RV32-FAST-NEXT:    sw a1, 24(a0)
+; RV32-FAST-NEXT:    sw a1, 28(a0)
 ; RV32-FAST-NEXT:    sw a1, 0(a0)
+; RV32-FAST-NEXT:    sw a1, 4(a0)
+; RV32-FAST-NEXT:    sw a1, 8(a0)
+; RV32-FAST-NEXT:    sw a1, 12(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: memset_64:
@@ -491,14 +491,14 @@ define void @memset_64(ptr %a, i8 %value) nounwind {
 ; RV64-FAST-NEXT:    slli a3, a2, 32
 ; RV64-FAST-NEXT:    add a2, a2, a3
 ; RV64-FAST-NEXT:    mul a1, a1, a2
-; RV64-FAST-NEXT:    sd a1, 56(a0)
-; RV64-FAST-NEXT:    sd a1, 48(a0)
-; RV64-FAST-NEXT:    sd a1, 40(a0)
 ; RV64-FAST-NEXT:    sd a1, 32(a0)
-; RV64-FAST-NEXT:    sd a1, 24(a0)
-; RV64-FAST-NEXT:    sd a1, 16(a0)
-; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    sd a1, 40(a0)
+; RV64-FAST-NEXT:    sd a1, 48(a0)
+; RV64-FAST-NEXT:    sd a1, 56(a0)
 ; RV64-FAST-NEXT:    sd a1, 0(a0)
+; RV64-FAST-NEXT:    sd a1, 8(a0)
+; RV64-FAST-NEXT:    sd a1, 16(a0)
+; RV64-FAST-NEXT:    sd a1, 24(a0)
 ; RV64-FAST-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 %value, i64 64, i1 0)
   ret void
@@ -556,8 +556,8 @@ define void @aligned_memset_8(ptr align 8 %a, i8 %value) nounwind {
 ; RV32-BOTH-NEXT:    lui a2, 4112
 ; RV32-BOTH-NEXT:    addi a2, a2, 257
 ; RV32-BOTH-NEXT:    mul a1, a1, a2
-; RV32-BOTH-NEXT:    sw a1, 4(a0)
 ; RV32-BOTH-NEXT:    sw a1, 0(a0)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_memset_8:
@@ -581,10 +581,10 @@ define void @aligned_memset_16(ptr align 16 %a, i8 %value) nounwind {
 ; RV32-BOTH-NEXT:    lui a2, 4112
 ; RV32-BOTH-NEXT:    addi a2, a2, 257
 ; RV32-BOTH-NEXT:    mul a1, a1, a2
-; RV32-BOTH-NEXT:    sw a1, 12(a0)
-; RV32-BOTH-NEXT:    sw a1, 8(a0)
-; RV32-BOTH-NEXT:    sw a1, 4(a0)
 ; RV32-BOTH-NEXT:    sw a1, 0(a0)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    sw a1, 8(a0)
+; RV32-BOTH-NEXT:    sw a1, 12(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_memset_16:
@@ -595,8 +595,8 @@ define void @aligned_memset_16(ptr align 16 %a, i8 %value) nounwind {
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
-; RV64-BOTH-NEXT:    sd a1, 8(a0)
 ; RV64-BOTH-NEXT:    sd a1, 0(a0)
+; RV64-BOTH-NEXT:    sd a1, 8(a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 16 %a, i8 %value, i64 16, i1 0)
   ret void
@@ -609,14 +609,14 @@ define void @aligned_memset_32(ptr align 32 %a, i8 %value) nounwind {
 ; RV32-BOTH-NEXT:    lui a2, 4112
 ; RV32-BOTH-NEXT:    addi a2, a2, 257
 ; RV32-BOTH-NEXT:    mul a1, a1, a2
-; RV32-BOTH-NEXT:    sw a1, 28(a0)
-; RV32-BOTH-NEXT:    sw a1, 24(a0)
-; RV32-BOTH-NEXT:    sw a1, 20(a0)
 ; RV32-BOTH-NEXT:    sw a1, 16(a0)
-; RV32-BOTH-NEXT:    sw a1, 12(a0)
-; RV32-BOTH-NEXT:    sw a1, 8(a0)
-; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    sw a1, 20(a0)
+; RV32-BOTH-NEXT:    sw a1, 24(a0)
+; RV32-BOTH-NEXT:    sw a1, 28(a0)
 ; RV32-BOTH-NEXT:    sw a1, 0(a0)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    sw a1, 8(a0)
+; RV32-BOTH-NEXT:    sw a1, 12(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_memset_32:
@@ -627,10 +627,10 @@ define void @aligned_memset_32(ptr align 32 %a, i8 %value) nounwind {
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
-; RV64-BOTH-NEXT:    sd a1, 24(a0)
-; RV64-BOTH-NEXT:    sd a1, 16(a0)
-; RV64-BOTH-NEXT:    sd a1, 8(a0)
 ; RV64-BOTH-NEXT:    sd a1, 0(a0)
+; RV64-BOTH-NEXT:    sd a1, 8(a0)
+; RV64-BOTH-NEXT:    sd a1, 16(a0)
+; RV64-BOTH-NEXT:    sd a1, 24(a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 32 %a, i8 %value, i64 32, i1 0)
   ret void
@@ -643,22 +643,22 @@ define void @aligned_memset_64(ptr align 64 %a, i8 %value) nounwind {
 ; RV32-BOTH-NEXT:    lui a2, 4112
 ; RV32-BOTH-NEXT:    addi a2, a2, 257
 ; RV32-BOTH-NEXT:    mul a1, a1, a2
-; RV32-BOTH-NEXT:    sw a1, 60(a0)
-; RV32-BOTH-NEXT:    sw a1, 56(a0)
-; RV32-BOTH-NEXT:    sw a1, 52(a0)
 ; RV32-BOTH-NEXT:    sw a1, 48(a0)
-; RV32-BOTH-NEXT:    sw a1, 44(a0)
-; RV32-BOTH-NEXT:    sw a1, 40(a0)
-; RV32-BOTH-NEXT:    sw a1, 36(a0)
+; RV32-BOTH-NEXT:    sw a1, 52(a0)
+; RV32-BOTH-NEXT:    sw a1, 56(a0)
+; RV32-BOTH-NEXT:    sw a1, 60(a0)
 ; RV32-BOTH-NEXT:    sw a1, 32(a0)
-; RV32-BOTH-NEXT:    sw a1, 28(a0)
-; RV32-BOTH-NEXT:    sw a1, 24(a0)
-; RV32-BOTH-NEXT:    sw a1, 20(a0)
+; RV32-BOTH-NEXT:    sw a1, 36(a0)
+; RV32-BOTH-NEXT:    sw a1, 40(a0)
+; RV32-BOTH-NEXT:    sw a1, 44(a0)
 ; RV32-BOTH-NEXT:    sw a1, 16(a0)
-; RV32-BOTH-NEXT:    sw a1, 12(a0)
-; RV32-BOTH-NEXT:    sw a1, 8(a0)
-; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    sw a1, 20(a0)
+; RV32-BOTH-NEXT:    sw a1, 24(a0)
+; RV32-BOTH-NEXT:    sw a1, 28(a0)
 ; RV32-BOTH-NEXT:    sw a1, 0(a0)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
+; RV32-BOTH-NEXT:    sw a1, 8(a0)
+; RV32-BOTH-NEXT:    sw a1, 12(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_memset_64:
@@ -669,14 +669,14 @@ define void @aligned_memset_64(ptr align 64 %a, i8 %value) nounwind {
 ; RV64-BOTH-NEXT:    slli a3, a2, 32
 ; RV64-BOTH-NEXT:    add a2, a2, a3
 ; RV64-BOTH-NEXT:    mul a1, a1, a2
-; RV64-BOTH-NEXT:    sd a1, 56(a0)
-; RV64-BOTH-NEXT:    sd a1, 48(a0)
-; RV64-BOTH-NEXT:    sd a1, 40(a0)
 ; RV64-BOTH-NEXT:    sd a1, 32(a0)
-; RV64-BOTH-NEXT:    sd a1, 24(a0)
-; RV64-BOTH-NEXT:    sd a1, 16(a0)
-; RV64-BOTH-NEXT:    sd a1, 8(a0)
+; RV64-BOTH-NEXT:    sd a1, 40(a0)
+; RV64-BOTH-NEXT:    sd a1, 48(a0)
+; RV64-BOTH-NEXT:    sd a1, 56(a0)
 ; RV64-BOTH-NEXT:    sd a1, 0(a0)
+; RV64-BOTH-NEXT:    sd a1, 8(a0)
+; RV64-BOTH-NEXT:    sd a1, 16(a0)
+; RV64-BOTH-NEXT:    sd a1, 24(a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 64 %a, i8 %value, i64 64, i1 0)
   ret void
@@ -701,14 +701,14 @@ define void @bzero_1(ptr %a) nounwind {
 define void @bzero_2(ptr %a) nounwind {
 ; RV32-LABEL: bzero_2:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 1(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_2:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 1(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_2:
@@ -727,18 +727,18 @@ define void @bzero_2(ptr %a) nounwind {
 define void @bzero_4(ptr %a) nounwind {
 ; RV32-LABEL: bzero_4:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 3(a0)
-; RV32-NEXT:    sb zero, 2(a0)
-; RV32-NEXT:    sb zero, 1(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 2(a0)
+; RV32-NEXT:    sb zero, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_4:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 3(a0)
-; RV64-NEXT:    sb zero, 2(a0)
-; RV64-NEXT:    sb zero, 1(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 2(a0)
+; RV64-NEXT:    sb zero, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_4:
@@ -757,32 +757,32 @@ define void @bzero_4(ptr %a) nounwind {
 define void @bzero_8(ptr %a) nounwind {
 ; RV32-LABEL: bzero_8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 7(a0)
-; RV32-NEXT:    sb zero, 6(a0)
-; RV32-NEXT:    sb zero, 5(a0)
 ; RV32-NEXT:    sb zero, 4(a0)
-; RV32-NEXT:    sb zero, 3(a0)
-; RV32-NEXT:    sb zero, 2(a0)
-; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 6(a0)
+; RV32-NEXT:    sb zero, 7(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 2(a0)
+; RV32-NEXT:    sb zero, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_8:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 7(a0)
-; RV64-NEXT:    sb zero, 6(a0)
-; RV64-NEXT:    sb zero, 5(a0)
 ; RV64-NEXT:    sb zero, 4(a0)
-; RV64-NEXT:    sb zero, 3(a0)
-; RV64-NEXT:    sb zero, 2(a0)
-; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 6(a0)
+; RV64-NEXT:    sb zero, 7(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 2(a0)
+; RV64-NEXT:    sb zero, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_8:
 ; RV32-FAST:       # %bb.0:
-; RV32-FAST-NEXT:    sw zero, 4(a0)
 ; RV32-FAST-NEXT:    sw zero, 0(a0)
+; RV32-FAST-NEXT:    sw zero, 4(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: bzero_8:
@@ -796,56 +796,56 @@ define void @bzero_8(ptr %a) nounwind {
 define void @bzero_16(ptr %a) nounwind {
 ; RV32-LABEL: bzero_16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 15(a0)
-; RV32-NEXT:    sb zero, 14(a0)
-; RV32-NEXT:    sb zero, 13(a0)
 ; RV32-NEXT:    sb zero, 12(a0)
-; RV32-NEXT:    sb zero, 11(a0)
-; RV32-NEXT:    sb zero, 10(a0)
-; RV32-NEXT:    sb zero, 9(a0)
+; RV32-NEXT:    sb zero, 13(a0)
+; RV32-NEXT:    sb zero, 14(a0)
+; RV32-NEXT:    sb zero, 15(a0)
 ; RV32-NEXT:    sb zero, 8(a0)
-; RV32-NEXT:    sb zero, 7(a0)
-; RV32-NEXT:    sb zero, 6(a0)
-; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 9(a0)
+; RV32-NEXT:    sb zero, 10(a0)
+; RV32-NEXT:    sb zero, 11(a0)
 ; RV32-NEXT:    sb zero, 4(a0)
-; RV32-NEXT:    sb zero, 3(a0)
-; RV32-NEXT:    sb zero, 2(a0)
-; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 6(a0)
+; RV32-NEXT:    sb zero, 7(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 2(a0)
+; RV32-NEXT:    sb zero, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_16:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 15(a0)
-; RV64-NEXT:    sb zero, 14(a0)
-; RV64-NEXT:    sb zero, 13(a0)
 ; RV64-NEXT:    sb zero, 12(a0)
-; RV64-NEXT:    sb zero, 11(a0)
-; RV64-NEXT:    sb zero, 10(a0)
-; RV64-NEXT:    sb zero, 9(a0)
+; RV64-NEXT:    sb zero, 13(a0)
+; RV64-NEXT:    sb zero, 14(a0)
+; RV64-NEXT:    sb zero, 15(a0)
 ; RV64-NEXT:    sb zero, 8(a0)
-; RV64-NEXT:    sb zero, 7(a0)
-; RV64-NEXT:    sb zero, 6(a0)
-; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 9(a0)
+; RV64-NEXT:    sb zero, 10(a0)
+; RV64-NEXT:    sb zero, 11(a0)
 ; RV64-NEXT:    sb zero, 4(a0)
-; RV64-NEXT:    sb zero, 3(a0)
-; RV64-NEXT:    sb zero, 2(a0)
-; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 6(a0)
+; RV64-NEXT:    sb zero, 7(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 2(a0)
+; RV64-NEXT:    sb zero, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_16:
 ; RV32-FAST:       # %bb.0:
-; RV32-FAST-NEXT:    sw zero, 12(a0)
-; RV32-FAST-NEXT:    sw zero, 8(a0)
-; RV32-FAST-NEXT:    sw zero, 4(a0)
 ; RV32-FAST-NEXT:    sw zero, 0(a0)
+; RV32-FAST-NEXT:    sw zero, 4(a0)
+; RV32-FAST-NEXT:    sw zero, 8(a0)
+; RV32-FAST-NEXT:    sw zero, 12(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: bzero_16:
 ; RV64-FAST:       # %bb.0:
-; RV64-FAST-NEXT:    sd zero, 8(a0)
 ; RV64-FAST-NEXT:    sd zero, 0(a0)
+; RV64-FAST-NEXT:    sd zero, 8(a0)
 ; RV64-FAST-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 0, i64 16, i1 0)
   ret void
@@ -854,94 +854,94 @@ define void @bzero_16(ptr %a) nounwind {
 define void @bzero_32(ptr %a) nounwind {
 ; RV32-LABEL: bzero_32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 31(a0)
-; RV32-NEXT:    sb zero, 30(a0)
-; RV32-NEXT:    sb zero, 29(a0)
 ; RV32-NEXT:    sb zero, 28(a0)
-; RV32-NEXT:    sb zero, 27(a0)
-; RV32-NEXT:    sb zero, 26(a0)
-; RV32-NEXT:    sb zero, 25(a0)
+; RV32-NEXT:    sb zero, 29(a0)
+; RV32-NEXT:    sb zero, 30(a0)
+; RV32-NEXT:    sb zero, 31(a0)
 ; RV32-NEXT:    sb zero, 24(a0)
-; RV32-NEXT:    sb zero, 23(a0)
-; RV32-NEXT:    sb zero, 22(a0)
-; RV32-NEXT:    sb zero, 21(a0)
+; RV32-NEXT:    sb zero, 25(a0)
+; RV32-NEXT:    sb zero, 26(a0)
+; RV32-NEXT:    sb zero, 27(a0)
 ; RV32-NEXT:    sb zero, 20(a0)
-; RV32-NEXT:    sb zero, 19(a0)
-; RV32-NEXT:    sb zero, 18(a0)
-; RV32-NEXT:    sb zero, 17(a0)
+; RV32-NEXT:    sb zero, 21(a0)
+; RV32-NEXT:    sb zero, 22(a0)
+; RV32-NEXT:    sb zero, 23(a0)
 ; RV32-NEXT:    sb zero, 16(a0)
-; RV32-NEXT:    sb zero, 15(a0)
-; RV32-NEXT:    sb zero, 14(a0)
-; RV32-NEXT:    sb zero, 13(a0)
+; RV32-NEXT:    sb zero, 17(a0)
+; RV32-NEXT:    sb zero, 18(a0)
+; RV32-NEXT:    sb zero, 19(a0)
 ; RV32-NEXT:    sb zero, 12(a0)
-; RV32-NEXT:    sb zero, 11(a0)
-; RV32-NEXT:    sb zero, 10(a0)
-; RV32-NEXT:    sb zero, 9(a0)
+; RV32-NEXT:    sb zero, 13(a0)
+; RV32-NEXT:    sb zero, 14(a0)
+; RV32-NEXT:    sb zero, 15(a0)
 ; RV32-NEXT:    sb zero, 8(a0)
-; RV32-NEXT:    sb zero, 7(a0)
-; RV32-NEXT:    sb zero, 6(a0)
-; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 9(a0)
+; RV32-NEXT:    sb zero, 10(a0)
+; RV32-NEXT:    sb zero, 11(a0)
 ; RV32-NEXT:    sb zero, 4(a0)
-; RV32-NEXT:    sb zero, 3(a0)
-; RV32-NEXT:    sb zero, 2(a0)
-; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 6(a0)
+; RV32-NEXT:    sb zero, 7(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 2(a0)
+; RV32-NEXT:    sb zero, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 31(a0)
-; RV64-NEXT:    sb zero, 30(a0)
-; RV64-NEXT:    sb zero, 29(a0)
 ; RV64-NEXT:    sb zero, 28(a0)
-; RV64-NEXT:    sb zero, 27(a0)
-; RV64-NEXT:    sb zero, 26(a0)
-; RV64-NEXT:    sb zero, 25(a0)
+; RV64-NEXT:    sb zero, 29(a0)
+; RV64-NEXT:    sb zero, 30(a0)
+; RV64-NEXT:    sb zero, 31(a0)
 ; RV64-NEXT:    sb zero, 24(a0)
-; RV64-NEXT:    sb zero, 23(a0)
-; RV64-NEXT:    sb zero, 22(a0)
-; RV64-NEXT:    sb zero, 21(a0)
+; RV64-NEXT:    sb zero, 25(a0)
+; RV64-NEXT:    sb zero, 26(a0)
+; RV64-NEXT:    sb zero, 27(a0)
 ; RV64-NEXT:    sb zero, 20(a0)
-; RV64-NEXT:    sb zero, 19(a0)
-; RV64-NEXT:    sb zero, 18(a0)
-; RV64-NEXT:    sb zero, 17(a0)
+; RV64-NEXT:    sb zero, 21(a0)
+; RV64-NEXT:    sb zero, 22(a0)
+; RV64-NEXT:    sb zero, 23(a0)
 ; RV64-NEXT:    sb zero, 16(a0)
-; RV64-NEXT:    sb zero, 15(a0)
-; RV64-NEXT:    sb zero, 14(a0)
-; RV64-NEXT:    sb zero, 13(a0)
+; RV64-NEXT:    sb zero, 17(a0)
+; RV64-NEXT:    sb zero, 18(a0)
+; RV64-NEXT:    sb zero, 19(a0)
 ; RV64-NEXT:    sb zero, 12(a0)
-; RV64-NEXT:    sb zero, 11(a0)
-; RV64-NEXT:    sb zero, 10(a0)
-; RV64-NEXT:    sb zero, 9(a0)
+; RV64-NEXT:    sb zero, 13(a0)
+; RV64-NEXT:    sb zero, 14(a0)
+; RV64-NEXT:    sb zero, 15(a0)
 ; RV64-NEXT:    sb zero, 8(a0)
-; RV64-NEXT:    sb zero, 7(a0)
-; RV64-NEXT:    sb zero, 6(a0)
-; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 9(a0)
+; RV64-NEXT:    sb zero, 10(a0)
+; RV64-NEXT:    sb zero, 11(a0)
 ; RV64-NEXT:    sb zero, 4(a0)
-; RV64-NEXT:    sb zero, 3(a0)
-; RV64-NEXT:    sb zero, 2(a0)
-; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 6(a0)
+; RV64-NEXT:    sb zero, 7(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 2(a0)
+; RV64-NEXT:    sb zero, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_32:
 ; RV32-FAST:       # %bb.0:
-; RV32-FAST-NEXT:    sw zero, 28(a0)
-; RV32-FAST-NEXT:    sw zero, 24(a0)
-; RV32-FAST-NEXT:    sw zero, 20(a0)
 ; RV32-FAST-NEXT:    sw zero, 16(a0)
-; RV32-FAST-NEXT:    sw zero, 12(a0)
-; RV32-FAST-NEXT:    sw zero, 8(a0)
-; RV32-FAST-NEXT:    sw zero, 4(a0)
+; RV32-FAST-NEXT:    sw zero, 20(a0)
+; RV32-FAST-NEXT:    sw zero, 24(a0)
+; RV32-FAST-NEXT:    sw zero, 28(a0)
 ; RV32-FAST-NEXT:    sw zero, 0(a0)
+; RV32-FAST-NEXT:    sw zero, 4(a0)
+; RV32-FAST-NEXT:    sw zero, 8(a0)
+; RV32-FAST-NEXT:    sw zero, 12(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: bzero_32:
 ; RV64-FAST:       # %bb.0:
-; RV64-FAST-NEXT:    sd zero, 24(a0)
-; RV64-FAST-NEXT:    sd zero, 16(a0)
-; RV64-FAST-NEXT:    sd zero, 8(a0)
 ; RV64-FAST-NEXT:    sd zero, 0(a0)
+; RV64-FAST-NEXT:    sd zero, 8(a0)
+; RV64-FAST-NEXT:    sd zero, 16(a0)
+; RV64-FAST-NEXT:    sd zero, 24(a0)
 ; RV64-FAST-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 0, i64 32, i1 0)
   ret void
@@ -950,170 +950,170 @@ define void @bzero_32(ptr %a) nounwind {
 define void @bzero_64(ptr %a) nounwind {
 ; RV32-LABEL: bzero_64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 63(a0)
-; RV32-NEXT:    sb zero, 62(a0)
-; RV32-NEXT:    sb zero, 61(a0)
 ; RV32-NEXT:    sb zero, 60(a0)
-; RV32-NEXT:    sb zero, 59(a0)
-; RV32-NEXT:    sb zero, 58(a0)
-; RV32-NEXT:    sb zero, 57(a0)
+; RV32-NEXT:    sb zero, 61(a0)
+; RV32-NEXT:    sb zero, 62(a0)
+; RV32-NEXT:    sb zero, 63(a0)
 ; RV32-NEXT:    sb zero, 56(a0)
-; RV32-NEXT:    sb zero, 55(a0)
-; RV32-NEXT:    sb zero, 54(a0)
-; RV32-NEXT:    sb zero, 53(a0)
+; RV32-NEXT:    sb zero, 57(a0)
+; RV32-NEXT:    sb zero, 58(a0)
+; RV32-NEXT:    sb zero, 59(a0)
 ; RV32-NEXT:    sb zero, 52(a0)
-; RV32-NEXT:    sb zero, 51(a0)
-; RV32-NEXT:    sb zero, 50(a0)
-; RV32-NEXT:    sb zero, 49(a0)
+; RV32-NEXT:    sb zero, 53(a0)
+; RV32-NEXT:    sb zero, 54(a0)
+; RV32-NEXT:    sb zero, 55(a0)
 ; RV32-NEXT:    sb zero, 48(a0)
-; RV32-NEXT:    sb zero, 47(a0)
-; RV32-NEXT:    sb zero, 46(a0)
-; RV32-NEXT:    sb zero, 45(a0)
+; RV32-NEXT:    sb zero, 49(a0)
+; RV32-NEXT:    sb zero, 50(a0)
+; RV32-NEXT:    sb zero, 51(a0)
 ; RV32-NEXT:    sb zero, 44(a0)
-; RV32-NEXT:    sb zero, 43(a0)
-; RV32-NEXT:    sb zero, 42(a0)
-; RV32-NEXT:    sb zero, 41(a0)
+; RV32-NEXT:    sb zero, 45(a0)
+; RV32-NEXT:    sb zero, 46(a0)
+; RV32-NEXT:    sb zero, 47(a0)
 ; RV32-NEXT:    sb zero, 40(a0)
-; RV32-NEXT:    sb zero, 39(a0)
-; RV32-NEXT:    sb zero, 38(a0)
-; RV32-NEXT:    sb zero, 37(a0)
+; RV32-NEXT:    sb zero, 41(a0)
+; RV32-NEXT:    sb zero, 42(a0)
+; RV32-NEXT:    sb zero, 43(a0)
 ; RV32-NEXT:    sb zero, 36(a0)
-; RV32-NEXT:    sb zero, 35(a0)
-; RV32-NEXT:    sb zero, 34(a0)
-; RV32-NEXT:    sb zero, 33(a0)
+; RV32-NEXT:    sb zero, 37(a0)
+; RV32-NEXT:    sb zero, 38(a0)
+; RV32-NEXT:    sb zero, 39(a0)
 ; RV32-NEXT:    sb zero, 32(a0)
-; RV32-NEXT:    sb zero, 31(a0)
-; RV32-NEXT:    sb zero, 30(a0)
-; RV32-NEXT:    sb zero, 29(a0)
+; RV32-NEXT:    sb zero, 33(a0)
+; RV32-NEXT:    sb zero, 34(a0)
+; RV32-NEXT:    sb zero, 35(a0)
 ; RV32-NEXT:    sb zero, 28(a0)
-; RV32-NEXT:    sb zero, 27(a0)
-; RV32-NEXT:    sb zero, 26(a0)
-; RV32-NEXT:    sb zero, 25(a0)
+; RV32-NEXT:    sb zero, 29(a0)
+; RV32-NEXT:    sb zero, 30(a0)
+; RV32-NEXT:    sb zero, 31(a0)
 ; RV32-NEXT:    sb zero, 24(a0)
-; RV32-NEXT:    sb zero, 23(a0)
-; RV32-NEXT:    sb zero, 22(a0)
-; RV32-NEXT:    sb zero, 21(a0)
+; RV32-NEXT:    sb zero, 25(a0)
+; RV32-NEXT:    sb zero, 26(a0)
+; RV32-NEXT:    sb zero, 27(a0)
 ; RV32-NEXT:    sb zero, 20(a0)
-; RV32-NEXT:    sb zero, 19(a0)
-; RV32-NEXT:    sb zero, 18(a0)
-; RV32-NEXT:    sb zero, 17(a0)
+; RV32-NEXT:    sb zero, 21(a0)
+; RV32-NEXT:    sb zero, 22(a0)
+; RV32-NEXT:    sb zero, 23(a0)
 ; RV32-NEXT:    sb zero, 16(a0)
-; RV32-NEXT:    sb zero, 15(a0)
-; RV32-NEXT:    sb zero, 14(a0)
-; RV32-NEXT:    sb zero, 13(a0)
+; RV32-NEXT:    sb zero, 17(a0)
+; RV32-NEXT:    sb zero, 18(a0)
+; RV32-NEXT:    sb zero, 19(a0)
 ; RV32-NEXT:    sb zero, 12(a0)
-; RV32-NEXT:    sb zero, 11(a0)
-; RV32-NEXT:    sb zero, 10(a0)
-; RV32-NEXT:    sb zero, 9(a0)
+; RV32-NEXT:    sb zero, 13(a0)
+; RV32-NEXT:    sb zero, 14(a0)
+; RV32-NEXT:    sb zero, 15(a0)
 ; RV32-NEXT:    sb zero, 8(a0)
-; RV32-NEXT:    sb zero, 7(a0)
-; RV32-NEXT:    sb zero, 6(a0)
-; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 9(a0)
+; RV32-NEXT:    sb zero, 10(a0)
+; RV32-NEXT:    sb zero, 11(a0)
 ; RV32-NEXT:    sb zero, 4(a0)
-; RV32-NEXT:    sb zero, 3(a0)
-; RV32-NEXT:    sb zero, 2(a0)
-; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 6(a0)
+; RV32-NEXT:    sb zero, 7(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 2(a0)
+; RV32-NEXT:    sb zero, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 63(a0)
-; RV64-NEXT:    sb zero, 62(a0)
-; RV64-NEXT:    sb zero, 61(a0)
 ; RV64-NEXT:    sb zero, 60(a0)
-; RV64-NEXT:    sb zero, 59(a0)
-; RV64-NEXT:    sb zero, 58(a0)
-; RV64-NEXT:    sb zero, 57(a0)
+; RV64-NEXT:    sb zero, 61(a0)
+; RV64-NEXT:    sb zero, 62(a0)
+; RV64-NEXT:    sb zero, 63(a0)
 ; RV64-NEXT:    sb zero, 56(a0)
-; RV64-NEXT:    sb zero, 55(a0)
-; RV64-NEXT:    sb zero, 54(a0)
-; RV64-NEXT:    sb zero, 53(a0)
+; RV64-NEXT:    sb zero, 57(a0)
+; RV64-NEXT:    sb zero, 58(a0)
+; RV64-NEXT:    sb zero, 59(a0)
 ; RV64-NEXT:    sb zero, 52(a0)
-; RV64-NEXT:    sb zero, 51(a0)
-; RV64-NEXT:    sb zero, 50(a0)
-; RV64-NEXT:    sb zero, 49(a0)
+; RV64-NEXT:    sb zero, 53(a0)
+; RV64-NEXT:    sb zero, 54(a0)
+; RV64-NEXT:    sb zero, 55(a0)
 ; RV64-NEXT:    sb zero, 48(a0)
-; RV64-NEXT:    sb zero, 47(a0)
-; RV64-NEXT:    sb zero, 46(a0)
-; RV64-NEXT:    sb zero, 45(a0)
+; RV64-NEXT:    sb zero, 49(a0)
+; RV64-NEXT:    sb zero, 50(a0)
+; RV64-NEXT:    sb zero, 51(a0)
 ; RV64-NEXT:    sb zero, 44(a0)
-; RV64-NEXT:    sb zero, 43(a0)
-; RV64-NEXT:    sb zero, 42(a0)
-; RV64-NEXT:    sb zero, 41(a0)
+; RV64-NEXT:    sb zero, 45(a0)
+; RV64-NEXT:    sb zero, 46(a0)
+; RV64-NEXT:    sb zero, 47(a0)
 ; RV64-NEXT:    sb zero, 40(a0)
-; RV64-NEXT:    sb zero, 39(a0)
-; RV64-NEXT:    sb zero, 38(a0)
-; RV64-NEXT:    sb zero, 37(a0)
+; RV64-NEXT:    sb zero, 41(a0)
+; RV64-NEXT:    sb zero, 42(a0)
+; RV64-NEXT:    sb zero, 43(a0)
 ; RV64-NEXT:    sb zero, 36(a0)
-; RV64-NEXT:    sb zero, 35(a0)
-; RV64-NEXT:    sb zero, 34(a0)
-; RV64-NEXT:    sb zero, 33(a0)
+; RV64-NEXT:    sb zero, 37(a0)
+; RV64-NEXT:    sb zero, 38(a0)
+; RV64-NEXT:    sb zero, 39(a0)
 ; RV64-NEXT:    sb zero, 32(a0)
-; RV64-NEXT:    sb zero, 31(a0)
-; RV64-NEXT:    sb zero, 30(a0)
-; RV64-NEXT:    sb zero, 29(a0)
+; RV64-NEXT:    sb zero, 33(a0)
+; RV64-NEXT:    sb zero, 34(a0)
+; RV64-NEXT:    sb zero, 35(a0)
 ; RV64-NEXT:    sb zero, 28(a0)
-; RV64-NEXT:    sb zero, 27(a0)
-; RV64-NEXT:    sb zero, 26(a0)
-; RV64-NEXT:    sb zero, 25(a0)
+; RV64-NEXT:    sb zero, 29(a0)
+; RV64-NEXT:    sb zero, 30(a0)
+; RV64-NEXT:    sb zero, 31(a0)
 ; RV64-NEXT:    sb zero, 24(a0)
-; RV64-NEXT:    sb zero, 23(a0)
-; RV64-NEXT:    sb zero, 22(a0)
-; RV64-NEXT:    sb zero, 21(a0)
+; RV64-NEXT:    sb zero, 25(a0)
+; RV64-NEXT:    sb zero, 26(a0)
+; RV64-NEXT:    sb zero, 27(a0)
 ; RV64-NEXT:    sb zero, 20(a0)
-; RV64-NEXT:    sb zero, 19(a0)
-; RV64-NEXT:    sb zero, 18(a0)
-; RV64-NEXT:    sb zero, 17(a0)
+; RV64-NEXT:    sb zero, 21(a0)
+; RV64-NEXT:    sb zero, 22(a0)
+; RV64-NEXT:    sb zero, 23(a0)
 ; RV64-NEXT:    sb zero, 16(a0)
-; RV64-NEXT:    sb zero, 15(a0)
-; RV64-NEXT:    sb zero, 14(a0)
-; RV64-NEXT:    sb zero, 13(a0)
+; RV64-NEXT:    sb zero, 17(a0)
+; RV64-NEXT:    sb zero, 18(a0)
+; RV64-NEXT:    sb zero, 19(a0)
 ; RV64-NEXT:    sb zero, 12(a0)
-; RV64-NEXT:    sb zero, 11(a0)
-; RV64-NEXT:    sb zero, 10(a0)
-; RV64-NEXT:    sb zero, 9(a0)
+; RV64-NEXT:    sb zero, 13(a0)
+; RV64-NEXT:    sb zero, 14(a0)
+; RV64-NEXT:    sb zero, 15(a0)
 ; RV64-NEXT:    sb zero, 8(a0)
-; RV64-NEXT:    sb zero, 7(a0)
-; RV64-NEXT:    sb zero, 6(a0)
-; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 9(a0)
+; RV64-NEXT:    sb zero, 10(a0)
+; RV64-NEXT:    sb zero, 11(a0)
 ; RV64-NEXT:    sb zero, 4(a0)
-; RV64-NEXT:    sb zero, 3(a0)
-; RV64-NEXT:    sb zero, 2(a0)
-; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 6(a0)
+; RV64-NEXT:    sb zero, 7(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 2(a0)
+; RV64-NEXT:    sb zero, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_64:
 ; RV32-FAST:       # %bb.0:
-; RV32-FAST-NEXT:    sw zero, 60(a0)
-; RV32-FAST-NEXT:    sw zero, 56(a0)
-; RV32-FAST-NEXT:    sw zero, 52(a0)
 ; RV32-FAST-NEXT:    sw zero, 48(a0)
-; RV32-FAST-NEXT:    sw zero, 44(a0)
-; RV32-FAST-NEXT:    sw zero, 40(a0)
-; RV32-FAST-NEXT:    sw zero, 36(a0)
+; RV32-FAST-NEXT:    sw zero, 52(a0)
+; RV32-FAST-NEXT:    sw zero, 56(a0)
+; RV32-FAST-NEXT:    sw zero, 60(a0)
 ; RV32-FAST-NEXT:    sw zero, 32(a0)
-; RV32-FAST-NEXT:    sw zero, 28(a0)
-; RV32-FAST-NEXT:    sw zero, 24(a0)
-; RV32-FAST-NEXT:    sw zero, 20(a0)
+; RV32-FAST-NEXT:    sw zero, 36(a0)
+; RV32-FAST-NEXT:    sw zero, 40(a0)
+; RV32-FAST-NEXT:    sw zero, 44(a0)
 ; RV32-FAST-NEXT:    sw zero, 16(a0)
-; RV32-FAST-NEXT:    sw zero, 12(a0)
-; RV32-FAST-NEXT:    sw zero, 8(a0)
-; RV32-FAST-NEXT:    sw zero, 4(a0)
+; RV32-FAST-NEXT:    sw zero, 20(a0)
+; RV32-FAST-NEXT:    sw zero, 24(a0)
+; RV32-FAST-NEXT:    sw zero, 28(a0)
 ; RV32-FAST-NEXT:    sw zero, 0(a0)
+; RV32-FAST-NEXT:    sw zero, 4(a0)
+; RV32-FAST-NEXT:    sw zero, 8(a0)
+; RV32-FAST-NEXT:    sw zero, 12(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: bzero_64:
 ; RV64-FAST:       # %bb.0:
-; RV64-FAST-NEXT:    sd zero, 56(a0)
-; RV64-FAST-NEXT:    sd zero, 48(a0)
-; RV64-FAST-NEXT:    sd zero, 40(a0)
 ; RV64-FAST-NEXT:    sd zero, 32(a0)
-; RV64-FAST-NEXT:    sd zero, 24(a0)
-; RV64-FAST-NEXT:    sd zero, 16(a0)
-; RV64-FAST-NEXT:    sd zero, 8(a0)
+; RV64-FAST-NEXT:    sd zero, 40(a0)
+; RV64-FAST-NEXT:    sd zero, 48(a0)
+; RV64-FAST-NEXT:    sd zero, 56(a0)
 ; RV64-FAST-NEXT:    sd zero, 0(a0)
+; RV64-FAST-NEXT:    sd zero, 8(a0)
+; RV64-FAST-NEXT:    sd zero, 16(a0)
+; RV64-FAST-NEXT:    sd zero, 24(a0)
 ; RV64-FAST-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 0, i64 64, i1 0)
   ret void
@@ -1152,8 +1152,8 @@ define void @aligned_bzero_4(ptr %a) nounwind {
 define void @aligned_bzero_8(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_8:
 ; RV32-BOTH:       # %bb.0:
-; RV32-BOTH-NEXT:    sw zero, 4(a0)
 ; RV32-BOTH-NEXT:    sw zero, 0(a0)
+; RV32-BOTH-NEXT:    sw zero, 4(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_bzero_8:
@@ -1168,16 +1168,16 @@ define void @aligned_bzero_8(ptr %a) nounwind {
 define void @aligned_bzero_16(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_16:
 ; RV32-BOTH:       # %bb.0:
-; RV32-BOTH-NEXT:    sw zero, 12(a0)
-; RV32-BOTH-NEXT:    sw zero, 8(a0)
-; RV32-BOTH-NEXT:    sw zero, 4(a0)
 ; RV32-BOTH-NEXT:    sw zero, 0(a0)
+; RV32-BOTH-NEXT:    sw zero, 4(a0)
+; RV32-BOTH-NEXT:    sw zero, 8(a0)
+; RV32-BOTH-NEXT:    sw zero, 12(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_bzero_16:
 ; RV64-BOTH:       # %bb.0:
-; RV64-BOTH-NEXT:    sd zero, 8(a0)
 ; RV64-BOTH-NEXT:    sd zero, 0(a0)
+; RV64-BOTH-NEXT:    sd zero, 8(a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 16 %a, i8 0, i64 16, i1 0)
   ret void
@@ -1186,22 +1186,22 @@ define void @aligned_bzero_16(ptr %a) nounwind {
 define void @aligned_bzero_32(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_32:
 ; RV32-BOTH:       # %bb.0:
-; RV32-BOTH-NEXT:    sw zero, 28(a0)
-; RV32-BOTH-NEXT:    sw zero, 24(a0)
-; RV32-BOTH-NEXT:    sw zero, 20(a0)
 ; RV32-BOTH-NEXT:    sw zero, 16(a0)
-; RV32-BOTH-NEXT:    sw zero, 12(a0)
-; RV32-BOTH-NEXT:    sw zero, 8(a0)
-; RV32-BOTH-NEXT:    sw zero, 4(a0)
+; RV32-BOTH-NEXT:    sw zero, 20(a0)
+; RV32-BOTH-NEXT:    sw zero, 24(a0)
+; RV32-BOTH-NEXT:    sw zero, 28(a0)
 ; RV32-BOTH-NEXT:    sw zero, 0(a0)
+; RV32-BOTH-NEXT:    sw zero, 4(a0)
+; RV32-BOTH-NEXT:    sw zero, 8(a0)
+; RV32-BOTH-NEXT:    sw zero, 12(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_bzero_32:
 ; RV64-BOTH:       # %bb.0:
-; RV64-BOTH-NEXT:    sd zero, 24(a0)
-; RV64-BOTH-NEXT:    sd zero, 16(a0)
-; RV64-BOTH-NEXT:    sd zero, 8(a0)
 ; RV64-BOTH-NEXT:    sd zero, 0(a0)
+; RV64-BOTH-NEXT:    sd zero, 8(a0)
+; RV64-BOTH-NEXT:    sd zero, 16(a0)
+; RV64-BOTH-NEXT:    sd zero, 24(a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 32 %a, i8 0, i64 32, i1 0)
   ret void
@@ -1210,34 +1210,34 @@ define void @aligned_bzero_32(ptr %a) nounwind {
 define void @aligned_bzero_64(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_64:
 ; RV32-BOTH:       # %bb.0:
-; RV32-BOTH-NEXT:    sw zero, 60(a0)
-; RV32-BOTH-NEXT:    sw zero, 56(a0)
-; RV32-BOTH-NEXT:    sw zero, 52(a0)
 ; RV32-BOTH-NEXT:    sw zero, 48(a0)
-; RV32-BOTH-NEXT:    sw zero, 44(a0)
-; RV32-BOTH-NEXT:    sw zero, 40(a0)
-; RV32-BOTH-NEXT:    sw zero, 36(a0)
+; RV32-BOTH-NEXT:    sw zero, 52(a0)
+; RV32-BOTH-NEXT:    sw zero, 56(a0)
+; RV32-BOTH-NEXT:    sw zero, 60(a0)
 ; RV32-BOTH-NEXT:    sw zero, 32(a0)
-; RV32-BOTH-NEXT:    sw zero, 28(a0)
-; RV32-BOTH-NEXT:    sw zero, 24(a0)
-; RV32-BOTH-NEXT:    sw zero, 20(a0)
+; RV32-BOTH-NEXT:    sw zero, 36(a0)
+; RV32-BOTH-NEXT:    sw zero, 40(a0)
+; RV32-BOTH-NEXT:    sw zero, 44(a0)
 ; RV32-BOTH-NEXT:    sw zero, 16(a0)
-; RV32-BOTH-NEXT:    sw zero, 12(a0)
-; RV32-BOTH-NEXT:    sw zero, 8(a0)
-; RV32-BOTH-NEXT:    sw zero, 4(a0)
+; RV32-BOTH-NEXT:    sw zero, 20(a0)
+; RV32-BOTH-NEXT:    sw zero, 24(a0)
+; RV32-BOTH-NEXT:    sw zero, 28(a0)
 ; RV32-BOTH-NEXT:    sw zero, 0(a0)
+; RV32-BOTH-NEXT:    sw zero, 4(a0)
+; RV32-BOTH-NEXT:    sw zero, 8(a0)
+; RV32-BOTH-NEXT:    sw zero, 12(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_bzero_64:
 ; RV64-BOTH:       # %bb.0:
-; RV64-BOTH-NEXT:    sd zero, 56(a0)
-; RV64-BOTH-NEXT:    sd zero, 48(a0)
-; RV64-BOTH-NEXT:    sd zero, 40(a0)
 ; RV64-BOTH-NEXT:    sd zero, 32(a0)
-; RV64-BOTH-NEXT:    sd zero, 24(a0)
-; RV64-BOTH-NEXT:    sd zero, 16(a0)
-; RV64-BOTH-NEXT:    sd zero, 8(a0)
+; RV64-BOTH-NEXT:    sd zero, 40(a0)
+; RV64-BOTH-NEXT:    sd zero, 48(a0)
+; RV64-BOTH-NEXT:    sd zero, 56(a0)
 ; RV64-BOTH-NEXT:    sd zero, 0(a0)
+; RV64-BOTH-NEXT:    sd zero, 8(a0)
+; RV64-BOTH-NEXT:    sd zero, 16(a0)
+; RV64-BOTH-NEXT:    sd zero, 24(a0)
 ; RV64-BOTH-NEXT:    ret
   tail call void @llvm.memset.inline.p0.i64(ptr align 64 %a, i8 0, i64 64, i1 0)
   ret void
@@ -1250,16 +1250,16 @@ define void @aligned_bzero_64(ptr %a) nounwind {
 define void @aligned_bzero_7(ptr %a) nounwind {
 ; RV32-LABEL: aligned_bzero_7:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 6(a0)
-; RV32-NEXT:    sh zero, 4(a0)
 ; RV32-NEXT:    sw zero, 0(a0)
+; RV32-NEXT:    sh zero, 4(a0)
+; RV32-NEXT:    sb zero, 6(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: aligned_bzero_7:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 6(a0)
-; RV64-NEXT:    sh zero, 4(a0)
 ; RV64-NEXT:    sw zero, 0(a0)
+; RV64-NEXT:    sh zero, 4(a0)
+; RV64-NEXT:    sb zero, 6(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: aligned_bzero_7:
@@ -1281,26 +1281,26 @@ define void @aligned_bzero_15(ptr %a) nounwind {
 ; RV32-LABEL: aligned_bzero_15:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    sb zero, 14(a0)
-; RV32-NEXT:    sh zero, 12(a0)
-; RV32-NEXT:    sw zero, 8(a0)
-; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    sw zero, 0(a0)
+; RV32-NEXT:    sw zero, 4(a0)
+; RV32-NEXT:    sw zero, 8(a0)
+; RV32-NEXT:    sh zero, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: aligned_bzero_15:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 14(a0)
-; RV64-NEXT:    sh zero, 12(a0)
-; RV64-NEXT:    sw zero, 8(a0)
 ; RV64-NEXT:    sd zero, 0(a0)
+; RV64-NEXT:    sw zero, 8(a0)
+; RV64-NEXT:    sh zero, 12(a0)
+; RV64-NEXT:    sb zero, 14(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: aligned_bzero_15:
 ; RV32-FAST:       # %bb.0:
 ; RV32-FAST-NEXT:    sw zero, 11(a0)
-; RV32-FAST-NEXT:    sw zero, 8(a0)
-; RV32-FAST-NEXT:    sw zero, 4(a0)
 ; RV32-FAST-NEXT:    sw zero, 0(a0)
+; RV32-FAST-NEXT:    sw zero, 4(a0)
+; RV32-FAST-NEXT:    sw zero, 8(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: aligned_bzero_15:
diff --git a/llvm/test/CodeGen/RISCV/misched-load-clustering.ll b/llvm/test/CodeGen/RISCV/misched-load-clustering.ll
index cf290a0b8682dade7fc9ae43c08431f11ef9f0ef..160f0aefa36a7b82162c38e3ef2480a4160a418a 100644
--- a/llvm/test/CodeGen/RISCV/misched-load-clustering.ll
+++ b/llvm/test/CodeGen/RISCV/misched-load-clustering.ll
@@ -1,8 +1,8 @@
 ; REQUIRES: asserts
-; RUN: llc -mtriple=riscv32 -verify-misched -riscv-misched-load-clustering=false \
+; RUN: llc -mtriple=riscv32 -verify-misched -riscv-misched-load-store-clustering=false \
 ; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
 ; RUN:   | FileCheck -check-prefix=NOCLUSTER %s
-; RUN: llc -mtriple=riscv64 -verify-misched -riscv-misched-load-clustering=false \
+; RUN: llc -mtriple=riscv64 -verify-misched -riscv-misched-load-store-clustering=false \
 ; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
 ; RUN:   | FileCheck -check-prefix=NOCLUSTER %s
 ; RUN: llc -mtriple=riscv32 -verify-misched \
diff --git a/llvm/test/CodeGen/RISCV/neg-abs.ll b/llvm/test/CodeGen/RISCV/neg-abs.ll
index 6f301882b452c0ff490360b421ca2668484cddf3..676b41344611636ceedfc94e62aca8c7416d9e34 100644
--- a/llvm/test/CodeGen/RISCV/neg-abs.ll
+++ b/llvm/test/CodeGen/RISCV/neg-abs.ll
@@ -208,12 +208,13 @@ define i64 @neg_abs64_multiuse(i64 %x, ptr %y) {
 ; RV32I-NEXT:    sub a1, a1, a3
 ; RV32I-NEXT:    neg a0, a0
 ; RV32I-NEXT:  .LBB5_2:
-; RV32I-NEXT:    sw a0, 0(a2)
 ; RV32I-NEXT:    snez a3, a0
 ; RV32I-NEXT:    neg a4, a1
 ; RV32I-NEXT:    sub a3, a4, a3
-; RV32I-NEXT:    neg a0, a0
+; RV32I-NEXT:    neg a4, a0
+; RV32I-NEXT:    sw a0, 0(a2)
 ; RV32I-NEXT:    sw a1, 4(a2)
+; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a3
 ; RV32I-NEXT:    ret
 ;
@@ -226,12 +227,13 @@ define i64 @neg_abs64_multiuse(i64 %x, ptr %y) {
 ; RV32ZBB-NEXT:    sub a1, a1, a3
 ; RV32ZBB-NEXT:    neg a0, a0
 ; RV32ZBB-NEXT:  .LBB5_2:
-; RV32ZBB-NEXT:    sw a0, 0(a2)
 ; RV32ZBB-NEXT:    snez a3, a0
 ; RV32ZBB-NEXT:    neg a4, a1
 ; RV32ZBB-NEXT:    sub a3, a4, a3
-; RV32ZBB-NEXT:    neg a0, a0
+; RV32ZBB-NEXT:    neg a4, a0
+; RV32ZBB-NEXT:    sw a0, 0(a2)
 ; RV32ZBB-NEXT:    sw a1, 4(a2)
+; RV32ZBB-NEXT:    mv a0, a4
 ; RV32ZBB-NEXT:    mv a1, a3
 ; RV32ZBB-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/nontemporal.ll b/llvm/test/CodeGen/RISCV/nontemporal.ll
index 55bd32e4857345b14b2f33a736546f0605597029..ada0758831a52920719335de65c8c78d00eb6c22 100644
--- a/llvm/test/CodeGen/RISCV/nontemporal.ll
+++ b/llvm/test/CodeGen/RISCV/nontemporal.ll
@@ -327,53 +327,53 @@ define <16 x i8> @test_nontemporal_load_v16i8(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_load_v16i8:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_load_v16i8:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_load_v16i8:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_load_v16i8:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_load_v16i8:
@@ -398,53 +398,53 @@ define <8 x i16> @test_nontemporal_load_v8i16(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_load_v8i16:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_load_v8i16:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_load_v8i16:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_load_v8i16:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_load_v8i16:
@@ -469,53 +469,53 @@ define <4 x i32> @test_nontemporal_load_v4i32(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_load_v4i32:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_load_v4i32:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_load_v4i32:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_load_v4i32:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_load_v4i32:
@@ -549,17 +549,17 @@ define <2 x i64> @test_nontemporal_load_v2i64(ptr %p) {
 ; CHECK-RV32-LABEL: test_nontemporal_load_v2i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_load_v2i64:
@@ -574,17 +574,17 @@ define <2 x i64> @test_nontemporal_load_v2i64(ptr %p) {
 ; CHECK-RV32C-LABEL: test_nontemporal_load_v2i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_load_v2i64:
@@ -615,9 +615,9 @@ define void @test_nontemporal_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32-LABEL: test_nontemporal_store_i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a2, 4(a0)
-; CHECK-RV32-NEXT:    ntl.all
 ; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_store_i64:
@@ -629,9 +629,9 @@ define void @test_nontemporal_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32C-LABEL: test_nontemporal_store_i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a2, 4(a0)
-; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_store_i64:
@@ -643,9 +643,9 @@ define void @test_nontemporal_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32V-LABEL: test_nontemporal_store_i64:
 ; CHECK-RV32V:       # %bb.0:
 ; CHECK-RV32V-NEXT:    ntl.all
-; CHECK-RV32V-NEXT:    sw a2, 4(a0)
-; CHECK-RV32V-NEXT:    ntl.all
 ; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32V-NEXT:    ret
 
   store i64 %v, ptr %p, !nontemporal !0
@@ -915,46 +915,46 @@ define void @test_nontemporal_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64-NEXT:    lbu a7, 40(a1)
 ; CHECK-RV64-NEXT:    lbu t0, 48(a1)
 ; CHECK-RV64-NEXT:    lbu t1, 56(a1)
-; CHECK-RV64-NEXT:    lbu t2, 96(a1)
-; CHECK-RV64-NEXT:    lbu t3, 104(a1)
-; CHECK-RV64-NEXT:    lbu t4, 112(a1)
-; CHECK-RV64-NEXT:    lbu t5, 120(a1)
-; CHECK-RV64-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64-NEXT:    lbu a1, 88(a1)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 96(a1)
+; CHECK-RV64-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t5, 15(a0)
+; CHECK-RV64-NEXT:    sb t6, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t4, 14(a0)
+; CHECK-RV64-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t3, 13(a0)
+; CHECK-RV64-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t2, 12(a0)
+; CHECK-RV64-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a1, 11(a0)
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb s1, 10(a0)
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb s0, 9(a0)
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t6, 8(a0)
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t1, 7(a0)
-; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.all
 ; CHECK-RV64-NEXT:    sb a7, 5(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.all
 ; CHECK-RV64-NEXT:    sb a3, 1(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
 ; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    addi sp, sp, 16
@@ -976,46 +976,46 @@ define void @test_nontemporal_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32-NEXT:    lbu a7, 20(a1)
 ; CHECK-RV32-NEXT:    lbu t0, 24(a1)
 ; CHECK-RV32-NEXT:    lbu t1, 28(a1)
-; CHECK-RV32-NEXT:    lbu t2, 48(a1)
-; CHECK-RV32-NEXT:    lbu t3, 52(a1)
-; CHECK-RV32-NEXT:    lbu t4, 56(a1)
-; CHECK-RV32-NEXT:    lbu t5, 60(a1)
-; CHECK-RV32-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32-NEXT:    lbu a1, 44(a1)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 48(a1)
+; CHECK-RV32-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t5, 15(a0)
+; CHECK-RV32-NEXT:    sb t6, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t4, 14(a0)
+; CHECK-RV32-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t3, 13(a0)
+; CHECK-RV32-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t2, 12(a0)
+; CHECK-RV32-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a1, 11(a0)
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb s1, 10(a0)
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb s0, 9(a0)
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t6, 8(a0)
-; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t1, 7(a0)
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
 ; CHECK-RV32-NEXT:    sb a7, 5(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
 ; CHECK-RV32-NEXT:    sb a3, 1(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
 ; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
@@ -1037,46 +1037,46 @@ define void @test_nontemporal_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
 ; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
 ; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
-; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
-; CHECK-RV64C-NEXT:    lbu a3, 104(a1)
-; CHECK-RV64C-NEXT:    lbu a4, 112(a1)
-; CHECK-RV64C-NEXT:    lbu a5, 120(a1)
 ; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64C-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64C-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64C-NEXT:    lbu a1, 88(a1)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a5, 15(a0)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a4, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a3, 13(a0)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a1, 11(a0)
+; CHECK-RV64C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb s1, 10(a0)
+; CHECK-RV64C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb s0, 9(a0)
+; CHECK-RV64C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    addi sp, sp, 16
@@ -1098,46 +1098,46 @@ define void @test_nontemporal_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
 ; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
 ; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
-; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
-; CHECK-RV32C-NEXT:    lbu a3, 52(a1)
-; CHECK-RV32C-NEXT:    lbu a4, 56(a1)
-; CHECK-RV32C-NEXT:    lbu a5, 60(a1)
 ; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32C-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32C-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32C-NEXT:    lbu a1, 44(a1)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a5, 15(a0)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a4, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a3, 13(a0)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a1, 11(a0)
+; CHECK-RV32C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb s1, 10(a0)
+; CHECK-RV32C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb s0, 9(a0)
+; CHECK-RV32C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    addi sp, sp, 16
@@ -1163,114 +1163,114 @@ define void @test_nontemporal_store_v16i8(ptr %p, <16 x i8> %v) {
 define void @test_nontemporal_store_v8i16(ptr %p, <8 x i16> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_store_v8i16:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lh a2, 32(a1)
-; CHECK-RV64-NEXT:    lh a3, 40(a1)
-; CHECK-RV64-NEXT:    lh a4, 48(a1)
-; CHECK-RV64-NEXT:    lh a5, 56(a1)
-; CHECK-RV64-NEXT:    lh a6, 0(a1)
-; CHECK-RV64-NEXT:    lh a7, 8(a1)
-; CHECK-RV64-NEXT:    lh t0, 16(a1)
-; CHECK-RV64-NEXT:    lh a1, 24(a1)
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 32(a1)
+; CHECK-RV64-NEXT:    lh a7, 40(a1)
+; CHECK-RV64-NEXT:    lh t0, 48(a1)
+; CHECK-RV64-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a5, 14(a0)
+; CHECK-RV64-NEXT:    sh a6, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a4, 12(a0)
+; CHECK-RV64-NEXT:    sh a7, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a3, 10(a0)
+; CHECK-RV64-NEXT:    sh t0, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a2, 8(a0)
+; CHECK-RV64-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a1, 6(a0)
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh t0, 4(a0)
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a7, 2(a0)
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a6, 0(a0)
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_store_v8i16:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lh a2, 16(a1)
-; CHECK-RV32-NEXT:    lh a3, 20(a1)
-; CHECK-RV32-NEXT:    lh a4, 24(a1)
-; CHECK-RV32-NEXT:    lh a5, 28(a1)
-; CHECK-RV32-NEXT:    lh a6, 0(a1)
-; CHECK-RV32-NEXT:    lh a7, 4(a1)
-; CHECK-RV32-NEXT:    lh t0, 8(a1)
-; CHECK-RV32-NEXT:    lh a1, 12(a1)
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 16(a1)
+; CHECK-RV32-NEXT:    lh a7, 20(a1)
+; CHECK-RV32-NEXT:    lh t0, 24(a1)
+; CHECK-RV32-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a5, 14(a0)
+; CHECK-RV32-NEXT:    sh a6, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a4, 12(a0)
+; CHECK-RV32-NEXT:    sh a7, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a3, 10(a0)
+; CHECK-RV32-NEXT:    sh t0, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a2, 8(a0)
+; CHECK-RV32-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a1, 6(a0)
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh t0, 4(a0)
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a7, 2(a0)
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a6, 0(a0)
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_store_v8i16:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lh a7, 32(a1)
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 32(a1)
 ; CHECK-RV64C-NEXT:    lh a3, 40(a1)
 ; CHECK-RV64C-NEXT:    lh a4, 48(a1)
-; CHECK-RV64C-NEXT:    lh a5, 56(a1)
-; CHECK-RV64C-NEXT:    lh a6, 0(a1)
-; CHECK-RV64C-NEXT:    lh t0, 8(a1)
-; CHECK-RV64C-NEXT:    lh a2, 16(a1)
-; CHECK-RV64C-NEXT:    lh a1, 24(a1)
+; CHECK-RV64C-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a5, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a7, 8(a0)
+; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a1, 6(a0)
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a2, 4(a0)
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh t0, 2(a0)
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_store_v8i16:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lh a7, 16(a1)
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 16(a1)
 ; CHECK-RV32C-NEXT:    lh a3, 20(a1)
 ; CHECK-RV32C-NEXT:    lh a4, 24(a1)
-; CHECK-RV32C-NEXT:    lh a5, 28(a1)
-; CHECK-RV32C-NEXT:    lh a6, 0(a1)
-; CHECK-RV32C-NEXT:    lh t0, 4(a1)
-; CHECK-RV32C-NEXT:    lh a2, 8(a1)
-; CHECK-RV32C-NEXT:    lh a1, 12(a1)
+; CHECK-RV32C-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a5, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a7, 8(a0)
+; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a1, 6(a0)
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a2, 4(a0)
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh t0, 2(a0)
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_store_v8i16:
@@ -1293,66 +1293,66 @@ define void @test_nontemporal_store_v8i16(ptr %p, <8 x i16> %v) {
 define void @test_nontemporal_store_v4i32(ptr %p, <4 x i32> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_store_v4i32:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lw a2, 24(a1)
-; CHECK-RV64-NEXT:    lw a3, 16(a1)
-; CHECK-RV64-NEXT:    lw a4, 8(a1)
-; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    lw a2, 0(a1)
+; CHECK-RV64-NEXT:    lw a3, 8(a1)
+; CHECK-RV64-NEXT:    lw a4, 16(a1)
+; CHECK-RV64-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_store_v4i32:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_store_v4i32:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lw a2, 24(a1)
-; CHECK-RV64C-NEXT:    lw a3, 16(a1)
-; CHECK-RV64C-NEXT:    lw a4, 8(a1)
-; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    lw a2, 0(a1)
+; CHECK-RV64C-NEXT:    lw a3, 8(a1)
+; CHECK-RV64C-NEXT:    lw a4, 16(a1)
+; CHECK-RV64C-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_store_v4i32:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_store_v4i32:
@@ -1376,49 +1376,49 @@ define void @test_nontemporal_store_v2i64(ptr %p, <2 x i64> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_store_v2i64:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    ntl.all
 ; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_store_v2i64:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_store_v2i64:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_store_v2i64:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_store_v2i64:
@@ -1752,53 +1752,53 @@ define <16 x i8> @test_nontemporal_P1_load_v16i8(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_P1_load_v16i8:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_P1_load_v16i8:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v16i8:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v16i8:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v16i8:
@@ -1822,53 +1822,53 @@ define <8 x i16> @test_nontemporal_P1_load_v8i16(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_P1_load_v8i16:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_P1_load_v8i16:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v8i16:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v8i16:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v8i16:
@@ -1892,53 +1892,53 @@ define <4 x i32> @test_nontemporal_P1_load_v4i32(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_P1_load_v4i32:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_P1_load_v4i32:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v4i32:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v4i32:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v4i32:
@@ -1971,17 +1971,17 @@ define <2 x i64> @test_nontemporal_P1_load_v2i64(ptr %p) {
 ; CHECK-RV32-LABEL: test_nontemporal_P1_load_v2i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v2i64:
@@ -1996,17 +1996,17 @@ define <2 x i64> @test_nontemporal_P1_load_v2i64(ptr %p) {
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v2i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v2i64:
@@ -2036,9 +2036,9 @@ define void @test_nontemporal_P1_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32-LABEL: test_nontemporal_P1_store_i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a2, 4(a0)
-; CHECK-RV32-NEXT:    ntl.p1
 ; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_store_i64:
@@ -2050,9 +2050,9 @@ define void @test_nontemporal_P1_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_store_i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a2, 4(a0)
-; CHECK-RV32C-NEXT:    c.ntl.p1
 ; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_store_i64:
@@ -2064,9 +2064,9 @@ define void @test_nontemporal_P1_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32V-LABEL: test_nontemporal_P1_store_i64:
 ; CHECK-RV32V:       # %bb.0:
 ; CHECK-RV32V-NEXT:    ntl.p1
-; CHECK-RV32V-NEXT:    sw a2, 4(a0)
-; CHECK-RV32V-NEXT:    ntl.p1
 ; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32V-NEXT:    ret
   store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
   ret void
@@ -2329,46 +2329,46 @@ define void @test_nontemporal_P1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64-NEXT:    lbu a7, 40(a1)
 ; CHECK-RV64-NEXT:    lbu t0, 48(a1)
 ; CHECK-RV64-NEXT:    lbu t1, 56(a1)
-; CHECK-RV64-NEXT:    lbu t2, 96(a1)
-; CHECK-RV64-NEXT:    lbu t3, 104(a1)
-; CHECK-RV64-NEXT:    lbu t4, 112(a1)
-; CHECK-RV64-NEXT:    lbu t5, 120(a1)
-; CHECK-RV64-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64-NEXT:    lbu a1, 88(a1)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 96(a1)
+; CHECK-RV64-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb t5, 15(a0)
+; CHECK-RV64-NEXT:    sb t6, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb t4, 14(a0)
+; CHECK-RV64-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb t3, 13(a0)
+; CHECK-RV64-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb t2, 12(a0)
+; CHECK-RV64-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb a1, 11(a0)
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb s1, 10(a0)
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb s0, 9(a0)
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb t6, 8(a0)
-; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb t1, 7(a0)
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
 ; CHECK-RV64-NEXT:    sb a7, 5(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
 ; CHECK-RV64-NEXT:    sb a3, 1(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
 ; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    addi sp, sp, 16
@@ -2390,46 +2390,46 @@ define void @test_nontemporal_P1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32-NEXT:    lbu a7, 20(a1)
 ; CHECK-RV32-NEXT:    lbu t0, 24(a1)
 ; CHECK-RV32-NEXT:    lbu t1, 28(a1)
-; CHECK-RV32-NEXT:    lbu t2, 48(a1)
-; CHECK-RV32-NEXT:    lbu t3, 52(a1)
-; CHECK-RV32-NEXT:    lbu t4, 56(a1)
-; CHECK-RV32-NEXT:    lbu t5, 60(a1)
-; CHECK-RV32-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32-NEXT:    lbu a1, 44(a1)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 48(a1)
+; CHECK-RV32-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb t5, 15(a0)
+; CHECK-RV32-NEXT:    sb t6, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb t4, 14(a0)
+; CHECK-RV32-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb t3, 13(a0)
+; CHECK-RV32-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb t2, 12(a0)
+; CHECK-RV32-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb a1, 11(a0)
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb s1, 10(a0)
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb s0, 9(a0)
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb t6, 8(a0)
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb t1, 7(a0)
-; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
 ; CHECK-RV32-NEXT:    sb a7, 5(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
 ; CHECK-RV32-NEXT:    sb a3, 1(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
 ; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
@@ -2451,46 +2451,46 @@ define void @test_nontemporal_P1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
 ; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
 ; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
-; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
-; CHECK-RV64C-NEXT:    lbu a3, 104(a1)
-; CHECK-RV64C-NEXT:    lbu a4, 112(a1)
-; CHECK-RV64C-NEXT:    lbu a5, 120(a1)
 ; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64C-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64C-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64C-NEXT:    lbu a1, 88(a1)
-; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb a5, 15(a0)
-; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb a4, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb a3, 13(a0)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
 ; CHECK-RV64C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb a1, 11(a0)
+; CHECK-RV64C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb s1, 10(a0)
+; CHECK-RV64C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb s0, 9(a0)
+; CHECK-RV64C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
 ; CHECK-RV64C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
 ; CHECK-RV64C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
 ; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    addi sp, sp, 16
@@ -2512,46 +2512,46 @@ define void @test_nontemporal_P1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
 ; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
 ; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
-; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
-; CHECK-RV32C-NEXT:    lbu a3, 52(a1)
-; CHECK-RV32C-NEXT:    lbu a4, 56(a1)
-; CHECK-RV32C-NEXT:    lbu a5, 60(a1)
 ; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32C-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32C-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32C-NEXT:    lbu a1, 44(a1)
-; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb a5, 15(a0)
-; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb a4, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb a3, 13(a0)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
 ; CHECK-RV32C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb a1, 11(a0)
+; CHECK-RV32C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb s1, 10(a0)
+; CHECK-RV32C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb s0, 9(a0)
+; CHECK-RV32C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
 ; CHECK-RV32C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
 ; CHECK-RV32C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
 ; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    addi sp, sp, 16
@@ -2577,114 +2577,114 @@ define void @test_nontemporal_P1_store_v16i8(ptr %p, <16 x i8> %v) {
 define void @test_nontemporal_P1_store_v8i16(ptr %p, <8 x i16> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_P1_store_v8i16:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lh a2, 32(a1)
-; CHECK-RV64-NEXT:    lh a3, 40(a1)
-; CHECK-RV64-NEXT:    lh a4, 48(a1)
-; CHECK-RV64-NEXT:    lh a5, 56(a1)
-; CHECK-RV64-NEXT:    lh a6, 0(a1)
-; CHECK-RV64-NEXT:    lh a7, 8(a1)
-; CHECK-RV64-NEXT:    lh t0, 16(a1)
-; CHECK-RV64-NEXT:    lh a1, 24(a1)
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 32(a1)
+; CHECK-RV64-NEXT:    lh a7, 40(a1)
+; CHECK-RV64-NEXT:    lh t0, 48(a1)
+; CHECK-RV64-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh a5, 14(a0)
+; CHECK-RV64-NEXT:    sh a6, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh a4, 12(a0)
+; CHECK-RV64-NEXT:    sh a7, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh a3, 10(a0)
+; CHECK-RV64-NEXT:    sh t0, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh a2, 8(a0)
+; CHECK-RV64-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh a1, 6(a0)
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh t0, 4(a0)
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh a7, 2(a0)
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sh a6, 0(a0)
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_P1_store_v8i16:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lh a2, 16(a1)
-; CHECK-RV32-NEXT:    lh a3, 20(a1)
-; CHECK-RV32-NEXT:    lh a4, 24(a1)
-; CHECK-RV32-NEXT:    lh a5, 28(a1)
-; CHECK-RV32-NEXT:    lh a6, 0(a1)
-; CHECK-RV32-NEXT:    lh a7, 4(a1)
-; CHECK-RV32-NEXT:    lh t0, 8(a1)
-; CHECK-RV32-NEXT:    lh a1, 12(a1)
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 16(a1)
+; CHECK-RV32-NEXT:    lh a7, 20(a1)
+; CHECK-RV32-NEXT:    lh t0, 24(a1)
+; CHECK-RV32-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh a5, 14(a0)
+; CHECK-RV32-NEXT:    sh a6, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh a4, 12(a0)
+; CHECK-RV32-NEXT:    sh a7, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh a3, 10(a0)
+; CHECK-RV32-NEXT:    sh t0, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh a2, 8(a0)
+; CHECK-RV32-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh a1, 6(a0)
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh t0, 4(a0)
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh a7, 2(a0)
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sh a6, 0(a0)
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_store_v8i16:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lh a7, 32(a1)
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 32(a1)
 ; CHECK-RV64C-NEXT:    lh a3, 40(a1)
 ; CHECK-RV64C-NEXT:    lh a4, 48(a1)
-; CHECK-RV64C-NEXT:    lh a5, 56(a1)
-; CHECK-RV64C-NEXT:    lh a6, 0(a1)
-; CHECK-RV64C-NEXT:    lh t0, 8(a1)
-; CHECK-RV64C-NEXT:    lh a2, 16(a1)
-; CHECK-RV64C-NEXT:    lh a1, 24(a1)
+; CHECK-RV64C-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sh a5, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
 ; CHECK-RV64C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sh a7, 8(a0)
+; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sh a1, 6(a0)
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sh a2, 4(a0)
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sh t0, 2(a0)
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_store_v8i16:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lh a7, 16(a1)
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 16(a1)
 ; CHECK-RV32C-NEXT:    lh a3, 20(a1)
 ; CHECK-RV32C-NEXT:    lh a4, 24(a1)
-; CHECK-RV32C-NEXT:    lh a5, 28(a1)
-; CHECK-RV32C-NEXT:    lh a6, 0(a1)
-; CHECK-RV32C-NEXT:    lh t0, 4(a1)
-; CHECK-RV32C-NEXT:    lh a2, 8(a1)
-; CHECK-RV32C-NEXT:    lh a1, 12(a1)
+; CHECK-RV32C-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sh a5, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
 ; CHECK-RV32C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sh a7, 8(a0)
+; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sh a1, 6(a0)
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sh a2, 4(a0)
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sh t0, 2(a0)
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_store_v8i16:
@@ -2707,66 +2707,66 @@ define void @test_nontemporal_P1_store_v8i16(ptr %p, <8 x i16> %v) {
 define void @test_nontemporal_P1_store_v4i32(ptr %p, <4 x i32> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_P1_store_v4i32:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lw a2, 24(a1)
-; CHECK-RV64-NEXT:    lw a3, 16(a1)
-; CHECK-RV64-NEXT:    lw a4, 8(a1)
-; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    lw a2, 0(a1)
+; CHECK-RV64-NEXT:    lw a3, 8(a1)
+; CHECK-RV64-NEXT:    lw a4, 16(a1)
+; CHECK-RV64-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_P1_store_v4i32:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_store_v4i32:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lw a2, 24(a1)
-; CHECK-RV64C-NEXT:    lw a3, 16(a1)
-; CHECK-RV64C-NEXT:    lw a4, 8(a1)
-; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    lw a2, 0(a1)
+; CHECK-RV64C-NEXT:    lw a3, 8(a1)
+; CHECK-RV64C-NEXT:    lw a4, 16(a1)
+; CHECK-RV64C-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_store_v4i32:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_store_v4i32:
@@ -2790,49 +2790,49 @@ define void @test_nontemporal_P1_store_v2i64(ptr %p, <2 x i64> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_P1_store_v2i64:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.p1
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    ntl.p1
 ; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_P1_store_v2i64:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.p1
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_P1_store_v2i64:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.p1
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    c.ntl.p1
 ; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_P1_store_v2i64:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.p1
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_P1_store_v2i64:
@@ -3166,53 +3166,53 @@ define <16 x i8> @test_nontemporal_PALL_load_v16i8(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_PALL_load_v16i8:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v16i8:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v16i8:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v16i8:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v16i8:
@@ -3236,53 +3236,53 @@ define <8 x i16> @test_nontemporal_PALL_load_v8i16(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_PALL_load_v8i16:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v8i16:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v8i16:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v8i16:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v8i16:
@@ -3306,53 +3306,53 @@ define <4 x i32> @test_nontemporal_PALL_load_v4i32(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_PALL_load_v4i32:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v4i32:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v4i32:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v4i32:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v4i32:
@@ -3385,17 +3385,17 @@ define <2 x i64> @test_nontemporal_PALL_load_v2i64(ptr %p) {
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v2i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v2i64:
@@ -3410,17 +3410,17 @@ define <2 x i64> @test_nontemporal_PALL_load_v2i64(ptr %p) {
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v2i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v2i64:
@@ -3450,9 +3450,9 @@ define void @test_nontemporal_PALL_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_store_i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a2, 4(a0)
-; CHECK-RV32-NEXT:    ntl.pall
 ; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_i64:
@@ -3464,9 +3464,9 @@ define void @test_nontemporal_PALL_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a2, 4(a0)
-; CHECK-RV32C-NEXT:    c.ntl.pall
 ; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_i64:
@@ -3478,9 +3478,9 @@ define void @test_nontemporal_PALL_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_i64:
 ; CHECK-RV32V:       # %bb.0:
 ; CHECK-RV32V-NEXT:    ntl.pall
-; CHECK-RV32V-NEXT:    sw a2, 4(a0)
-; CHECK-RV32V-NEXT:    ntl.pall
 ; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32V-NEXT:    ret
   store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
   ret void
@@ -3743,46 +3743,46 @@ define void @test_nontemporal_PALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64-NEXT:    lbu a7, 40(a1)
 ; CHECK-RV64-NEXT:    lbu t0, 48(a1)
 ; CHECK-RV64-NEXT:    lbu t1, 56(a1)
-; CHECK-RV64-NEXT:    lbu t2, 96(a1)
-; CHECK-RV64-NEXT:    lbu t3, 104(a1)
-; CHECK-RV64-NEXT:    lbu t4, 112(a1)
-; CHECK-RV64-NEXT:    lbu t5, 120(a1)
-; CHECK-RV64-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64-NEXT:    lbu a1, 88(a1)
-; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb t5, 15(a0)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 96(a1)
+; CHECK-RV64-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb t4, 14(a0)
+; CHECK-RV64-NEXT:    sb t6, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb t3, 13(a0)
+; CHECK-RV64-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb t2, 12(a0)
+; CHECK-RV64-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb a1, 11(a0)
+; CHECK-RV64-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb s1, 10(a0)
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb s0, 9(a0)
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb t6, 8(a0)
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb t1, 7(a0)
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
 ; CHECK-RV64-NEXT:    sb a7, 5(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
 ; CHECK-RV64-NEXT:    sb a3, 1(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
 ; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    addi sp, sp, 16
@@ -3804,46 +3804,46 @@ define void @test_nontemporal_PALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32-NEXT:    lbu a7, 20(a1)
 ; CHECK-RV32-NEXT:    lbu t0, 24(a1)
 ; CHECK-RV32-NEXT:    lbu t1, 28(a1)
-; CHECK-RV32-NEXT:    lbu t2, 48(a1)
-; CHECK-RV32-NEXT:    lbu t3, 52(a1)
-; CHECK-RV32-NEXT:    lbu t4, 56(a1)
-; CHECK-RV32-NEXT:    lbu t5, 60(a1)
-; CHECK-RV32-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32-NEXT:    lbu a1, 44(a1)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 48(a1)
+; CHECK-RV32-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb t5, 15(a0)
+; CHECK-RV32-NEXT:    sb t6, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb t4, 14(a0)
+; CHECK-RV32-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb t3, 13(a0)
+; CHECK-RV32-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb t2, 12(a0)
+; CHECK-RV32-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb a1, 11(a0)
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb s1, 10(a0)
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb s0, 9(a0)
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb t6, 8(a0)
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb t1, 7(a0)
-; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
 ; CHECK-RV32-NEXT:    sb a7, 5(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
 ; CHECK-RV32-NEXT:    sb a3, 1(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
 ; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
@@ -3865,46 +3865,46 @@ define void @test_nontemporal_PALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
 ; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
 ; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
-; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
-; CHECK-RV64C-NEXT:    lbu a3, 104(a1)
-; CHECK-RV64C-NEXT:    lbu a4, 112(a1)
-; CHECK-RV64C-NEXT:    lbu a5, 120(a1)
 ; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64C-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64C-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64C-NEXT:    lbu a1, 88(a1)
-; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb a5, 15(a0)
-; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb a4, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb a3, 13(a0)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
 ; CHECK-RV64C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb a1, 11(a0)
+; CHECK-RV64C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb s1, 10(a0)
+; CHECK-RV64C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb s0, 9(a0)
+; CHECK-RV64C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
 ; CHECK-RV64C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
 ; CHECK-RV64C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
 ; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    addi sp, sp, 16
@@ -3926,46 +3926,46 @@ define void @test_nontemporal_PALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
 ; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
 ; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
-; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
-; CHECK-RV32C-NEXT:    lbu a3, 52(a1)
-; CHECK-RV32C-NEXT:    lbu a4, 56(a1)
-; CHECK-RV32C-NEXT:    lbu a5, 60(a1)
 ; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32C-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32C-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32C-NEXT:    lbu a1, 44(a1)
-; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb a5, 15(a0)
-; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb a4, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb a3, 13(a0)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
 ; CHECK-RV32C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb a1, 11(a0)
+; CHECK-RV32C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb s1, 10(a0)
+; CHECK-RV32C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb s0, 9(a0)
+; CHECK-RV32C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
 ; CHECK-RV32C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
 ; CHECK-RV32C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
 ; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    addi sp, sp, 16
@@ -3991,114 +3991,114 @@ define void @test_nontemporal_PALL_store_v16i8(ptr %p, <16 x i8> %v) {
 define void @test_nontemporal_PALL_store_v8i16(ptr %p, <8 x i16> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_PALL_store_v8i16:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lh a2, 32(a1)
-; CHECK-RV64-NEXT:    lh a3, 40(a1)
-; CHECK-RV64-NEXT:    lh a4, 48(a1)
-; CHECK-RV64-NEXT:    lh a5, 56(a1)
-; CHECK-RV64-NEXT:    lh a6, 0(a1)
-; CHECK-RV64-NEXT:    lh a7, 8(a1)
-; CHECK-RV64-NEXT:    lh t0, 16(a1)
-; CHECK-RV64-NEXT:    lh a1, 24(a1)
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 32(a1)
+; CHECK-RV64-NEXT:    lh a7, 40(a1)
+; CHECK-RV64-NEXT:    lh t0, 48(a1)
+; CHECK-RV64-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh a5, 14(a0)
+; CHECK-RV64-NEXT:    sh a6, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh a4, 12(a0)
+; CHECK-RV64-NEXT:    sh a7, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh a3, 10(a0)
+; CHECK-RV64-NEXT:    sh t0, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh a2, 8(a0)
+; CHECK-RV64-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh a1, 6(a0)
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh t0, 4(a0)
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh a7, 2(a0)
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sh a6, 0(a0)
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_store_v8i16:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lh a2, 16(a1)
-; CHECK-RV32-NEXT:    lh a3, 20(a1)
-; CHECK-RV32-NEXT:    lh a4, 24(a1)
-; CHECK-RV32-NEXT:    lh a5, 28(a1)
-; CHECK-RV32-NEXT:    lh a6, 0(a1)
-; CHECK-RV32-NEXT:    lh a7, 4(a1)
-; CHECK-RV32-NEXT:    lh t0, 8(a1)
-; CHECK-RV32-NEXT:    lh a1, 12(a1)
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 16(a1)
+; CHECK-RV32-NEXT:    lh a7, 20(a1)
+; CHECK-RV32-NEXT:    lh t0, 24(a1)
+; CHECK-RV32-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh a5, 14(a0)
+; CHECK-RV32-NEXT:    sh a6, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh a4, 12(a0)
+; CHECK-RV32-NEXT:    sh a7, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh a3, 10(a0)
+; CHECK-RV32-NEXT:    sh t0, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh a2, 8(a0)
+; CHECK-RV32-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh a1, 6(a0)
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh t0, 4(a0)
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh a7, 2(a0)
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sh a6, 0(a0)
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_v8i16:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lh a7, 32(a1)
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 32(a1)
 ; CHECK-RV64C-NEXT:    lh a3, 40(a1)
 ; CHECK-RV64C-NEXT:    lh a4, 48(a1)
-; CHECK-RV64C-NEXT:    lh a5, 56(a1)
-; CHECK-RV64C-NEXT:    lh a6, 0(a1)
-; CHECK-RV64C-NEXT:    lh t0, 8(a1)
-; CHECK-RV64C-NEXT:    lh a2, 16(a1)
-; CHECK-RV64C-NEXT:    lh a1, 24(a1)
+; CHECK-RV64C-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sh a5, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
 ; CHECK-RV64C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sh a7, 8(a0)
+; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sh a1, 6(a0)
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sh a2, 4(a0)
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sh t0, 2(a0)
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_v8i16:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lh a7, 16(a1)
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 16(a1)
 ; CHECK-RV32C-NEXT:    lh a3, 20(a1)
 ; CHECK-RV32C-NEXT:    lh a4, 24(a1)
-; CHECK-RV32C-NEXT:    lh a5, 28(a1)
-; CHECK-RV32C-NEXT:    lh a6, 0(a1)
-; CHECK-RV32C-NEXT:    lh t0, 4(a1)
-; CHECK-RV32C-NEXT:    lh a2, 8(a1)
-; CHECK-RV32C-NEXT:    lh a1, 12(a1)
-; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sh a5, 14(a0)
+; CHECK-RV32C-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
 ; CHECK-RV32C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sh a7, 8(a0)
+; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sh a1, 6(a0)
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sh a2, 4(a0)
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sh t0, 2(a0)
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_v8i16:
@@ -4121,66 +4121,66 @@ define void @test_nontemporal_PALL_store_v8i16(ptr %p, <8 x i16> %v) {
 define void @test_nontemporal_PALL_store_v4i32(ptr %p, <4 x i32> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_PALL_store_v4i32:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lw a2, 24(a1)
-; CHECK-RV64-NEXT:    lw a3, 16(a1)
-; CHECK-RV64-NEXT:    lw a4, 8(a1)
-; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    lw a2, 0(a1)
+; CHECK-RV64-NEXT:    lw a3, 8(a1)
+; CHECK-RV64-NEXT:    lw a4, 16(a1)
+; CHECK-RV64-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_store_v4i32:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_v4i32:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lw a2, 24(a1)
-; CHECK-RV64C-NEXT:    lw a3, 16(a1)
-; CHECK-RV64C-NEXT:    lw a4, 8(a1)
-; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    lw a2, 0(a1)
+; CHECK-RV64C-NEXT:    lw a3, 8(a1)
+; CHECK-RV64C-NEXT:    lw a4, 16(a1)
+; CHECK-RV64C-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_v4i32:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_v4i32:
@@ -4204,49 +4204,49 @@ define void @test_nontemporal_PALL_store_v2i64(ptr %p, <2 x i64> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_PALL_store_v2i64:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.pall
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    ntl.pall
 ; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_PALL_store_v2i64:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.pall
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_v2i64:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.pall
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    c.ntl.pall
 ; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_v2i64:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.pall
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_v2i64:
@@ -4580,53 +4580,53 @@ define <16 x i8> @test_nontemporal_S1_load_v16i8(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_S1_load_v16i8:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_S1_load_v16i8:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v16i8:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v16i8:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v16i8:
@@ -4650,53 +4650,53 @@ define <8 x i16> @test_nontemporal_S1_load_v8i16(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_S1_load_v8i16:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_S1_load_v8i16:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v8i16:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v8i16:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v8i16:
@@ -4720,53 +4720,53 @@ define <4 x i32> @test_nontemporal_S1_load_v4i32(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_S1_load_v4i32:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_S1_load_v4i32:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v4i32:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v4i32:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v4i32:
@@ -4799,17 +4799,17 @@ define <2 x i64> @test_nontemporal_S1_load_v2i64(ptr %p) {
 ; CHECK-RV32-LABEL: test_nontemporal_S1_load_v2i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v2i64:
@@ -4824,17 +4824,17 @@ define <2 x i64> @test_nontemporal_S1_load_v2i64(ptr %p) {
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v2i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v2i64:
@@ -4864,9 +4864,9 @@ define void @test_nontemporal_S1_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32-LABEL: test_nontemporal_S1_store_i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a2, 4(a0)
-; CHECK-RV32-NEXT:    ntl.s1
 ; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_store_i64:
@@ -4878,9 +4878,9 @@ define void @test_nontemporal_S1_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_store_i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a2, 4(a0)
-; CHECK-RV32C-NEXT:    c.ntl.s1
 ; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_store_i64:
@@ -4892,9 +4892,9 @@ define void @test_nontemporal_S1_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32V-LABEL: test_nontemporal_S1_store_i64:
 ; CHECK-RV32V:       # %bb.0:
 ; CHECK-RV32V-NEXT:    ntl.s1
-; CHECK-RV32V-NEXT:    sw a2, 4(a0)
-; CHECK-RV32V-NEXT:    ntl.s1
 ; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32V-NEXT:    ret
   store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
   ret void
@@ -5157,46 +5157,46 @@ define void @test_nontemporal_S1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64-NEXT:    lbu a7, 40(a1)
 ; CHECK-RV64-NEXT:    lbu t0, 48(a1)
 ; CHECK-RV64-NEXT:    lbu t1, 56(a1)
-; CHECK-RV64-NEXT:    lbu t2, 96(a1)
-; CHECK-RV64-NEXT:    lbu t3, 104(a1)
-; CHECK-RV64-NEXT:    lbu t4, 112(a1)
-; CHECK-RV64-NEXT:    lbu t5, 120(a1)
-; CHECK-RV64-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64-NEXT:    lbu a1, 88(a1)
-; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb t5, 15(a0)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 96(a1)
+; CHECK-RV64-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb t4, 14(a0)
+; CHECK-RV64-NEXT:    sb t6, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb t3, 13(a0)
+; CHECK-RV64-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb t2, 12(a0)
+; CHECK-RV64-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb a1, 11(a0)
+; CHECK-RV64-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb s1, 10(a0)
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb s0, 9(a0)
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb t6, 8(a0)
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb t1, 7(a0)
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
 ; CHECK-RV64-NEXT:    sb a7, 5(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
 ; CHECK-RV64-NEXT:    sb a3, 1(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
 ; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    addi sp, sp, 16
@@ -5218,46 +5218,46 @@ define void @test_nontemporal_S1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32-NEXT:    lbu a7, 20(a1)
 ; CHECK-RV32-NEXT:    lbu t0, 24(a1)
 ; CHECK-RV32-NEXT:    lbu t1, 28(a1)
-; CHECK-RV32-NEXT:    lbu t2, 48(a1)
-; CHECK-RV32-NEXT:    lbu t3, 52(a1)
-; CHECK-RV32-NEXT:    lbu t4, 56(a1)
-; CHECK-RV32-NEXT:    lbu t5, 60(a1)
-; CHECK-RV32-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32-NEXT:    lbu a1, 44(a1)
-; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb t5, 15(a0)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 48(a1)
+; CHECK-RV32-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb t4, 14(a0)
+; CHECK-RV32-NEXT:    sb t6, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb t3, 13(a0)
+; CHECK-RV32-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb t2, 12(a0)
+; CHECK-RV32-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb a1, 11(a0)
+; CHECK-RV32-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb s1, 10(a0)
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb s0, 9(a0)
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb t6, 8(a0)
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb t1, 7(a0)
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
 ; CHECK-RV32-NEXT:    sb a7, 5(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
 ; CHECK-RV32-NEXT:    sb a3, 1(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
 ; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
@@ -5279,46 +5279,46 @@ define void @test_nontemporal_S1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
 ; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
 ; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
-; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
-; CHECK-RV64C-NEXT:    lbu a3, 104(a1)
-; CHECK-RV64C-NEXT:    lbu a4, 112(a1)
-; CHECK-RV64C-NEXT:    lbu a5, 120(a1)
 ; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64C-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64C-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64C-NEXT:    lbu a1, 88(a1)
-; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb a5, 15(a0)
-; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb a4, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb a3, 13(a0)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
 ; CHECK-RV64C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb a1, 11(a0)
+; CHECK-RV64C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb s1, 10(a0)
+; CHECK-RV64C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb s0, 9(a0)
+; CHECK-RV64C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
 ; CHECK-RV64C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
 ; CHECK-RV64C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
 ; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    addi sp, sp, 16
@@ -5340,46 +5340,46 @@ define void @test_nontemporal_S1_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
 ; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
 ; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
-; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
-; CHECK-RV32C-NEXT:    lbu a3, 52(a1)
-; CHECK-RV32C-NEXT:    lbu a4, 56(a1)
-; CHECK-RV32C-NEXT:    lbu a5, 60(a1)
 ; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32C-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32C-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32C-NEXT:    lbu a1, 44(a1)
-; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb a5, 15(a0)
-; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb a4, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb a3, 13(a0)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
 ; CHECK-RV32C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb a1, 11(a0)
+; CHECK-RV32C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb s1, 10(a0)
+; CHECK-RV32C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb s0, 9(a0)
+; CHECK-RV32C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
 ; CHECK-RV32C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
 ; CHECK-RV32C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
 ; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    addi sp, sp, 16
@@ -5405,114 +5405,114 @@ define void @test_nontemporal_S1_store_v16i8(ptr %p, <16 x i8> %v) {
 define void @test_nontemporal_S1_store_v8i16(ptr %p, <8 x i16> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_S1_store_v8i16:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lh a2, 32(a1)
-; CHECK-RV64-NEXT:    lh a3, 40(a1)
-; CHECK-RV64-NEXT:    lh a4, 48(a1)
-; CHECK-RV64-NEXT:    lh a5, 56(a1)
-; CHECK-RV64-NEXT:    lh a6, 0(a1)
-; CHECK-RV64-NEXT:    lh a7, 8(a1)
-; CHECK-RV64-NEXT:    lh t0, 16(a1)
-; CHECK-RV64-NEXT:    lh a1, 24(a1)
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 32(a1)
+; CHECK-RV64-NEXT:    lh a7, 40(a1)
+; CHECK-RV64-NEXT:    lh t0, 48(a1)
+; CHECK-RV64-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh a5, 14(a0)
+; CHECK-RV64-NEXT:    sh a6, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh a4, 12(a0)
+; CHECK-RV64-NEXT:    sh a7, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh a3, 10(a0)
+; CHECK-RV64-NEXT:    sh t0, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh a2, 8(a0)
+; CHECK-RV64-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh a1, 6(a0)
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh t0, 4(a0)
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh a7, 2(a0)
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sh a6, 0(a0)
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_S1_store_v8i16:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lh a2, 16(a1)
-; CHECK-RV32-NEXT:    lh a3, 20(a1)
-; CHECK-RV32-NEXT:    lh a4, 24(a1)
-; CHECK-RV32-NEXT:    lh a5, 28(a1)
-; CHECK-RV32-NEXT:    lh a6, 0(a1)
-; CHECK-RV32-NEXT:    lh a7, 4(a1)
-; CHECK-RV32-NEXT:    lh t0, 8(a1)
-; CHECK-RV32-NEXT:    lh a1, 12(a1)
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 16(a1)
+; CHECK-RV32-NEXT:    lh a7, 20(a1)
+; CHECK-RV32-NEXT:    lh t0, 24(a1)
+; CHECK-RV32-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh a5, 14(a0)
+; CHECK-RV32-NEXT:    sh a6, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh a4, 12(a0)
+; CHECK-RV32-NEXT:    sh a7, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh a3, 10(a0)
+; CHECK-RV32-NEXT:    sh t0, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh a2, 8(a0)
+; CHECK-RV32-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh a1, 6(a0)
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh t0, 4(a0)
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh a7, 2(a0)
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sh a6, 0(a0)
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_store_v8i16:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lh a7, 32(a1)
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 32(a1)
 ; CHECK-RV64C-NEXT:    lh a3, 40(a1)
 ; CHECK-RV64C-NEXT:    lh a4, 48(a1)
-; CHECK-RV64C-NEXT:    lh a5, 56(a1)
-; CHECK-RV64C-NEXT:    lh a6, 0(a1)
-; CHECK-RV64C-NEXT:    lh t0, 8(a1)
-; CHECK-RV64C-NEXT:    lh a2, 16(a1)
-; CHECK-RV64C-NEXT:    lh a1, 24(a1)
-; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sh a5, 14(a0)
+; CHECK-RV64C-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
 ; CHECK-RV64C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sh a7, 8(a0)
+; CHECK-RV64C-NEXT:    sh a4, 12(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sh a1, 6(a0)
+; CHECK-RV64C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sh a2, 4(a0)
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sh t0, 2(a0)
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_store_v8i16:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lh a7, 16(a1)
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 16(a1)
 ; CHECK-RV32C-NEXT:    lh a3, 20(a1)
 ; CHECK-RV32C-NEXT:    lh a4, 24(a1)
-; CHECK-RV32C-NEXT:    lh a5, 28(a1)
-; CHECK-RV32C-NEXT:    lh a6, 0(a1)
-; CHECK-RV32C-NEXT:    lh t0, 4(a1)
-; CHECK-RV32C-NEXT:    lh a2, 8(a1)
-; CHECK-RV32C-NEXT:    lh a1, 12(a1)
+; CHECK-RV32C-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sh a5, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
 ; CHECK-RV32C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sh a7, 8(a0)
+; CHECK-RV32C-NEXT:    sh a4, 12(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sh a1, 6(a0)
+; CHECK-RV32C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sh a2, 4(a0)
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sh t0, 2(a0)
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_store_v8i16:
@@ -5535,66 +5535,66 @@ define void @test_nontemporal_S1_store_v8i16(ptr %p, <8 x i16> %v) {
 define void @test_nontemporal_S1_store_v4i32(ptr %p, <4 x i32> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_S1_store_v4i32:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lw a2, 24(a1)
-; CHECK-RV64-NEXT:    lw a3, 16(a1)
-; CHECK-RV64-NEXT:    lw a4, 8(a1)
-; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    lw a2, 0(a1)
+; CHECK-RV64-NEXT:    lw a3, 8(a1)
+; CHECK-RV64-NEXT:    lw a4, 16(a1)
+; CHECK-RV64-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_S1_store_v4i32:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_store_v4i32:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lw a2, 24(a1)
-; CHECK-RV64C-NEXT:    lw a3, 16(a1)
-; CHECK-RV64C-NEXT:    lw a4, 8(a1)
-; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    lw a2, 0(a1)
+; CHECK-RV64C-NEXT:    lw a3, 8(a1)
+; CHECK-RV64C-NEXT:    lw a4, 16(a1)
+; CHECK-RV64C-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_store_v4i32:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_store_v4i32:
@@ -5618,49 +5618,49 @@ define void @test_nontemporal_S1_store_v2i64(ptr %p, <2 x i64> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_S1_store_v2i64:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.s1
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    ntl.s1
 ; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_S1_store_v2i64:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.s1
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_S1_store_v2i64:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.s1
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    c.ntl.s1
 ; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_S1_store_v2i64:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.s1
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_S1_store_v2i64:
@@ -5994,53 +5994,53 @@ define <16 x i8> @test_nontemporal_ALL_load_v16i8(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_ALL_load_v16i8:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v16i8:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v16i8:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v16i8:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v16i8:
@@ -6064,53 +6064,53 @@ define <8 x i16> @test_nontemporal_ALL_load_v8i16(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_ALL_load_v8i16:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v8i16:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v8i16:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v8i16:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v8i16:
@@ -6134,53 +6134,53 @@ define <4 x i32> @test_nontemporal_ALL_load_v4i32(ptr %p) {
 ; CHECK-RV64-LABEL: test_nontemporal_ALL_load_v4i32:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    ld a1, 0(a1)
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ld a1, 8(a1)
+; CHECK-RV64-NEXT:    sd a2, 0(a0)
+; CHECK-RV64-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v4i32:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v4i32:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    ld a2, 0(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    ld a1, 0(a1)
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ld a1, 8(a1)
+; CHECK-RV64C-NEXT:    sd a2, 0(a0)
+; CHECK-RV64C-NEXT:    sd a1, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v4i32:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v4i32:
@@ -6213,17 +6213,17 @@ define <2 x i64> @test_nontemporal_ALL_load_v2i64(ptr %p) {
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v2i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v2i64:
@@ -6238,17 +6238,17 @@ define <2 x i64> @test_nontemporal_ALL_load_v2i64(ptr %p) {
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v2i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v2i64:
@@ -6278,9 +6278,9 @@ define void @test_nontemporal_ALL_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_store_i64:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a2, 4(a0)
-; CHECK-RV32-NEXT:    ntl.all
 ; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_i64:
@@ -6292,9 +6292,9 @@ define void @test_nontemporal_ALL_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_i64:
 ; CHECK-RV32C:       # %bb.0:
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a2, 4(a0)
-; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_i64:
@@ -6306,9 +6306,9 @@ define void @test_nontemporal_ALL_store_i64(ptr %p, i64 %v) {
 ; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_i64:
 ; CHECK-RV32V:       # %bb.0:
 ; CHECK-RV32V-NEXT:    ntl.all
-; CHECK-RV32V-NEXT:    sw a2, 4(a0)
-; CHECK-RV32V-NEXT:    ntl.all
 ; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
 ; CHECK-RV32V-NEXT:    ret
   store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
   ret void
@@ -6571,46 +6571,46 @@ define void @test_nontemporal_ALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64-NEXT:    lbu a7, 40(a1)
 ; CHECK-RV64-NEXT:    lbu t0, 48(a1)
 ; CHECK-RV64-NEXT:    lbu t1, 56(a1)
-; CHECK-RV64-NEXT:    lbu t2, 96(a1)
-; CHECK-RV64-NEXT:    lbu t3, 104(a1)
-; CHECK-RV64-NEXT:    lbu t4, 112(a1)
-; CHECK-RV64-NEXT:    lbu t5, 120(a1)
-; CHECK-RV64-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64-NEXT:    lbu a1, 88(a1)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 96(a1)
+; CHECK-RV64-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t5, 15(a0)
+; CHECK-RV64-NEXT:    sb t6, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t4, 14(a0)
+; CHECK-RV64-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t3, 13(a0)
+; CHECK-RV64-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t2, 12(a0)
+; CHECK-RV64-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a1, 11(a0)
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb s1, 10(a0)
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb s0, 9(a0)
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t6, 8(a0)
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t1, 7(a0)
-; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.all
 ; CHECK-RV64-NEXT:    sb a7, 5(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.all
 ; CHECK-RV64-NEXT:    sb a3, 1(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
 ; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    addi sp, sp, 16
@@ -6632,46 +6632,46 @@ define void @test_nontemporal_ALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32-NEXT:    lbu a7, 20(a1)
 ; CHECK-RV32-NEXT:    lbu t0, 24(a1)
 ; CHECK-RV32-NEXT:    lbu t1, 28(a1)
-; CHECK-RV32-NEXT:    lbu t2, 48(a1)
-; CHECK-RV32-NEXT:    lbu t3, 52(a1)
-; CHECK-RV32-NEXT:    lbu t4, 56(a1)
-; CHECK-RV32-NEXT:    lbu t5, 60(a1)
-; CHECK-RV32-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32-NEXT:    lbu a1, 44(a1)
-; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t5, 15(a0)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 48(a1)
+; CHECK-RV32-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t4, 14(a0)
+; CHECK-RV32-NEXT:    sb t6, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t3, 13(a0)
+; CHECK-RV32-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t2, 12(a0)
+; CHECK-RV32-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a1, 11(a0)
+; CHECK-RV32-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb s1, 10(a0)
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb s0, 9(a0)
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t6, 8(a0)
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t1, 7(a0)
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
 ; CHECK-RV32-NEXT:    sb a7, 5(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
 ; CHECK-RV32-NEXT:    sb a3, 1(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
 ; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
@@ -6693,46 +6693,46 @@ define void @test_nontemporal_ALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
 ; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
 ; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
-; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
-; CHECK-RV64C-NEXT:    lbu a3, 104(a1)
-; CHECK-RV64C-NEXT:    lbu a4, 112(a1)
-; CHECK-RV64C-NEXT:    lbu a5, 120(a1)
 ; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
-; CHECK-RV64C-NEXT:    lbu s0, 72(a1)
-; CHECK-RV64C-NEXT:    lbu s1, 80(a1)
-; CHECK-RV64C-NEXT:    lbu a1, 88(a1)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a5, 15(a0)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a4, 14(a0)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a3, 13(a0)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 96(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 104(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 112(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 120(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a1, 11(a0)
+; CHECK-RV64C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb s1, 10(a0)
+; CHECK-RV64C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb s0, 9(a0)
+; CHECK-RV64C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64C-NEXT:    addi sp, sp, 16
@@ -6754,46 +6754,46 @@ define void @test_nontemporal_ALL_store_v16i8(ptr %p, <16 x i8> %v) {
 ; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
 ; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
 ; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
-; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
-; CHECK-RV32C-NEXT:    lbu a3, 52(a1)
-; CHECK-RV32C-NEXT:    lbu a4, 56(a1)
-; CHECK-RV32C-NEXT:    lbu a5, 60(a1)
 ; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
-; CHECK-RV32C-NEXT:    lbu s0, 36(a1)
-; CHECK-RV32C-NEXT:    lbu s1, 40(a1)
-; CHECK-RV32C-NEXT:    lbu a1, 44(a1)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a5, 15(a0)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a4, 14(a0)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a3, 13(a0)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 48(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 52(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 56(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 60(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb a2, 12(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a1, 11(a0)
+; CHECK-RV32C-NEXT:    sb s0, 13(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb s1, 10(a0)
+; CHECK-RV32C-NEXT:    sb s1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb s0, 9(a0)
+; CHECK-RV32C-NEXT:    sb a1, 15(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb t6, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb t2, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
 ; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
 ; CHECK-RV32C-NEXT:    addi sp, sp, 16
@@ -6819,114 +6819,114 @@ define void @test_nontemporal_ALL_store_v16i8(ptr %p, <16 x i8> %v) {
 define void @test_nontemporal_ALL_store_v8i16(ptr %p, <8 x i16> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_ALL_store_v8i16:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lh a2, 32(a1)
-; CHECK-RV64-NEXT:    lh a3, 40(a1)
-; CHECK-RV64-NEXT:    lh a4, 48(a1)
-; CHECK-RV64-NEXT:    lh a5, 56(a1)
-; CHECK-RV64-NEXT:    lh a6, 0(a1)
-; CHECK-RV64-NEXT:    lh a7, 8(a1)
-; CHECK-RV64-NEXT:    lh t0, 16(a1)
-; CHECK-RV64-NEXT:    lh a1, 24(a1)
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 32(a1)
+; CHECK-RV64-NEXT:    lh a7, 40(a1)
+; CHECK-RV64-NEXT:    lh t0, 48(a1)
+; CHECK-RV64-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a5, 14(a0)
+; CHECK-RV64-NEXT:    sh a6, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a4, 12(a0)
+; CHECK-RV64-NEXT:    sh a7, 10(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a3, 10(a0)
+; CHECK-RV64-NEXT:    sh t0, 12(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a2, 8(a0)
+; CHECK-RV64-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a1, 6(a0)
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh t0, 4(a0)
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a7, 2(a0)
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sh a6, 0(a0)
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_store_v8i16:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lh a2, 16(a1)
-; CHECK-RV32-NEXT:    lh a3, 20(a1)
-; CHECK-RV32-NEXT:    lh a4, 24(a1)
-; CHECK-RV32-NEXT:    lh a5, 28(a1)
-; CHECK-RV32-NEXT:    lh a6, 0(a1)
-; CHECK-RV32-NEXT:    lh a7, 4(a1)
-; CHECK-RV32-NEXT:    lh t0, 8(a1)
-; CHECK-RV32-NEXT:    lh a1, 12(a1)
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 16(a1)
+; CHECK-RV32-NEXT:    lh a7, 20(a1)
+; CHECK-RV32-NEXT:    lh t0, 24(a1)
+; CHECK-RV32-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a5, 14(a0)
+; CHECK-RV32-NEXT:    sh a6, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a4, 12(a0)
+; CHECK-RV32-NEXT:    sh a7, 10(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a3, 10(a0)
+; CHECK-RV32-NEXT:    sh t0, 12(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a2, 8(a0)
+; CHECK-RV32-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a1, 6(a0)
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh t0, 4(a0)
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a7, 2(a0)
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sh a6, 0(a0)
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_v8i16:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lh a7, 32(a1)
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 32(a1)
 ; CHECK-RV64C-NEXT:    lh a3, 40(a1)
 ; CHECK-RV64C-NEXT:    lh a4, 48(a1)
-; CHECK-RV64C-NEXT:    lh a5, 56(a1)
-; CHECK-RV64C-NEXT:    lh a6, 0(a1)
-; CHECK-RV64C-NEXT:    lh t0, 8(a1)
-; CHECK-RV64C-NEXT:    lh a2, 16(a1)
-; CHECK-RV64C-NEXT:    lh a1, 24(a1)
-; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a5, 14(a0)
+; CHECK-RV64C-NEXT:    lh a1, 56(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a4, 12(a0)
+; CHECK-RV64C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a7, 8(a0)
+; CHECK-RV64C-NEXT:    sh a4, 12(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a1, 6(a0)
+; CHECK-RV64C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a2, 4(a0)
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh t0, 2(a0)
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_v8i16:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lh a7, 16(a1)
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 16(a1)
 ; CHECK-RV32C-NEXT:    lh a3, 20(a1)
 ; CHECK-RV32C-NEXT:    lh a4, 24(a1)
-; CHECK-RV32C-NEXT:    lh a5, 28(a1)
-; CHECK-RV32C-NEXT:    lh a6, 0(a1)
-; CHECK-RV32C-NEXT:    lh t0, 4(a1)
-; CHECK-RV32C-NEXT:    lh a2, 8(a1)
-; CHECK-RV32C-NEXT:    lh a1, 12(a1)
-; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a5, 14(a0)
+; CHECK-RV32C-NEXT:    lh a1, 28(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a4, 12(a0)
+; CHECK-RV32C-NEXT:    sh a2, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
 ; CHECK-RV32C-NEXT:    sh a3, 10(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a7, 8(a0)
+; CHECK-RV32C-NEXT:    sh a4, 12(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a1, 6(a0)
+; CHECK-RV32C-NEXT:    sh a1, 14(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a2, 4(a0)
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh t0, 2(a0)
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_v8i16:
@@ -6949,66 +6949,66 @@ define void @test_nontemporal_ALL_store_v8i16(ptr %p, <8 x i16> %v) {
 define void @test_nontemporal_ALL_store_v4i32(ptr %p, <4 x i32> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_ALL_store_v4i32:
 ; CHECK-RV64:       # %bb.0:
-; CHECK-RV64-NEXT:    lw a2, 24(a1)
-; CHECK-RV64-NEXT:    lw a3, 16(a1)
-; CHECK-RV64-NEXT:    lw a4, 8(a1)
-; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    lw a2, 0(a1)
+; CHECK-RV64-NEXT:    lw a3, 8(a1)
+; CHECK-RV64-NEXT:    lw a4, 16(a1)
+; CHECK-RV64-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_store_v4i32:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_v4i32:
 ; CHECK-RV64C:       # %bb.0:
-; CHECK-RV64C-NEXT:    lw a2, 24(a1)
-; CHECK-RV64C-NEXT:    lw a3, 16(a1)
-; CHECK-RV64C-NEXT:    lw a4, 8(a1)
-; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    lw a2, 0(a1)
+; CHECK-RV64C-NEXT:    lw a3, 8(a1)
+; CHECK-RV64C-NEXT:    lw a4, 16(a1)
+; CHECK-RV64C-NEXT:    lw a1, 24(a1)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_v4i32:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_v4i32:
@@ -7032,49 +7032,49 @@ define void @test_nontemporal_ALL_store_v2i64(ptr %p, <2 x i64> %v) {
 ; CHECK-RV64-LABEL: test_nontemporal_ALL_store_v2i64:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    ntl.all
-; CHECK-RV64-NEXT:    sd a2, 8(a0)
-; CHECK-RV64-NEXT:    ntl.all
 ; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64-NEXT:    ret
 ;
 ; CHECK-RV32-LABEL: test_nontemporal_ALL_store_v2i64:
 ; CHECK-RV32:       # %bb.0:
-; CHECK-RV32-NEXT:    lw a2, 12(a1)
-; CHECK-RV32-NEXT:    lw a3, 8(a1)
-; CHECK-RV32-NEXT:    lw a4, 4(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    lw a2, 0(a1)
+; CHECK-RV32-NEXT:    lw a3, 4(a1)
+; CHECK-RV32-NEXT:    lw a4, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32-NEXT:    ntl.all
-; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_v2i64:
 ; CHECK-RV64C:       # %bb.0:
 ; CHECK-RV64C-NEXT:    c.ntl.all
-; CHECK-RV64C-NEXT:    sd a2, 8(a0)
-; CHECK-RV64C-NEXT:    c.ntl.all
 ; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
 ; CHECK-RV64C-NEXT:    ret
 ;
 ; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_v2i64:
 ; CHECK-RV32C:       # %bb.0:
-; CHECK-RV32C-NEXT:    lw a2, 12(a1)
-; CHECK-RV32C-NEXT:    lw a3, 8(a1)
-; CHECK-RV32C-NEXT:    lw a4, 4(a1)
-; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    lw a2, 0(a1)
+; CHECK-RV32C-NEXT:    lw a3, 4(a1)
+; CHECK-RV32C-NEXT:    lw a4, 8(a1)
+; CHECK-RV32C-NEXT:    lw a1, 12(a1)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a2, 0(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a3, 4(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a4, 8(a0)
 ; CHECK-RV32C-NEXT:    c.ntl.all
-; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    sw a1, 12(a0)
 ; CHECK-RV32C-NEXT:    ret
 ;
 ; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_v2i64:
diff --git a/llvm/test/CodeGen/RISCV/orc-b-patterns.ll b/llvm/test/CodeGen/RISCV/orc-b-patterns.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5ede992e844f15584f188c5004025b608a3b8a44
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/orc-b-patterns.ll
@@ -0,0 +1,372 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32I
+; RUN: llc -mtriple=riscv32 -mattr=+zbb -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32ZBB
+
+define i32 @orc_b_i32_mul255(i32 %x) nounwind {
+; RV32I-LABEL: orc_b_i32_mul255:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi a1, a1, 257
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 8
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_mul255:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    lui a1, 4112
+; RV32ZBB-NEXT:    addi a1, a1, 257
+; RV32ZBB-NEXT:    and a0, a0, a1
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    ret
+entry:
+  %and = and i32 %x, 16843009
+  %mul = mul nuw nsw i32 %and, 255
+  ret i32 %mul
+}
+
+
+define i32 @orc_b_i32_sub_shl8x_x_lsb(i32  %x)  {
+; RV32I-LABEL: orc_b_i32_sub_shl8x_x_lsb:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi a1, a1, 257
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 8
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_sub_shl8x_x_lsb:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    lui a1, 4112
+; RV32ZBB-NEXT:    addi a1, a1, 257
+; RV32ZBB-NEXT:    and a0, a0, a1
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    ret
+entry:
+  %and = and i32 %x, 16843009
+  %sub = mul nuw i32 %and, 255
+  ret i32 %sub
+}
+
+define i32 @orc_b_i32_sub_shl8x_x_lsb_preshifted(i32 %x){
+; RV32I-LABEL: orc_b_i32_sub_shl8x_x_lsb_preshifted:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    srli a0, a0, 11
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, 257
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 8
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_sub_shl8x_x_lsb_preshifted:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    srli a0, a0, 11
+; RV32ZBB-NEXT:    lui a1, 16
+; RV32ZBB-NEXT:    addi a1, a1, 257
+; RV32ZBB-NEXT:    and a0, a0, a1
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    ret
+entry:
+  %shr = lshr i32 %x, 11
+  %and = and i32 %shr, 16843009
+  %sub = mul nuw i32 %and, 255
+  ret i32 %sub
+}
+
+
+define  i32 @orc_b_i32_sub_shl8x_x_b1(i32  %x)  {
+; RV32I-LABEL: orc_b_i32_sub_shl8x_x_b1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, 8224
+; RV32I-NEXT:    addi a1, a1, 514
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 7
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_sub_shl8x_x_b1:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    lui a1, 8224
+; RV32ZBB-NEXT:    addi a1, a1, 514
+; RV32ZBB-NEXT:    and a0, a0, a1
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    ret
+entry:
+  %and = and i32 %x, 33686018
+  %shl = shl i32 %and, 7
+  %shr = lshr exact i32 %and, 1
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define  i32 @orc_b_i32_sub_shl8x_x_b2(i32  %x)  {
+; RV32I-LABEL: orc_b_i32_sub_shl8x_x_b2:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, 16448
+; RV32I-NEXT:    addi a1, a1, 1028
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 6
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_sub_shl8x_x_b2:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    lui a1, 16448
+; RV32ZBB-NEXT:    addi a1, a1, 1028
+; RV32ZBB-NEXT:    and a0, a0, a1
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    ret
+entry:
+  %and = and i32 %x, 67372036
+  %shl = shl i32 %and, 6
+  %shr = lshr exact i32 %and, 2
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define i32 @orc_b_i32_sub_shl8x_x_b3(i32  %x)  {
+; CHECK-LABEL: orc_b_i32_sub_shl8x_x_b3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a1, 24672
+; CHECK-NEXT:    addi a1, a1, 1542
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    slli a1, a0, 5
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    sub a0, a1, a0
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 101058054
+  %shl = shl nuw i32 %and, 5
+  %shr = lshr i32 %and, 3
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define  i32 @orc_b_i32_sub_shl8x_x_b4(i32  %x)  {
+; CHECK-LABEL: orc_b_i32_sub_shl8x_x_b4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a1, 32897
+; CHECK-NEXT:    addi a1, a1, -2040
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    slli a1, a0, 4
+; CHECK-NEXT:    srli a0, a0, 4
+; CHECK-NEXT:    sub a0, a1, a0
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 134744072
+  %shl = shl nuw i32 %and, 4
+  %shr = lshr i32 %and, 4
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define  i32 @orc_b_i32_sub_shl8x_x_b5(i32  %x)  {
+; CHECK-LABEL: orc_b_i32_sub_shl8x_x_b5:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a1, 65793
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    slli a1, a0, 3
+; CHECK-NEXT:    srli a0, a0, 5
+; CHECK-NEXT:    sub a0, a1, a0
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 269488144
+  %shl = shl nuw i32 %and, 3
+  %shr = lshr i32 %and, 5
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define i32 @orc_b_i32_sub_shl8x_x_b6(i32 %x)  {
+; CHECK-LABEL: orc_b_i32_sub_shl8x_x_b6:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a1, 131586
+; CHECK-NEXT:    addi a1, a1, 32
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    slli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 6
+; CHECK-NEXT:    sub a0, a1, a0
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 538976288
+  %shl = shl nuw i32 %and, 2
+  %shr = lshr i32 %and, 6
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define i32 @orc_b_i32_sub_shl8x_x_b7(i32 %x)  {
+; CHECK-LABEL: orc_b_i32_sub_shl8x_x_b7:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a1, 263172
+; CHECK-NEXT:    addi a1, a1, 64
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    slli a1, a0, 1
+; CHECK-NEXT:    srli a0, a0, 7
+; CHECK-NEXT:    sub a0, a1, a0
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 1077952576
+  %shl = shl nuw i32 %and, 1
+  %shr = lshr i32 %and, 7
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+define i32 @orc_b_i32_sub_shl8x_x_b1_shl_used(i32 %x, ptr %arr) {
+; RV32I-LABEL: orc_b_i32_sub_shl8x_x_b1_shl_used:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a2, 8224
+; RV32I-NEXT:    addi a2, a2, 514
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    slli a2, a0, 7
+; RV32I-NEXT:    srli a3, a0, 1
+; RV32I-NEXT:    sub a0, a2, a3
+; RV32I-NEXT:    sw a3, 0(a1)
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_sub_shl8x_x_b1_shl_used:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    lui a2, 8224
+; RV32ZBB-NEXT:    addi a2, a2, 514
+; RV32ZBB-NEXT:    and a0, a0, a2
+; RV32ZBB-NEXT:    srli a2, a0, 1
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    sw a2, 0(a1)
+; RV32ZBB-NEXT:    ret
+entry:
+  %and = and i32 %x, 33686018
+  %shl = shl i32 %and, 7
+  %shr = lshr exact i32 %and, 1
+  store i32 %shr, ptr %arr, align 4
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+define i32 @orc_b_i32_sub_shl8x_x_b1_srl_used(i32  %x, ptr %arr) {
+; RV32I-LABEL: orc_b_i32_sub_shl8x_x_b1_srl_used:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a2, 8224
+; RV32I-NEXT:    addi a2, a2, 514
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    slli a2, a0, 7
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    sub a0, a2, a0
+; RV32I-NEXT:    sw a2, 0(a1)
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_sub_shl8x_x_b1_srl_used:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    lui a2, 8224
+; RV32ZBB-NEXT:    addi a2, a2, 514
+; RV32ZBB-NEXT:    and a0, a0, a2
+; RV32ZBB-NEXT:    slli a2, a0, 7
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    sw a2, 0(a1)
+; RV32ZBB-NEXT:    ret
+entry:
+  %and = and i32 %x, 33686018
+  %shl = shl i32 %and, 7
+  %shr = lshr exact i32 %and, 1
+  store i32 %shl, ptr %arr, align 4
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define i32 @orc_b_i32_sub_shl8x_x_b1_not_used(i32  %x, ptr %arr) {
+; RV32I-LABEL: orc_b_i32_sub_shl8x_x_b1_not_used:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lui a1, 8224
+; RV32I-NEXT:    addi a1, a1, 514
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 7
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    sub a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: orc_b_i32_sub_shl8x_x_b1_not_used:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    lui a1, 8224
+; RV32ZBB-NEXT:    addi a1, a1, 514
+; RV32ZBB-NEXT:    and a0, a0, a1
+; RV32ZBB-NEXT:    orc.b a0, a0
+; RV32ZBB-NEXT:    ret
+entry:
+  %and = and i32 %x, 33686018
+  %shl = shl i32 %and, 7
+  %shr = lshr exact i32 %and, 1
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+define i32 @orc_b_i32_sub_shl8x_x_shl_used(i32  %x, ptr %arr){
+; CHECK-LABEL: orc_b_i32_sub_shl8x_x_shl_used:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a2, 4112
+; CHECK-NEXT:    addi a2, a2, 257
+; CHECK-NEXT:    and a0, a0, a2
+; CHECK-NEXT:    slli a2, a0, 8
+; CHECK-NEXT:    sub a0, a2, a0
+; CHECK-NEXT:    sw a2, 0(a1)
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 16843009
+  %shl = shl i32 %and, 8
+  store i32 %shl, ptr %arr, align 4
+  %sub = mul nuw i32 %and, 255
+  ret i32 %sub
+}
+
+define i32 @orc_b_i32_sub_shl8x_x_b1_both_used(i32  %x, ptr %arr) {
+; CHECK-LABEL: orc_b_i32_sub_shl8x_x_b1_both_used:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a2, 8224
+; CHECK-NEXT:    addi a2, a2, 514
+; CHECK-NEXT:    and a0, a0, a2
+; CHECK-NEXT:    slli a2, a0, 7
+; CHECK-NEXT:    srli a3, a0, 1
+; CHECK-NEXT:    sub a0, a2, a3
+; CHECK-NEXT:    sw a2, 0(a1)
+; CHECK-NEXT:    sw a3, 4(a1)
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 33686018
+  %shl = shl i32 %and, 7
+  %shr = lshr exact i32 %and, 1
+  store i32 %shl, ptr %arr, align 4
+  %arrayidx1 = getelementptr inbounds i8, ptr %arr, i32 4
+  store i32 %shr, ptr %arrayidx1, align 4
+  %sub = sub nsw i32 %shl, %shr
+  ret i32 %sub
+}
+
+
+define i32 @orc_b_i32_sub_x_shr8x(i32 %x)  {
+; CHECK-LABEL: orc_b_i32_sub_x_shr8x:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a1, 4112
+; CHECK-NEXT:    addi a1, a1, 257
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    srli a1, a0, 8
+; CHECK-NEXT:    sub a0, a0, a1
+; CHECK-NEXT:    ret
+entry:
+  %and = and i32 %x, 16843009
+  %shr = lshr i32 %and, 8
+  %sub = sub nsw i32 %and, %shr
+  ret i32 %sub
+}
diff --git a/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll b/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
index fe602b5b8fc2bcd1c636e9e7b1e1fca31d192d2a..0d571600a2d8130a6591e05e77bd5929499eddb6 100644
--- a/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
@@ -757,8 +757,8 @@ define i1 @uaddo_i42_increment_illegal_type(i42 %x, ptr %p) {
 ; RV64-NEXT:    slli a0, a2, 22
 ; RV64-NEXT:    srli a3, a0, 22
 ; RV64-NEXT:    seqz a0, a3
-; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    srli a3, a3, 32
+; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    sh a3, 4(a1)
 ; RV64-NEXT:    ret
   %a = add i42 %x, 1
@@ -1247,8 +1247,8 @@ define void @PR41129(ptr %p64) {
 ; RV32-NEXT:    beqz a3, .LBB37_2
 ; RV32-NEXT:  # %bb.1: # %false
 ; RV32-NEXT:    andi a1, a1, 7
-; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    ret
 ; RV32-NEXT:  .LBB37_2: # %true
 ; RV32-NEXT:    seqz a3, a1
diff --git a/llvm/test/CodeGen/RISCV/pr63816.ll b/llvm/test/CodeGen/RISCV/pr63816.ll
index 2e33a0e994996af6f1394c907dff0e8a1884449a..75ddeda3de5071f4d6123c069a476c31dfd243bb 100644
--- a/llvm/test/CodeGen/RISCV/pr63816.ll
+++ b/llvm/test/CodeGen/RISCV/pr63816.ll
@@ -55,14 +55,14 @@ define void @test(ptr %0, ptr %1) nounwind {
 ; CHECK-NEXT:    fcvt.d.s fs0, fs0
 ; CHECK-NEXT:    call __extendhfsf2
 ; CHECK-NEXT:    fcvt.d.s fa5, fa0
-; CHECK-NEXT:    fsd fa5, 56(s0)
-; CHECK-NEXT:    fsd fs0, 48(s0)
-; CHECK-NEXT:    fsd fs1, 40(s0)
 ; CHECK-NEXT:    fsd fs2, 32(s0)
-; CHECK-NEXT:    fsd fs3, 24(s0)
-; CHECK-NEXT:    fsd fs4, 16(s0)
-; CHECK-NEXT:    fsd fs5, 8(s0)
+; CHECK-NEXT:    fsd fs1, 40(s0)
+; CHECK-NEXT:    fsd fs0, 48(s0)
+; CHECK-NEXT:    fsd fa5, 56(s0)
 ; CHECK-NEXT:    fsd fs6, 0(s0)
+; CHECK-NEXT:    fsd fs5, 8(s0)
+; CHECK-NEXT:    fsd fs4, 16(s0)
+; CHECK-NEXT:    fsd fs3, 24(s0)
 ; CHECK-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s1, 56(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/push-pop-popret.ll b/llvm/test/CodeGen/RISCV/push-pop-popret.ll
index 85c2997e268a94703e3466f04c2c83ee8223e880..5cb00e617273a6fe1f42473c378318f663633551 100644
--- a/llvm/test/CodeGen/RISCV/push-pop-popret.ll
+++ b/llvm/test/CodeGen/RISCV/push-pop-popret.ll
@@ -1000,13 +1000,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV32IZCMP:       # %bb.0:
 ; RV32IZCMP-NEXT:    addi sp, sp, -48
 ; RV32IZCMP-NEXT:    mv a0, a1
-; RV32IZCMP-NEXT:    sw a7, 44(sp)
-; RV32IZCMP-NEXT:    sw a6, 40(sp)
 ; RV32IZCMP-NEXT:    sw a5, 36(sp)
-; RV32IZCMP-NEXT:    sw a4, 32(sp)
-; RV32IZCMP-NEXT:    sw a3, 28(sp)
-; RV32IZCMP-NEXT:    sw a2, 24(sp)
+; RV32IZCMP-NEXT:    sw a6, 40(sp)
+; RV32IZCMP-NEXT:    sw a7, 44(sp)
 ; RV32IZCMP-NEXT:    sw a1, 20(sp)
+; RV32IZCMP-NEXT:    sw a2, 24(sp)
+; RV32IZCMP-NEXT:    sw a3, 28(sp)
+; RV32IZCMP-NEXT:    sw a4, 32(sp)
 ; RV32IZCMP-NEXT:    addi a1, sp, 24
 ; RV32IZCMP-NEXT:    sw a1, 12(sp)
 ; RV32IZCMP-NEXT:    addi sp, sp, 48
@@ -1016,15 +1016,15 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV64IZCMP:       # %bb.0:
 ; RV64IZCMP-NEXT:    addi sp, sp, -80
 ; RV64IZCMP-NEXT:    sd a1, 24(sp)
-; RV64IZCMP-NEXT:    sd a7, 72(sp)
-; RV64IZCMP-NEXT:    sd a6, 64(sp)
 ; RV64IZCMP-NEXT:    addi a0, sp, 28
 ; RV64IZCMP-NEXT:    sd a0, 8(sp)
 ; RV64IZCMP-NEXT:    lw a0, 24(sp)
 ; RV64IZCMP-NEXT:    sd a5, 56(sp)
-; RV64IZCMP-NEXT:    sd a4, 48(sp)
-; RV64IZCMP-NEXT:    sd a3, 40(sp)
+; RV64IZCMP-NEXT:    sd a6, 64(sp)
+; RV64IZCMP-NEXT:    sd a7, 72(sp)
 ; RV64IZCMP-NEXT:    sd a2, 32(sp)
+; RV64IZCMP-NEXT:    sd a3, 40(sp)
+; RV64IZCMP-NEXT:    sd a4, 48(sp)
 ; RV64IZCMP-NEXT:    addi sp, sp, 80
 ; RV64IZCMP-NEXT:    ret
 ;
@@ -1032,13 +1032,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV32IZCMP-SR:       # %bb.0:
 ; RV32IZCMP-SR-NEXT:    addi sp, sp, -48
 ; RV32IZCMP-SR-NEXT:    mv a0, a1
-; RV32IZCMP-SR-NEXT:    sw a7, 44(sp)
-; RV32IZCMP-SR-NEXT:    sw a6, 40(sp)
 ; RV32IZCMP-SR-NEXT:    sw a5, 36(sp)
-; RV32IZCMP-SR-NEXT:    sw a4, 32(sp)
-; RV32IZCMP-SR-NEXT:    sw a3, 28(sp)
-; RV32IZCMP-SR-NEXT:    sw a2, 24(sp)
+; RV32IZCMP-SR-NEXT:    sw a6, 40(sp)
+; RV32IZCMP-SR-NEXT:    sw a7, 44(sp)
 ; RV32IZCMP-SR-NEXT:    sw a1, 20(sp)
+; RV32IZCMP-SR-NEXT:    sw a2, 24(sp)
+; RV32IZCMP-SR-NEXT:    sw a3, 28(sp)
+; RV32IZCMP-SR-NEXT:    sw a4, 32(sp)
 ; RV32IZCMP-SR-NEXT:    addi a1, sp, 24
 ; RV32IZCMP-SR-NEXT:    sw a1, 12(sp)
 ; RV32IZCMP-SR-NEXT:    addi sp, sp, 48
@@ -1048,15 +1048,15 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV64IZCMP-SR:       # %bb.0:
 ; RV64IZCMP-SR-NEXT:    addi sp, sp, -80
 ; RV64IZCMP-SR-NEXT:    sd a1, 24(sp)
-; RV64IZCMP-SR-NEXT:    sd a7, 72(sp)
-; RV64IZCMP-SR-NEXT:    sd a6, 64(sp)
 ; RV64IZCMP-SR-NEXT:    addi a0, sp, 28
 ; RV64IZCMP-SR-NEXT:    sd a0, 8(sp)
 ; RV64IZCMP-SR-NEXT:    lw a0, 24(sp)
 ; RV64IZCMP-SR-NEXT:    sd a5, 56(sp)
-; RV64IZCMP-SR-NEXT:    sd a4, 48(sp)
-; RV64IZCMP-SR-NEXT:    sd a3, 40(sp)
+; RV64IZCMP-SR-NEXT:    sd a6, 64(sp)
+; RV64IZCMP-SR-NEXT:    sd a7, 72(sp)
 ; RV64IZCMP-SR-NEXT:    sd a2, 32(sp)
+; RV64IZCMP-SR-NEXT:    sd a3, 40(sp)
+; RV64IZCMP-SR-NEXT:    sd a4, 48(sp)
 ; RV64IZCMP-SR-NEXT:    addi sp, sp, 80
 ; RV64IZCMP-SR-NEXT:    ret
 ;
@@ -1064,13 +1064,13 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    sw a7, 44(sp)
-; RV32I-NEXT:    sw a6, 40(sp)
 ; RV32I-NEXT:    sw a5, 36(sp)
-; RV32I-NEXT:    sw a4, 32(sp)
-; RV32I-NEXT:    sw a3, 28(sp)
-; RV32I-NEXT:    sw a2, 24(sp)
+; RV32I-NEXT:    sw a6, 40(sp)
+; RV32I-NEXT:    sw a7, 44(sp)
 ; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a2, 24(sp)
+; RV32I-NEXT:    sw a3, 28(sp)
+; RV32I-NEXT:    sw a4, 32(sp)
 ; RV32I-NEXT:    addi a1, sp, 24
 ; RV32I-NEXT:    sw a1, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -1080,15 +1080,15 @@ define i32 @varargs(ptr %fmt, ...) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -80
 ; RV64I-NEXT:    sd a1, 24(sp)
-; RV64I-NEXT:    sd a7, 72(sp)
-; RV64I-NEXT:    sd a6, 64(sp)
 ; RV64I-NEXT:    addi a0, sp, 28
 ; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    lw a0, 24(sp)
 ; RV64I-NEXT:    sd a5, 56(sp)
-; RV64I-NEXT:    sd a4, 48(sp)
-; RV64I-NEXT:    sd a3, 40(sp)
+; RV64I-NEXT:    sd a6, 64(sp)
+; RV64I-NEXT:    sd a7, 72(sp)
 ; RV64I-NEXT:    sd a2, 32(sp)
+; RV64I-NEXT:    sd a3, 40(sp)
+; RV64I-NEXT:    sd a4, 48(sp)
 ; RV64I-NEXT:    addi sp, sp, 80
 ; RV64I-NEXT:    ret
   %va = alloca ptr
diff --git a/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll b/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll
index 31ca8eab33508c7e94fc416a5e35c9cc762348db..352184c2d85ada08d216cac44f6e117ba93a38d1 100644
--- a/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll
+++ b/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll
@@ -65,8 +65,8 @@ define void @test2(ptr nocapture noundef %a, i32 noundef signext %n) {
 ; CHECK-NEXT:    lw a5, -4(a4)
 ; CHECK-NEXT:    lw a6, 0(a4)
 ; CHECK-NEXT:    addi a5, a5, 4
-; CHECK-NEXT:    sw a5, -4(a4)
 ; CHECK-NEXT:    addi a6, a6, 4
+; CHECK-NEXT:    sw a5, -4(a4)
 ; CHECK-NEXT:    sw a6, 0(a4)
 ; CHECK-NEXT:    addi a3, a3, 2
 ; CHECK-NEXT:    addi a4, a4, 8
diff --git a/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll b/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
index 957f44f9f669deac308b2724baef78760c8c621f..4901e268ec11a0c7a5a0eea157310d5803d1ff69 100644
--- a/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
+++ b/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
@@ -8,9 +8,9 @@ define void @test(ptr nocapture noundef writeonly %array1, i32 noundef signext %
 ; RV64-NEXT:    addiw a3, a1, 5
 ; RV64-NEXT:    slli a4, a3, 2
 ; RV64-NEXT:    add a4, a0, a4
-; RV64-NEXT:    sw a2, 0(a4)
 ; RV64-NEXT:    slli a1, a1, 2
 ; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sw a2, 0(a4)
 ; RV64-NEXT:    sw a2, 24(a0)
 ; RV64-NEXT:    sw a3, 140(a0)
 ; RV64-NEXT:    ret
@@ -42,9 +42,9 @@ define void @test1(ptr nocapture noundef %array1, i32 noundef signext %a, i32 no
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a6, a2
 ; RV64-NEXT:  .LBB1_2: # %entry
-; RV64-NEXT:    sw a6, 0(a5)
 ; RV64-NEXT:    slli a1, a1, 2
 ; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sw a6, 0(a5)
 ; RV64-NEXT:    sw a6, 24(a0)
 ; RV64-NEXT:    sw a4, 140(a0)
 ; RV64-NEXT:    ret
@@ -72,9 +72,9 @@ define void @test2(ptr nocapture noundef writeonly %array1, i64 noundef %a, i64
 ; RV64-NEXT:    addi a3, a1, 5
 ; RV64-NEXT:    slli a4, a3, 3
 ; RV64-NEXT:    add a4, a0, a4
-; RV64-NEXT:    sd a2, 0(a4)
 ; RV64-NEXT:    slli a1, a1, 3
 ; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sd a2, 0(a4)
 ; RV64-NEXT:    sd a2, 48(a0)
 ; RV64-NEXT:    sd a3, 280(a0)
 ; RV64-NEXT:    ret
@@ -102,9 +102,9 @@ define void @test3(ptr nocapture noundef %array1, i64 noundef %a, i64 noundef %b
 ; RV64-NEXT:  .LBB3_2: # %entry
 ; RV64-NEXT:    slli a2, a4, 3
 ; RV64-NEXT:    add a2, a0, a2
-; RV64-NEXT:    sd a5, 0(a2)
 ; RV64-NEXT:    slli a1, a1, 3
 ; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sd a5, 0(a2)
 ; RV64-NEXT:    sd a5, 48(a0)
 ; RV64-NEXT:    sd a4, 280(a0)
 ; RV64-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
index 197366e7e05fe85051c0e4b63d7c16b66ca0d9bd..248d6209d58239edbc9f23996377da6905075155 100644
--- a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
@@ -266,6 +266,37 @@ define i64 @cttz_i64(i64 %a) nounwind {
   ret i64 %1
 }
 
+define i32 @sexti1_i32(i32 %a) nounwind {
+; RV32I-LABEL: sexti1_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 31
+; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: sexti1_i32:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    th.ext a0, a0, 0, 0
+; RV32XTHEADBB-NEXT:    ret
+  %shl = shl i32 %a, 31
+  %shr = ashr exact i32 %shl, 31
+  ret i32 %shr
+}
+
+define i32 @sexti1_i32_2(i1 %a) nounwind {
+; RV32I-LABEL: sexti1_i32_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 31
+; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: sexti1_i32_2:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    th.ext a0, a0, 0, 0
+; RV32XTHEADBB-NEXT:    ret
+  %sext = sext i1 %a to i32
+  ret i32 %sext
+}
+
 define i32 @sextb_i32(i32 %a) nounwind {
 ; RV32I-LABEL: sextb_i32:
 ; RV32I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv32zbb.ll b/llvm/test/CodeGen/RISCV/rv32zbb.ll
index e24b1b41645cdf9cbf6a6b99e181d5548a65f2dd..af2ea35cf26c1ba0c14e07d91fa778198171f4e7 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbb.ll
@@ -754,10 +754,10 @@ define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
 ; RV32I-NEXT:    add a2, a2, a4
 ; RV32I-NEXT:    srli a2, a2, 24
 ; RV32I-NEXT:    add a1, a2, a1
-; RV32I-NEXT:    sw zero, 12(a0)
+; RV32I-NEXT:    sw a3, 0(a0)
 ; RV32I-NEXT:    sw zero, 4(a0)
 ; RV32I-NEXT:    sw a1, 8(a0)
-; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw zero, 12(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: ctpop_v2i64:
@@ -772,10 +772,10 @@ define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
 ; RV32ZBB-NEXT:    cpop a1, a1
 ; RV32ZBB-NEXT:    cpop a3, a4
 ; RV32ZBB-NEXT:    add a1, a3, a1
-; RV32ZBB-NEXT:    sw zero, 12(a0)
+; RV32ZBB-NEXT:    sw a2, 0(a0)
 ; RV32ZBB-NEXT:    sw zero, 4(a0)
 ; RV32ZBB-NEXT:    sw a1, 8(a0)
-; RV32ZBB-NEXT:    sw a2, 0(a0)
+; RV32ZBB-NEXT:    sw zero, 12(a0)
 ; RV32ZBB-NEXT:    ret
   %1 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
   ret <2 x i64> %1
@@ -1441,3 +1441,42 @@ define i32 @srai_slli2(i16 signext %0) {
   %3 = sext i16 %sext to i32
   ret i32 %3
 }
+
+define i1 @ctpop32_eq_one_nonzero(i32 %x) {
+; RV32I-LABEL: ctpop32_eq_one_nonzero:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    addi a1, a0, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop32_eq_one_nonzero:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    ret
+entry:
+  %popcnt = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp eq i32 %popcnt, 1
+  ret i1 %cmp
+}
+
+define i1 @ctpop32_ne_one_nonzero(i32 %x) {
+; RV32I-LABEL: ctpop32_ne_one_nonzero:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    addi a1, a0, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBB-LABEL: ctpop32_ne_one_nonzero:
+; RV32ZBB:       # %bb.0: # %entry
+; RV32ZBB-NEXT:    cpop a0, a0
+; RV32ZBB-NEXT:    sltiu a0, a0, 2
+; RV32ZBB-NEXT:    xori a0, a0, 1
+; RV32ZBB-NEXT:    ret
+entry:
+  %popcnt = tail call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp ne i32 %popcnt, 1
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
index 8ce4c44d8efcc2a941e5122d499d6b6ee7ccd0dd..47c4e8beecced095114a29abed8968a4fc128c77 100644
--- a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
@@ -558,6 +558,68 @@ define i64 @cttz_i64(i64 %a) nounwind {
   ret i64 %1
 }
 
+define signext i32 @sexti1_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: sexti1_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 63
+; RV64I-NEXT:    srai a0, a0, 63
+; RV64I-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: sexti1_i32:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    th.ext a0, a0, 0, 0
+; RV64XTHEADBB-NEXT:    ret
+  %shl = shl i32 %a, 31
+  %shr = ashr exact i32 %shl, 31
+  ret i32 %shr
+}
+
+define signext i32 @sexti1_i32_2(i1 %a) nounwind {
+; RV64I-LABEL: sexti1_i32_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 63
+; RV64I-NEXT:    srai a0, a0, 63
+; RV64I-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: sexti1_i32_2:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    th.ext a0, a0, 0, 0
+; RV64XTHEADBB-NEXT:    ret
+  %sext = sext i1 %a to i32
+  ret i32 %sext
+}
+
+define i64 @sexti1_i64(i64 %a) nounwind {
+; RV64I-LABEL: sexti1_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 63
+; RV64I-NEXT:    srai a0, a0, 63
+; RV64I-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: sexti1_i64:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    th.ext a0, a0, 0, 0
+; RV64XTHEADBB-NEXT:    ret
+  %shl = shl i64 %a, 63
+  %shr = ashr exact i64 %shl, 63
+  ret i64 %shr
+}
+
+define i64 @sexti1_i64_2(i1 %a) nounwind {
+; RV64I-LABEL: sexti1_i64_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 63
+; RV64I-NEXT:    srai a0, a0, 63
+; RV64I-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: sexti1_i64_2:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    th.ext a0, a0, 0, 0
+; RV64XTHEADBB-NEXT:    ret
+  %sext = sext i1 %a to i64
+  ret i64 %sext
+}
+
 define signext i32 @sextb_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: sextb_i32:
 ; RV64I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64zbb.ll
index 43a499806ab5ae8c9d37b98aefcd252e7a95682c..1e7814d588e4c0c1b69ef70e9e1ff3ec5d1e5074 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -1618,3 +1618,84 @@ entry:
   %5 = add nsw i32 %4, %0
   ret i32 %5
 }
+
+define i1 @ctpop32_eq_one_nonzero(i32 %x) {
+; RV64I-LABEL: ctpop32_eq_one_nonzero:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop32_eq_one_nonzero:
+; RV64ZBB:       # %bb.0: # %entry
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    ret
+entry:
+  %popcnt = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp eq i32 %popcnt, 1
+  ret i1 %cmp
+}
+
+define i1 @ctpop32_ne_one_nonzero(i32 %x) {
+; RV64I-LABEL: ctpop32_ne_one_nonzero:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop32_ne_one_nonzero:
+; RV64ZBB:       # %bb.0: # %entry
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    xori a0, a0, 1
+; RV64ZBB-NEXT:    ret
+entry:
+  %popcnt = tail call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp ne i32 %popcnt, 1
+  ret i1 %cmp
+}
+
+define i1 @ctpop64_eq_one_nonzero(i64 %x) {
+; RV64I-LABEL: ctpop64_eq_one_nonzero:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop64_eq_one_nonzero:
+; RV64ZBB:       # %bb.0: # %entry
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    sltiu a0, a0, 2
+; RV64ZBB-NEXT:    ret
+entry:
+  %popcnt = call range(i64 1, 65) i64 @llvm.ctpop.i64(i64 %x)
+  %cmp = icmp eq i64 %popcnt, 1
+  ret i1 %cmp
+}
+
+define i1 @ctpop32_eq_one_maybezero(i32 %x) {
+; RV64I-LABEL: ctpop32_eq_one_maybezero:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addiw a1, a0, -1
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sltu a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop32_eq_one_maybezero:
+; RV64ZBB:       # %bb.0: # %entry
+; RV64ZBB-NEXT:    cpopw a0, a0
+; RV64ZBB-NEXT:    addi a0, a0, -1
+; RV64ZBB-NEXT:    seqz a0, a0
+; RV64ZBB-NEXT:    ret
+entry:
+  %popcnt = call range(i32 0, 16) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp eq i32 %popcnt, 1
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/binop-splats.ll b/llvm/test/CodeGen/RISCV/rvv/binop-splats.ll
index f26e57b5a0b7330d3bc803537c2eede08484bda9..0e426ee309ababbd86c1c1933e5acca0a7a7d809 100644
--- a/llvm/test/CodeGen/RISCV/rvv/binop-splats.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/binop-splats.ll
@@ -397,10 +397,10 @@ define <vscale x 1 x i64> @nxv1i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
@@ -428,10 +428,10 @@ define <vscale x 2 x i64> @nxv2i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
@@ -459,10 +459,10 @@ define <vscale x 4 x i64> @nxv4i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
@@ -490,10 +490,10 @@ define <vscale x 8 x i64> @nxv8i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
@@ -580,13 +580,13 @@ define <vscale x 1 x i64> @uaddsatnxv1i64(i64 %x, i64 %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
-; RV32-NEXT:    sw a3, 4(sp)
 ; RV32-NEXT:    sw a2, 0(sp)
+; RV32-NEXT:    sw a3, 4(sp)
 ; RV32-NEXT:    mv a0, sp
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
 ; RV32-NEXT:    vsaddu.vv v8, v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
index 94e945f8032059d9a111d593d29e4145028d8d74..a34f06948a762cc0213fadbb5a9bf875bc4a6594 100644
--- a/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
@@ -706,9 +706,9 @@ define <vscale x 1 x i64> @bitreverse_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vsrl.vx v9, v8, a0
@@ -846,9 +846,9 @@ define <vscale x 2 x i64> @bitreverse_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vsrl.vx v10, v8, a0
@@ -986,9 +986,9 @@ define <vscale x 4 x i64> @bitreverse_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vsrl.vx v12, v8, a0
@@ -1130,9 +1130,9 @@ define <vscale x 8 x i64> @bitreverse_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    sub sp, sp, a0
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vx v16, v8, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll b/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
index 5709de567c18dc8cf57fdc2e7050d516bc81a2c9..afce04d107e728f9de5efa2f7a0805b0e7e3233f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/bitreverse-vp.ll
@@ -1437,9 +1437,9 @@ define <vscale x 1 x i64> @vp_bitreverse_nxv1i64(<vscale x 1 x i64> %va, <vscale
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1, v0.t
@@ -1578,9 +1578,9 @@ define <vscale x 1 x i64> @vp_bitreverse_nxv1i64_unmasked(<vscale x 1 x i64> %va
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1
@@ -1721,9 +1721,9 @@ define <vscale x 2 x i64> @vp_bitreverse_nxv2i64(<vscale x 2 x i64> %va, <vscale
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1, v0.t
@@ -1862,9 +1862,9 @@ define <vscale x 2 x i64> @vp_bitreverse_nxv2i64_unmasked(<vscale x 2 x i64> %va
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1
@@ -2005,9 +2005,9 @@ define <vscale x 4 x i64> @vp_bitreverse_nxv4i64(<vscale x 4 x i64> %va, <vscale
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1, v0.t
@@ -2146,9 +2146,9 @@ define <vscale x 4 x i64> @vp_bitreverse_nxv4i64_unmasked(<vscale x 4 x i64> %va
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1
@@ -2294,9 +2294,9 @@ define <vscale x 7 x i64> @vp_bitreverse_nxv7i64(<vscale x 7 x i64> %va, <vscale
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -2497,9 +2497,9 @@ define <vscale x 7 x i64> @vp_bitreverse_nxv7i64_unmasked(<vscale x 7 x i64> %va
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
@@ -2655,9 +2655,9 @@ define <vscale x 8 x i64> @vp_bitreverse_nxv8i64(<vscale x 8 x i64> %va, <vscale
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -2858,9 +2858,9 @@ define <vscale x 8 x i64> @vp_bitreverse_nxv8i64_unmasked(<vscale x 8 x i64> %va
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll
index d3fce3caf8aef95c8469b928b4ea4b2c4eb13258..e8e362b1f042dd9d51079d427f3b532fb0f7d5b5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/bswap-sdnode.ll
@@ -258,9 +258,9 @@ define <vscale x 1 x i64> @bswap_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vsrl.vx v9, v8, a0
@@ -341,9 +341,9 @@ define <vscale x 2 x i64> @bswap_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vsrl.vx v10, v8, a0
@@ -424,9 +424,9 @@ define <vscale x 4 x i64> @bswap_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vsrl.vx v12, v8, a0
@@ -511,9 +511,9 @@ define <vscale x 8 x i64> @bswap_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    sub sp, sp, a0
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a0, 1044480
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vx v16, v8, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll b/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
index 6917d7e44a8e6a9db2923f09b7c334b090d88058..171de6c2fddf179f921ad45109bc8bd09de25d6c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/bswap-vp.ll
@@ -513,9 +513,9 @@ define <vscale x 1 x i64> @vp_bswap_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1, v0.t
@@ -597,9 +597,9 @@ define <vscale x 1 x i64> @vp_bswap_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1
@@ -683,9 +683,9 @@ define <vscale x 2 x i64> @vp_bswap_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1, v0.t
@@ -767,9 +767,9 @@ define <vscale x 2 x i64> @vp_bswap_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1
@@ -853,9 +853,9 @@ define <vscale x 4 x i64> @vp_bswap_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1, v0.t
@@ -937,9 +937,9 @@ define <vscale x 4 x i64> @vp_bswap_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1
@@ -1028,9 +1028,9 @@ define <vscale x 7 x i64> @vp_bswap_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -1174,9 +1174,9 @@ define <vscale x 7 x i64> @vp_bswap_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
@@ -1274,9 +1274,9 @@ define <vscale x 8 x i64> @vp_bswap_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -1420,9 +1420,9 @@ define <vscale x 8 x i64> @vp_bswap_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
@@ -1640,9 +1640,9 @@ define <vscale x 1 x i48> @vp_bswap_nxv1i48(<vscale x 1 x i48> %va, <vscale x 1
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1, v0.t
diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-store.ll b/llvm/test/CodeGen/RISCV/rvv/combine-store.ll
index 5c4cc00a969580afd37383549336d55b3e690a16..181ab26c16af78cea3941f8d30465cd573f24896 100644
--- a/llvm/test/CodeGen/RISCV/rvv/combine-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/combine-store.ll
@@ -210,10 +210,10 @@ define void @combine_zero_stores_2xi32_unaligned(ptr %p) {
 define void @combine_zero_stores_2xi64(ptr %p) {
 ; RV32-LABEL: combine_zero_stores_2xi64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    sw zero, 0(a0)
-; RV32-NEXT:    sw zero, 12(a0)
+; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    sw zero, 8(a0)
+; RV32-NEXT:    sw zero, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: combine_zero_stores_2xi64:
@@ -300,10 +300,10 @@ define void @combine_allones_stores_2xi64(ptr %p) {
 ; RV32-LABEL: combine_allones_stores_2xi64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    li a1, -1
-; RV32-NEXT:    sw a1, 4(a0)
 ; RV32-NEXT:    sw a1, 0(a0)
-; RV32-NEXT:    sw a1, 12(a0)
+; RV32-NEXT:    sw a1, 4(a0)
 ; RV32-NEXT:    sw a1, 8(a0)
+; RV32-NEXT:    sw a1, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: combine_allones_stores_2xi64:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
index 8f40b02423094a0f56479c5388743b0a119923e2..29489be4dcb569a833f30c9e1aeec5e3af23b4eb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-binop-splats.ll
@@ -462,10 +462,10 @@ define <1 x i64> @v1i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
@@ -493,10 +493,10 @@ define <2 x i64> @v2i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
@@ -525,10 +525,10 @@ define <4 x i64> @v4i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
@@ -557,10 +557,10 @@ define <8 x i64> @v8i64(i64 %x, i64 %y) {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
index 90bedf87e04d30f230566ff3e186e02bdfc420b0..54265193b09f6eb9a8e475b1a7f6867f88e27a24 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
@@ -835,9 +835,9 @@ define <2 x i64> @vp_bitreverse_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1, v0.t
@@ -970,9 +970,9 @@ define <2 x i64> @vp_bitreverse_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl)
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1
@@ -1107,9 +1107,9 @@ define <4 x i64> @vp_bitreverse_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1, v0.t
@@ -1242,9 +1242,9 @@ define <4 x i64> @vp_bitreverse_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl)
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1
@@ -1379,9 +1379,9 @@ define <8 x i64> @vp_bitreverse_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1, v0.t
@@ -1514,9 +1514,9 @@ define <8 x i64> @vp_bitreverse_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl)
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1
@@ -1656,21 +1656,21 @@ define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroex
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
-; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -1862,21 +1862,21 @@ define <15 x i64> @vp_bitreverse_v15i64_unmasked(<15 x i64> %va, i32 zeroext %ev
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 8 * vlenb
-; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
@@ -2023,21 +2023,21 @@ define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroex
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
-; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -2229,21 +2229,21 @@ define <16 x i64> @vp_bitreverse_v16i64_unmasked(<16 x i64> %va, i32 zeroext %ev
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 8 * vlenb
-; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw zero, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
index 7f211d0f8f9bade79ec6dc8c9a94f6ab27729a60..37c22d328a4b1a18382ea0f77c378f1a729e556d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
@@ -113,9 +113,9 @@ define void @bitreverse_v2i64(ptr %x, ptr %y) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsrl.vx v9, v8, a1
 ; RV32-NEXT:    li a2, 40
@@ -364,9 +364,9 @@ define void @bitreverse_v4i64(ptr %x, ptr %y) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsrl.vx v10, v8, a1
 ; RV32-NEXT:    li a2, 40
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll
index 6f2e86097d6ff92cd392534c4436060af1f4a376..b8ddf74c30dbdced1a7b7e610f5b522920795ea4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap-vp.ll
@@ -283,9 +283,9 @@ define <2 x i64> @vp_bswap_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1, v0.t
@@ -361,9 +361,9 @@ define <2 x i64> @vp_bswap_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; RV32-NEXT:    vsll.vx v9, v8, a1
@@ -441,9 +441,9 @@ define <4 x i64> @vp_bswap_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1, v0.t
@@ -519,9 +519,9 @@ define <4 x i64> @vp_bswap_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; RV32-NEXT:    vsll.vx v10, v8, a1
@@ -599,9 +599,9 @@ define <8 x i64> @vp_bswap_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1, v0.t
@@ -677,9 +677,9 @@ define <8 x i64> @vp_bswap_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; RV32-NEXT:    vsll.vx v12, v8, a1
@@ -762,9 +762,9 @@ define <15 x i64> @vp_bswap_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -902,9 +902,9 @@ define <15 x i64> @vp_bswap_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
@@ -996,9 +996,9 @@ define <16 x i64> @vp_bswap_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
@@ -1136,9 +1136,9 @@ define <16 x i64> @vp_bswap_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vx v16, v8, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
index d5338f9b6c6fc09af28f6d57fcae37c3002b2e24..217597f6d57d6976de04210c37a32c1a9c2f1818 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
@@ -71,9 +71,9 @@ define void @bswap_v2i64(ptr %x, ptr %y) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsrl.vx v9, v8, a1
 ; RV32-NEXT:    li a2, 40
@@ -223,9 +223,9 @@ define void @bswap_v4i64(ptr %x, ptr %y) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    lui a1, 1044480
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsrl.vx v10, v8, a1
 ; RV32-NEXT:    li a2, 40
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll
index 9f48fdb3608a0bd267f33b76963f92b59c4a6c6d..d8e294d32cd0805119a1e2e4fae4d8e0b4fcbb4c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-calling-conv-fastcc.ll
@@ -308,8 +308,7 @@ define fastcc <32 x i32> @pass_vector_arg_direct_stack(<32 x i32> %x, <32 x i32>
 ; CHECK-NEXT:    li a0, 1
 ; CHECK-NEXT:    sd a0, 144(sp)
 ; CHECK-NEXT:    li a0, 13
-; CHECK-NEXT:    sd a0, 8(sp)
-; CHECK-NEXT:    li a0, 12
+; CHECK-NEXT:    li t0, 12
 ; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    li a2, 2
 ; CHECK-NEXT:    li a3, 3
@@ -321,7 +320,8 @@ define fastcc <32 x i32> @pass_vector_arg_direct_stack(<32 x i32> %x, <32 x i32>
 ; CHECK-NEXT:    li t4, 9
 ; CHECK-NEXT:    li t5, 10
 ; CHECK-NEXT:    li t6, 11
-; CHECK-NEXT:    sd a0, 0(sp)
+; CHECK-NEXT:    sd t0, 0(sp)
+; CHECK-NEXT:    sd a0, 8(sp)
 ; CHECK-NEXT:    li a0, 0
 ; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    call vector_arg_direct_stack
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
index c85547a57f550f78d900102a068ff111195e3a9e..a2217724ca8318afb2e56b0ac3c1660ef2a7e06b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
@@ -1511,20 +1511,20 @@ define <15 x i64> @vp_ctlz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
@@ -1653,20 +1653,20 @@ define <15 x i64> @vp_ctlz_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    vor.vv v8, v8, v16
@@ -1774,20 +1774,20 @@ define <16 x i64> @vp_ctlz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
@@ -1916,20 +1916,20 @@ define <16 x i64> @vp_ctlz_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    vor.vv v8, v8, v16
@@ -2045,21 +2045,21 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vslidedown.vi v24, v0, 2
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a3, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    bltu a0, a3, .LBB34_2
 ; RV32-NEXT:  # %bb.1:
@@ -2472,21 +2472,21 @@ define <32 x i64> @vp_ctlz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a1, a0
 ; RV32-NEXT:    bltu a0, a2, .LBB35_2
 ; RV32-NEXT:  # %bb.1:
@@ -4147,20 +4147,20 @@ define <15 x i64> @vp_ctlz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 z
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
@@ -4289,20 +4289,20 @@ define <15 x i64> @vp_ctlz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroex
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    vor.vv v8, v8, v16
@@ -4408,20 +4408,20 @@ define <16 x i64> @vp_ctlz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 z
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
@@ -4550,20 +4550,20 @@ define <16 x i64> @vp_ctlz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroex
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    vor.vv v8, v8, v16
@@ -4677,21 +4677,21 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vslidedown.vi v24, v0, 2
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a3, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    bltu a0, a3, .LBB70_2
 ; RV32-NEXT:  # %bb.1:
@@ -5104,21 +5104,21 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroex
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a1, a0
 ; RV32-NEXT:    bltu a0, a2, .LBB71_2
 ; RV32-NEXT:  # %bb.1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
index e90e52fba642b122148f02400e71bf07443a0945..ffc1bfd240804ef8566fffc5e63876282518b7dc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
@@ -1128,20 +1128,20 @@ define <15 x i64> @vp_ctpop_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    addi a1, sp, 40
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a1), zero
@@ -1252,20 +1252,20 @@ define <15 x i64> @vp_ctpop_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    addi a1, sp, 24
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a1), zero
@@ -1345,20 +1345,20 @@ define <16 x i64> @vp_ctpop_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    addi a1, sp, 40
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a1), zero
@@ -1469,20 +1469,20 @@ define <16 x i64> @vp_ctpop_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    addi a1, sp, 24
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a1), zero
@@ -1561,7 +1561,8 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 48 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    li a2, 24
+; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 48
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -1575,21 +1576,21 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    vslidedown.vi v7, v0, 2
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a1, a0
 ; RV32-NEXT:    bltu a0, a2, .LBB34_2
 ; RV32-NEXT:  # %bb.1:
@@ -1606,8 +1607,7 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    addi a2, sp, 32
 ; RV32-NEXT:    vlse64.v v16, (a2), zero
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
-; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    slli a2, a2, 4
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
 ; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
@@ -1618,47 +1618,40 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    addi a2, a2, 48
 ; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
+; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 5
+; RV32-NEXT:    slli a2, a2, 3
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 40
-; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    slli a2, a2, 5
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vsub.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vand.vv v8, v24, v16, v0.t
+; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 40
-; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    slli a2, a2, 3
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vi v8, v24, 2, v0.t
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    li a3, 40
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vand.vv v24, v8, v16, v0.t
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vadd.vv v16, v24, v8, v0.t
 ; RV32-NEXT:    addi a2, sp, 24
 ; RV32-NEXT:    addi a3, sp, 16
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a2), zero
+; RV32-NEXT:    vlse64.v v24, (a2), zero
 ; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    vlse64.v v8, (a3), zero
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    li a3, 40
@@ -1666,15 +1659,11 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
 ; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vmv8r.v v8, v16
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsrl.vi v16, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vand.vv v16, v16, v24, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 40
 ; RV32-NEXT:    mul a1, a1, a2
@@ -1695,44 +1684,28 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    and a0, a0, a2
 ; RV32-NEXT:    vmv1r.v v0, v7
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
 ; RV32-NEXT:    addi a2, a2, 48
 ; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v16, 1, v0.t
+; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v24, v8, v0.t
+; RV32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
 ; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a2, 24
-; RV32-NEXT:    mul a0, a0, a2
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add a0, sp, a0
 ; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
 ; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a2, 24
-; RV32-NEXT:    mul a0, a0, a2
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
 ; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
@@ -1862,21 +1835,21 @@ define <32 x i64> @vp_ctpop_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a1, a0
 ; RV32-NEXT:    bltu a0, a2, .LBB35_2
 ; RV32-NEXT:  # %bb.1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
index 7681280272596549d83e33f72168d6a8df1750b1..5e14689235e6b0ce4e0b58b743822ab893224994 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
@@ -1271,20 +1271,20 @@ define <15 x i64> @vp_cttz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
@@ -1393,20 +1393,20 @@ define <15 x i64> @vp_cttz_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1
@@ -1494,20 +1494,20 @@ define <16 x i64> @vp_cttz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
@@ -1616,20 +1616,20 @@ define <16 x i64> @vp_cttz_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1
@@ -1725,21 +1725,21 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vslidedown.vi v24, v0, 2
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a3, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    bltu a0, a3, .LBB34_2
 ; RV32-NEXT:  # %bb.1:
@@ -2112,21 +2112,21 @@ define <32 x i64> @vp_cttz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a1, a0
 ; RV32-NEXT:    bltu a0, a2, .LBB35_2
 ; RV32-NEXT:  # %bb.1:
@@ -3507,20 +3507,20 @@ define <15 x i64> @vp_cttz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 z
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
@@ -3629,20 +3629,20 @@ define <15 x i64> @vp_cttz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroex
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1
@@ -3728,20 +3728,20 @@ define <16 x i64> @vp_cttz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 z
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
@@ -3850,20 +3850,20 @@ define <16 x i64> @vp_cttz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroex
 ; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a1, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsub.vx v16, v8, a1
@@ -3957,21 +3957,21 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vslidedown.vi v24, v0, 2
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a3, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    bltu a0, a3, .LBB70_2
 ; RV32-NEXT:  # %bb.1:
@@ -4344,21 +4344,21 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroex
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    sw a1, 40(sp)
+; RV32-NEXT:    sw a1, 44(sp)
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    sw a1, 32(sp)
+; RV32-NEXT:    sw a1, 36(sp)
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    sw a1, 24(sp)
+; RV32-NEXT:    sw a1, 28(sp)
 ; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a1, a0
 ; RV32-NEXT:    bltu a0, a2, .LBB71_2
 ; RV32-NEXT:  # %bb.1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll
index 81e20a298816305a5dc3f4805ce64371aa98cbc5..04ebc7ca6b2b89824d2ddf8a3722ac48578fb81f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-elen.ll
@@ -42,10 +42,10 @@ define void @add_v2i64(ptr %x, ptr %y) {
 ; RV32-NEXT:    add t0, a6, t0
 ; RV32-NEXT:    sltu a4, t0, a6
 ; RV32-NEXT:    add a1, a1, a4
-; RV32-NEXT:    sw t0, 8(a0)
 ; RV32-NEXT:    sw a2, 0(a0)
-; RV32-NEXT:    sw a1, 12(a0)
 ; RV32-NEXT:    sw a3, 4(a0)
+; RV32-NEXT:    sw t0, 8(a0)
+; RV32-NEXT:    sw a1, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: add_v2i64:
@@ -56,8 +56,8 @@ define void @add_v2i64(ptr %x, ptr %y) {
 ; RV64-NEXT:    ld a1, 8(a1)
 ; RV64-NEXT:    add a2, a2, a4
 ; RV64-NEXT:    add a1, a3, a1
-; RV64-NEXT:    sd a1, 8(a0)
 ; RV64-NEXT:    sd a2, 0(a0)
+; RV64-NEXT:    sd a1, 8(a0)
 ; RV64-NEXT:    ret
   %a = load <2 x i64>, ptr %x
   %b = load <2 x i64>, ptr %y
@@ -140,8 +140,8 @@ define void @fadd_v2f64(ptr %x, ptr %y) {
 ; CHECK-NEXT:    fld fa2, 8(a1)
 ; CHECK-NEXT:    fadd.d fa5, fa5, fa3
 ; CHECK-NEXT:    fadd.d fa4, fa4, fa2
-; CHECK-NEXT:    fsd fa4, 8(a0)
 ; CHECK-NEXT:    fsd fa5, 0(a0)
+; CHECK-NEXT:    fsd fa4, 8(a0)
 ; CHECK-NEXT:    ret
   %a = load <2 x double>, ptr %x
   %b = load <2 x double>, ptr %y
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-int.ll
index 5bf8b07efc1dafd40b57f99070c26ed7ca2e7b2f..d34235127e8389744ef2da0a7c901a5aaa81a664 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-expandload-int.ll
@@ -603,10 +603,10 @@ define <1 x i64> @expandload_v1i64(ptr %base, <1 x i64> %src0, <1 x i1> %mask) {
 ; RV32-NEXT:  # %bb.1: # %cond.load
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    lw a1, 4(a0)
-; RV32-NEXT:    lw a0, 0(a0)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    lw a1, 0(a0)
+; RV32-NEXT:    lw a0, 4(a0)
+; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
index 53059a4f28d42bc88bcabd77ffc475a834aab0b3..96b9b2bac2993ca1760e098e0ad603cfdb5aa5ef 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
@@ -502,22 +502,22 @@ define <16 x float> @buildvec_v16f32(float %e0, float %e1, float %e2, float %e3,
 ; RV32-NEXT:    addi s0, sp, 128
 ; RV32-NEXT:    .cfi_def_cfa s0, 0
 ; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    sw a7, 60(sp)
-; RV32-NEXT:    sw a6, 56(sp)
-; RV32-NEXT:    sw a5, 52(sp)
 ; RV32-NEXT:    sw a4, 48(sp)
-; RV32-NEXT:    sw a3, 44(sp)
-; RV32-NEXT:    sw a2, 40(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    sw a5, 52(sp)
+; RV32-NEXT:    sw a6, 56(sp)
+; RV32-NEXT:    sw a7, 60(sp)
 ; RV32-NEXT:    sw a0, 32(sp)
-; RV32-NEXT:    fsw fa7, 28(sp)
-; RV32-NEXT:    fsw fa6, 24(sp)
-; RV32-NEXT:    fsw fa5, 20(sp)
+; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    sw a2, 40(sp)
+; RV32-NEXT:    sw a3, 44(sp)
 ; RV32-NEXT:    fsw fa4, 16(sp)
-; RV32-NEXT:    fsw fa3, 12(sp)
-; RV32-NEXT:    fsw fa2, 8(sp)
-; RV32-NEXT:    fsw fa1, 4(sp)
+; RV32-NEXT:    fsw fa5, 20(sp)
+; RV32-NEXT:    fsw fa6, 24(sp)
+; RV32-NEXT:    fsw fa7, 28(sp)
 ; RV32-NEXT:    fsw fa0, 0(sp)
+; RV32-NEXT:    fsw fa1, 4(sp)
+; RV32-NEXT:    fsw fa2, 8(sp)
+; RV32-NEXT:    fsw fa3, 12(sp)
 ; RV32-NEXT:    mv a0, sp
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
@@ -546,22 +546,22 @@ define <16 x float> @buildvec_v16f32(float %e0, float %e1, float %e2, float %e3,
 ; RV64-NEXT:    fmv.w.x ft5, a5
 ; RV64-NEXT:    fmv.w.x ft6, a6
 ; RV64-NEXT:    fmv.w.x ft7, a7
-; RV64-NEXT:    fsw fa7, 28(sp)
-; RV64-NEXT:    fsw fa6, 24(sp)
-; RV64-NEXT:    fsw fa5, 20(sp)
 ; RV64-NEXT:    fsw fa4, 16(sp)
-; RV64-NEXT:    fsw fa3, 12(sp)
-; RV64-NEXT:    fsw fa2, 8(sp)
-; RV64-NEXT:    fsw fa1, 4(sp)
+; RV64-NEXT:    fsw fa5, 20(sp)
+; RV64-NEXT:    fsw fa6, 24(sp)
+; RV64-NEXT:    fsw fa7, 28(sp)
 ; RV64-NEXT:    fsw fa0, 0(sp)
-; RV64-NEXT:    fsw ft7, 60(sp)
-; RV64-NEXT:    fsw ft6, 56(sp)
-; RV64-NEXT:    fsw ft5, 52(sp)
+; RV64-NEXT:    fsw fa1, 4(sp)
+; RV64-NEXT:    fsw fa2, 8(sp)
+; RV64-NEXT:    fsw fa3, 12(sp)
 ; RV64-NEXT:    fsw ft4, 48(sp)
-; RV64-NEXT:    fsw ft3, 44(sp)
-; RV64-NEXT:    fsw ft2, 40(sp)
-; RV64-NEXT:    fsw ft1, 36(sp)
+; RV64-NEXT:    fsw ft5, 52(sp)
+; RV64-NEXT:    fsw ft6, 56(sp)
+; RV64-NEXT:    fsw ft7, 60(sp)
 ; RV64-NEXT:    fsw ft0, 32(sp)
+; RV64-NEXT:    fsw ft1, 36(sp)
+; RV64-NEXT:    fsw ft2, 40(sp)
+; RV64-NEXT:    fsw ft3, 44(sp)
 ; RV64-NEXT:    mv a0, sp
 ; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a0)
@@ -621,42 +621,42 @@ define <32 x float> @buildvec_v32f32(float %e0, float %e1, float %e2, float %e3,
 ; RV32-NEXT:    flw ft9, 36(s0)
 ; RV32-NEXT:    flw ft10, 40(s0)
 ; RV32-NEXT:    flw ft11, 44(s0)
-; RV32-NEXT:    flw fs0, 60(s0)
-; RV32-NEXT:    flw fs1, 56(s0)
-; RV32-NEXT:    flw fs2, 52(s0)
-; RV32-NEXT:    flw fs3, 48(s0)
-; RV32-NEXT:    fsw fs0, 124(sp)
-; RV32-NEXT:    fsw fs1, 120(sp)
-; RV32-NEXT:    fsw fs2, 116(sp)
-; RV32-NEXT:    fsw fs3, 112(sp)
-; RV32-NEXT:    fsw ft11, 108(sp)
-; RV32-NEXT:    fsw ft10, 104(sp)
-; RV32-NEXT:    fsw ft9, 100(sp)
+; RV32-NEXT:    flw fs0, 48(s0)
+; RV32-NEXT:    flw fs1, 52(s0)
+; RV32-NEXT:    flw fs2, 56(s0)
+; RV32-NEXT:    flw fs3, 60(s0)
+; RV32-NEXT:    fsw fs0, 112(sp)
+; RV32-NEXT:    fsw fs1, 116(sp)
+; RV32-NEXT:    fsw fs2, 120(sp)
+; RV32-NEXT:    fsw fs3, 124(sp)
 ; RV32-NEXT:    fsw ft8, 96(sp)
-; RV32-NEXT:    fsw ft7, 92(sp)
-; RV32-NEXT:    fsw ft6, 88(sp)
-; RV32-NEXT:    fsw ft5, 84(sp)
+; RV32-NEXT:    fsw ft9, 100(sp)
+; RV32-NEXT:    fsw ft10, 104(sp)
+; RV32-NEXT:    fsw ft11, 108(sp)
 ; RV32-NEXT:    fsw ft4, 80(sp)
-; RV32-NEXT:    fsw ft3, 76(sp)
-; RV32-NEXT:    fsw ft2, 72(sp)
-; RV32-NEXT:    fsw ft1, 68(sp)
+; RV32-NEXT:    fsw ft5, 84(sp)
+; RV32-NEXT:    fsw ft6, 88(sp)
+; RV32-NEXT:    fsw ft7, 92(sp)
 ; RV32-NEXT:    fsw ft0, 64(sp)
-; RV32-NEXT:    sw a7, 60(sp)
-; RV32-NEXT:    sw a6, 56(sp)
-; RV32-NEXT:    sw a5, 52(sp)
+; RV32-NEXT:    fsw ft1, 68(sp)
+; RV32-NEXT:    fsw ft2, 72(sp)
+; RV32-NEXT:    fsw ft3, 76(sp)
 ; RV32-NEXT:    sw a4, 48(sp)
-; RV32-NEXT:    sw a3, 44(sp)
-; RV32-NEXT:    sw a2, 40(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    sw a5, 52(sp)
+; RV32-NEXT:    sw a6, 56(sp)
+; RV32-NEXT:    sw a7, 60(sp)
 ; RV32-NEXT:    sw a0, 32(sp)
-; RV32-NEXT:    fsw fa7, 28(sp)
-; RV32-NEXT:    fsw fa6, 24(sp)
-; RV32-NEXT:    fsw fa5, 20(sp)
+; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    sw a2, 40(sp)
+; RV32-NEXT:    sw a3, 44(sp)
 ; RV32-NEXT:    fsw fa4, 16(sp)
-; RV32-NEXT:    fsw fa3, 12(sp)
-; RV32-NEXT:    fsw fa2, 8(sp)
-; RV32-NEXT:    fsw fa1, 4(sp)
+; RV32-NEXT:    fsw fa5, 20(sp)
+; RV32-NEXT:    fsw fa6, 24(sp)
+; RV32-NEXT:    fsw fa7, 28(sp)
 ; RV32-NEXT:    fsw fa0, 0(sp)
+; RV32-NEXT:    fsw fa1, 4(sp)
+; RV32-NEXT:    fsw fa2, 8(sp)
+; RV32-NEXT:    fsw fa3, 12(sp)
 ; RV32-NEXT:    li a0, 32
 ; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -726,42 +726,42 @@ define <32 x float> @buildvec_v32f32(float %e0, float %e1, float %e2, float %e3,
 ; RV64-NEXT:    flw fs5, 72(s0)
 ; RV64-NEXT:    flw fs6, 80(s0)
 ; RV64-NEXT:    flw fs7, 88(s0)
-; RV64-NEXT:    flw fs8, 120(s0)
-; RV64-NEXT:    flw fs9, 112(s0)
-; RV64-NEXT:    flw fs10, 104(s0)
-; RV64-NEXT:    flw fs11, 96(s0)
-; RV64-NEXT:    fsw fs8, 124(sp)
-; RV64-NEXT:    fsw fs9, 120(sp)
-; RV64-NEXT:    fsw fs10, 116(sp)
-; RV64-NEXT:    fsw fs11, 112(sp)
-; RV64-NEXT:    fsw fs7, 108(sp)
-; RV64-NEXT:    fsw fs6, 104(sp)
-; RV64-NEXT:    fsw fs5, 100(sp)
+; RV64-NEXT:    flw fs8, 96(s0)
+; RV64-NEXT:    flw fs9, 104(s0)
+; RV64-NEXT:    flw fs10, 112(s0)
+; RV64-NEXT:    flw fs11, 120(s0)
+; RV64-NEXT:    fsw fs8, 112(sp)
+; RV64-NEXT:    fsw fs9, 116(sp)
+; RV64-NEXT:    fsw fs10, 120(sp)
+; RV64-NEXT:    fsw fs11, 124(sp)
 ; RV64-NEXT:    fsw fs4, 96(sp)
-; RV64-NEXT:    fsw fs3, 92(sp)
-; RV64-NEXT:    fsw fs2, 88(sp)
-; RV64-NEXT:    fsw fs1, 84(sp)
+; RV64-NEXT:    fsw fs5, 100(sp)
+; RV64-NEXT:    fsw fs6, 104(sp)
+; RV64-NEXT:    fsw fs7, 108(sp)
 ; RV64-NEXT:    fsw fs0, 80(sp)
-; RV64-NEXT:    fsw ft11, 76(sp)
-; RV64-NEXT:    fsw ft10, 72(sp)
-; RV64-NEXT:    fsw ft9, 68(sp)
+; RV64-NEXT:    fsw fs1, 84(sp)
+; RV64-NEXT:    fsw fs2, 88(sp)
+; RV64-NEXT:    fsw fs3, 92(sp)
 ; RV64-NEXT:    fsw ft8, 64(sp)
-; RV64-NEXT:    fsw fa7, 28(sp)
-; RV64-NEXT:    fsw fa6, 24(sp)
-; RV64-NEXT:    fsw fa5, 20(sp)
+; RV64-NEXT:    fsw ft9, 68(sp)
+; RV64-NEXT:    fsw ft10, 72(sp)
+; RV64-NEXT:    fsw ft11, 76(sp)
 ; RV64-NEXT:    fsw fa4, 16(sp)
-; RV64-NEXT:    fsw fa3, 12(sp)
-; RV64-NEXT:    fsw fa2, 8(sp)
-; RV64-NEXT:    fsw fa1, 4(sp)
+; RV64-NEXT:    fsw fa5, 20(sp)
+; RV64-NEXT:    fsw fa6, 24(sp)
+; RV64-NEXT:    fsw fa7, 28(sp)
 ; RV64-NEXT:    fsw fa0, 0(sp)
-; RV64-NEXT:    fsw ft7, 60(sp)
-; RV64-NEXT:    fsw ft6, 56(sp)
-; RV64-NEXT:    fsw ft5, 52(sp)
+; RV64-NEXT:    fsw fa1, 4(sp)
+; RV64-NEXT:    fsw fa2, 8(sp)
+; RV64-NEXT:    fsw fa3, 12(sp)
 ; RV64-NEXT:    fsw ft4, 48(sp)
-; RV64-NEXT:    fsw ft3, 44(sp)
-; RV64-NEXT:    fsw ft2, 40(sp)
-; RV64-NEXT:    fsw ft1, 36(sp)
+; RV64-NEXT:    fsw ft5, 52(sp)
+; RV64-NEXT:    fsw ft6, 56(sp)
+; RV64-NEXT:    fsw ft7, 60(sp)
 ; RV64-NEXT:    fsw ft0, 32(sp)
+; RV64-NEXT:    fsw ft1, 36(sp)
+; RV64-NEXT:    fsw ft2, 40(sp)
+; RV64-NEXT:    fsw ft3, 44(sp)
 ; RV64-NEXT:    li a0, 32
 ; RV64-NEXT:    mv a1, sp
 ; RV64-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -830,14 +830,14 @@ define <8 x double> @buildvec_v8f64(double %e0, double %e1, double %e2, double %
 ; RV32-NEXT:    addi s0, sp, 128
 ; RV32-NEXT:    .cfi_def_cfa s0, 0
 ; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    fsd fa7, 56(sp)
-; RV32-NEXT:    fsd fa6, 48(sp)
-; RV32-NEXT:    fsd fa5, 40(sp)
 ; RV32-NEXT:    fsd fa4, 32(sp)
-; RV32-NEXT:    fsd fa3, 24(sp)
-; RV32-NEXT:    fsd fa2, 16(sp)
-; RV32-NEXT:    fsd fa1, 8(sp)
+; RV32-NEXT:    fsd fa5, 40(sp)
+; RV32-NEXT:    fsd fa6, 48(sp)
+; RV32-NEXT:    fsd fa7, 56(sp)
 ; RV32-NEXT:    fsd fa0, 0(sp)
+; RV32-NEXT:    fsd fa1, 8(sp)
+; RV32-NEXT:    fsd fa2, 16(sp)
+; RV32-NEXT:    fsd fa3, 24(sp)
 ; RV32-NEXT:    mv a0, sp
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
@@ -858,14 +858,14 @@ define <8 x double> @buildvec_v8f64(double %e0, double %e1, double %e2, double %
 ; RV64-NEXT:    addi s0, sp, 128
 ; RV64-NEXT:    .cfi_def_cfa s0, 0
 ; RV64-NEXT:    andi sp, sp, -64
-; RV64-NEXT:    fsd fa7, 56(sp)
-; RV64-NEXT:    fsd fa6, 48(sp)
-; RV64-NEXT:    fsd fa5, 40(sp)
 ; RV64-NEXT:    fsd fa4, 32(sp)
-; RV64-NEXT:    fsd fa3, 24(sp)
-; RV64-NEXT:    fsd fa2, 16(sp)
-; RV64-NEXT:    fsd fa1, 8(sp)
+; RV64-NEXT:    fsd fa5, 40(sp)
+; RV64-NEXT:    fsd fa6, 48(sp)
+; RV64-NEXT:    fsd fa7, 56(sp)
 ; RV64-NEXT:    fsd fa0, 0(sp)
+; RV64-NEXT:    fsd fa1, 8(sp)
+; RV64-NEXT:    fsd fa2, 16(sp)
+; RV64-NEXT:    fsd fa3, 24(sp)
 ; RV64-NEXT:    mv a0, sp
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
@@ -909,26 +909,26 @@ define <16 x double> @buildvec_v16f64(double %e0, double %e1, double %e2, double
 ; RV32-NEXT:    sw a6, 120(sp)
 ; RV32-NEXT:    sw a7, 124(sp)
 ; RV32-NEXT:    fld ft3, 120(sp)
-; RV32-NEXT:    fld ft4, 24(s0)
-; RV32-NEXT:    fld ft5, 16(s0)
-; RV32-NEXT:    fld ft6, 8(s0)
-; RV32-NEXT:    fld ft7, 0(s0)
-; RV32-NEXT:    fsd ft4, 248(sp)
-; RV32-NEXT:    fsd ft5, 240(sp)
-; RV32-NEXT:    fsd ft6, 232(sp)
-; RV32-NEXT:    fsd ft7, 224(sp)
-; RV32-NEXT:    fsd fa7, 184(sp)
-; RV32-NEXT:    fsd fa6, 176(sp)
-; RV32-NEXT:    fsd fa5, 168(sp)
+; RV32-NEXT:    fld ft4, 0(s0)
+; RV32-NEXT:    fld ft5, 8(s0)
+; RV32-NEXT:    fld ft6, 16(s0)
+; RV32-NEXT:    fld ft7, 24(s0)
+; RV32-NEXT:    fsd ft4, 224(sp)
+; RV32-NEXT:    fsd ft5, 232(sp)
+; RV32-NEXT:    fsd ft6, 240(sp)
+; RV32-NEXT:    fsd ft7, 248(sp)
 ; RV32-NEXT:    fsd fa4, 160(sp)
-; RV32-NEXT:    fsd fa3, 152(sp)
-; RV32-NEXT:    fsd fa2, 144(sp)
-; RV32-NEXT:    fsd fa1, 136(sp)
+; RV32-NEXT:    fsd fa5, 168(sp)
+; RV32-NEXT:    fsd fa6, 176(sp)
+; RV32-NEXT:    fsd fa7, 184(sp)
 ; RV32-NEXT:    fsd fa0, 128(sp)
-; RV32-NEXT:    fsd ft3, 216(sp)
-; RV32-NEXT:    fsd ft2, 208(sp)
-; RV32-NEXT:    fsd ft1, 200(sp)
+; RV32-NEXT:    fsd fa1, 136(sp)
+; RV32-NEXT:    fsd fa2, 144(sp)
+; RV32-NEXT:    fsd fa3, 152(sp)
 ; RV32-NEXT:    fsd ft0, 192(sp)
+; RV32-NEXT:    fsd ft1, 200(sp)
+; RV32-NEXT:    fsd ft2, 208(sp)
+; RV32-NEXT:    fsd ft3, 216(sp)
 ; RV32-NEXT:    addi a0, sp, 128
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
@@ -949,22 +949,22 @@ define <16 x double> @buildvec_v16f64(double %e0, double %e1, double %e2, double
 ; RV64-NEXT:    addi s0, sp, 256
 ; RV64-NEXT:    .cfi_def_cfa s0, 0
 ; RV64-NEXT:    andi sp, sp, -128
-; RV64-NEXT:    sd a7, 120(sp)
-; RV64-NEXT:    sd a6, 112(sp)
-; RV64-NEXT:    sd a5, 104(sp)
 ; RV64-NEXT:    sd a4, 96(sp)
-; RV64-NEXT:    sd a3, 88(sp)
-; RV64-NEXT:    sd a2, 80(sp)
-; RV64-NEXT:    sd a1, 72(sp)
+; RV64-NEXT:    sd a5, 104(sp)
+; RV64-NEXT:    sd a6, 112(sp)
+; RV64-NEXT:    sd a7, 120(sp)
 ; RV64-NEXT:    sd a0, 64(sp)
-; RV64-NEXT:    fsd fa7, 56(sp)
-; RV64-NEXT:    fsd fa6, 48(sp)
-; RV64-NEXT:    fsd fa5, 40(sp)
+; RV64-NEXT:    sd a1, 72(sp)
+; RV64-NEXT:    sd a2, 80(sp)
+; RV64-NEXT:    sd a3, 88(sp)
 ; RV64-NEXT:    fsd fa4, 32(sp)
-; RV64-NEXT:    fsd fa3, 24(sp)
-; RV64-NEXT:    fsd fa2, 16(sp)
-; RV64-NEXT:    fsd fa1, 8(sp)
+; RV64-NEXT:    fsd fa5, 40(sp)
+; RV64-NEXT:    fsd fa6, 48(sp)
+; RV64-NEXT:    fsd fa7, 56(sp)
 ; RV64-NEXT:    fsd fa0, 0(sp)
+; RV64-NEXT:    fsd fa1, 8(sp)
+; RV64-NEXT:    fsd fa2, 16(sp)
+; RV64-NEXT:    fsd fa3, 24(sp)
 ; RV64-NEXT:    mv a0, sp
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
@@ -1056,42 +1056,42 @@ define <32 x double> @buildvec_v32f64(double %e0, double %e1, double %e2, double
 ; RV32-NEXT:    fld fs5, 104(s0)
 ; RV32-NEXT:    fld fs6, 112(s0)
 ; RV32-NEXT:    fld fs7, 120(s0)
-; RV32-NEXT:    fld fs8, 152(s0)
-; RV32-NEXT:    fld fs9, 144(s0)
-; RV32-NEXT:    fld fs10, 136(s0)
-; RV32-NEXT:    fld fs11, 128(s0)
-; RV32-NEXT:    fsd fs8, 248(sp)
-; RV32-NEXT:    fsd fs9, 240(sp)
-; RV32-NEXT:    fsd fs10, 232(sp)
-; RV32-NEXT:    fsd fs11, 224(sp)
-; RV32-NEXT:    fsd fs7, 216(sp)
-; RV32-NEXT:    fsd fs6, 208(sp)
-; RV32-NEXT:    fsd fs5, 200(sp)
+; RV32-NEXT:    fld fs8, 128(s0)
+; RV32-NEXT:    fld fs9, 136(s0)
+; RV32-NEXT:    fld fs10, 144(s0)
+; RV32-NEXT:    fld fs11, 152(s0)
+; RV32-NEXT:    fsd fs8, 224(sp)
+; RV32-NEXT:    fsd fs9, 232(sp)
+; RV32-NEXT:    fsd fs10, 240(sp)
+; RV32-NEXT:    fsd fs11, 248(sp)
 ; RV32-NEXT:    fsd fs4, 192(sp)
-; RV32-NEXT:    fsd fs3, 184(sp)
-; RV32-NEXT:    fsd fs2, 176(sp)
-; RV32-NEXT:    fsd fs1, 168(sp)
+; RV32-NEXT:    fsd fs5, 200(sp)
+; RV32-NEXT:    fsd fs6, 208(sp)
+; RV32-NEXT:    fsd fs7, 216(sp)
 ; RV32-NEXT:    fsd fs0, 160(sp)
-; RV32-NEXT:    fsd ft11, 152(sp)
-; RV32-NEXT:    fsd ft10, 144(sp)
-; RV32-NEXT:    fsd ft9, 136(sp)
+; RV32-NEXT:    fsd fs1, 168(sp)
+; RV32-NEXT:    fsd fs2, 176(sp)
+; RV32-NEXT:    fsd fs3, 184(sp)
 ; RV32-NEXT:    fsd ft8, 128(sp)
-; RV32-NEXT:    fsd ft7, 376(sp)
-; RV32-NEXT:    fsd ft6, 368(sp)
-; RV32-NEXT:    fsd ft5, 360(sp)
+; RV32-NEXT:    fsd ft9, 136(sp)
+; RV32-NEXT:    fsd ft10, 144(sp)
+; RV32-NEXT:    fsd ft11, 152(sp)
 ; RV32-NEXT:    fsd ft4, 352(sp)
-; RV32-NEXT:    fsd fa7, 312(sp)
-; RV32-NEXT:    fsd fa6, 304(sp)
-; RV32-NEXT:    fsd fa5, 296(sp)
+; RV32-NEXT:    fsd ft5, 360(sp)
+; RV32-NEXT:    fsd ft6, 368(sp)
+; RV32-NEXT:    fsd ft7, 376(sp)
 ; RV32-NEXT:    fsd fa4, 288(sp)
-; RV32-NEXT:    fsd fa3, 280(sp)
-; RV32-NEXT:    fsd fa2, 272(sp)
-; RV32-NEXT:    fsd fa1, 264(sp)
+; RV32-NEXT:    fsd fa5, 296(sp)
+; RV32-NEXT:    fsd fa6, 304(sp)
+; RV32-NEXT:    fsd fa7, 312(sp)
 ; RV32-NEXT:    fsd fa0, 256(sp)
-; RV32-NEXT:    fsd ft3, 344(sp)
-; RV32-NEXT:    fsd ft2, 336(sp)
-; RV32-NEXT:    fsd ft1, 328(sp)
+; RV32-NEXT:    fsd fa1, 264(sp)
+; RV32-NEXT:    fsd fa2, 272(sp)
+; RV32-NEXT:    fsd fa3, 280(sp)
 ; RV32-NEXT:    fsd ft0, 320(sp)
+; RV32-NEXT:    fsd ft1, 328(sp)
+; RV32-NEXT:    fsd ft2, 336(sp)
+; RV32-NEXT:    fsd ft3, 344(sp)
 ; RV32-NEXT:    addi a0, sp, 128
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vle64.v v16, (a0)
@@ -1150,38 +1150,38 @@ define <32 x double> @buildvec_v32f64(double %e0, double %e1, double %e2, double
 ; RV64-NEXT:    fld fs1, 104(s0)
 ; RV64-NEXT:    fld fs2, 112(s0)
 ; RV64-NEXT:    fld fs3, 120(s0)
-; RV64-NEXT:    sd a7, 248(sp)
-; RV64-NEXT:    sd a6, 240(sp)
-; RV64-NEXT:    sd a5, 232(sp)
 ; RV64-NEXT:    sd a4, 224(sp)
-; RV64-NEXT:    sd a3, 216(sp)
-; RV64-NEXT:    sd a2, 208(sp)
-; RV64-NEXT:    sd a1, 200(sp)
+; RV64-NEXT:    sd a5, 232(sp)
+; RV64-NEXT:    sd a6, 240(sp)
+; RV64-NEXT:    sd a7, 248(sp)
 ; RV64-NEXT:    sd a0, 192(sp)
-; RV64-NEXT:    fsd fa7, 184(sp)
-; RV64-NEXT:    fsd fa6, 176(sp)
-; RV64-NEXT:    fsd fa5, 168(sp)
+; RV64-NEXT:    sd a1, 200(sp)
+; RV64-NEXT:    sd a2, 208(sp)
+; RV64-NEXT:    sd a3, 216(sp)
 ; RV64-NEXT:    fsd fa4, 160(sp)
-; RV64-NEXT:    fsd fa3, 152(sp)
-; RV64-NEXT:    fsd fa2, 144(sp)
-; RV64-NEXT:    fsd fa1, 136(sp)
+; RV64-NEXT:    fsd fa5, 168(sp)
+; RV64-NEXT:    fsd fa6, 176(sp)
+; RV64-NEXT:    fsd fa7, 184(sp)
 ; RV64-NEXT:    fsd fa0, 128(sp)
-; RV64-NEXT:    fsd fs3, 120(sp)
-; RV64-NEXT:    fsd fs2, 112(sp)
-; RV64-NEXT:    fsd fs1, 104(sp)
+; RV64-NEXT:    fsd fa1, 136(sp)
+; RV64-NEXT:    fsd fa2, 144(sp)
+; RV64-NEXT:    fsd fa3, 152(sp)
 ; RV64-NEXT:    fsd fs0, 96(sp)
-; RV64-NEXT:    fsd ft11, 88(sp)
-; RV64-NEXT:    fsd ft10, 80(sp)
-; RV64-NEXT:    fsd ft9, 72(sp)
+; RV64-NEXT:    fsd fs1, 104(sp)
+; RV64-NEXT:    fsd fs2, 112(sp)
+; RV64-NEXT:    fsd fs3, 120(sp)
 ; RV64-NEXT:    fsd ft8, 64(sp)
-; RV64-NEXT:    fsd ft7, 56(sp)
-; RV64-NEXT:    fsd ft6, 48(sp)
-; RV64-NEXT:    fsd ft5, 40(sp)
+; RV64-NEXT:    fsd ft9, 72(sp)
+; RV64-NEXT:    fsd ft10, 80(sp)
+; RV64-NEXT:    fsd ft11, 88(sp)
 ; RV64-NEXT:    fsd ft4, 32(sp)
-; RV64-NEXT:    fsd ft3, 24(sp)
-; RV64-NEXT:    fsd ft2, 16(sp)
-; RV64-NEXT:    fsd ft1, 8(sp)
+; RV64-NEXT:    fsd ft5, 40(sp)
+; RV64-NEXT:    fsd ft6, 48(sp)
+; RV64-NEXT:    fsd ft7, 56(sp)
 ; RV64-NEXT:    fsd ft0, 0(sp)
+; RV64-NEXT:    fsd ft1, 8(sp)
+; RV64-NEXT:    fsd ft2, 16(sp)
+; RV64-NEXT:    fsd ft3, 24(sp)
 ; RV64-NEXT:    addi a0, sp, 128
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
index a4a491989c7f0211e13444ed6ede9ec71f2419eb..9d92018db2e88ea273058b4530905045a9cbaebb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
@@ -126,9 +126,6 @@ define <3 x i15> @fp2si_v3f32_v3i15(<3 x float> %x) {
 ; ZVFH32-NEXT:    vfncvt.rtz.x.f.w v9, v8
 ; ZVFH32-NEXT:    vslidedown.vi v8, v9, 2
 ; ZVFH32-NEXT:    vmv.x.s a1, v8
-; ZVFH32-NEXT:    slli a2, a1, 17
-; ZVFH32-NEXT:    srli a2, a2, 19
-; ZVFH32-NEXT:    sh a2, 4(a0)
 ; ZVFH32-NEXT:    vmv.x.s a2, v9
 ; ZVFH32-NEXT:    lui a3, 8
 ; ZVFH32-NEXT:    addi a3, a3, -1
@@ -136,11 +133,14 @@ define <3 x i15> @fp2si_v3f32_v3i15(<3 x float> %x) {
 ; ZVFH32-NEXT:    vslidedown.vi v8, v9, 1
 ; ZVFH32-NEXT:    vmv.x.s a4, v8
 ; ZVFH32-NEXT:    and a3, a4, a3
+; ZVFH32-NEXT:    slli a4, a1, 17
+; ZVFH32-NEXT:    srli a4, a4, 19
 ; ZVFH32-NEXT:    slli a3, a3, 15
 ; ZVFH32-NEXT:    slli a1, a1, 30
 ; ZVFH32-NEXT:    or a1, a2, a1
 ; ZVFH32-NEXT:    or a1, a1, a3
 ; ZVFH32-NEXT:    sw a1, 0(a0)
+; ZVFH32-NEXT:    sh a4, 4(a0)
 ; ZVFH32-NEXT:    ret
 ;
 ; ZVFH64-LABEL: fp2si_v3f32_v3i15:
@@ -160,10 +160,10 @@ define <3 x i15> @fp2si_v3f32_v3i15(<3 x float> %x) {
 ; ZVFH64-NEXT:    slli a3, a3, 30
 ; ZVFH64-NEXT:    or a1, a1, a3
 ; ZVFH64-NEXT:    or a1, a1, a2
+; ZVFH64-NEXT:    slli a2, a1, 19
+; ZVFH64-NEXT:    srli a2, a2, 51
 ; ZVFH64-NEXT:    sw a1, 0(a0)
-; ZVFH64-NEXT:    slli a1, a1, 19
-; ZVFH64-NEXT:    srli a1, a1, 51
-; ZVFH64-NEXT:    sh a1, 4(a0)
+; ZVFH64-NEXT:    sh a2, 4(a0)
 ; ZVFH64-NEXT:    ret
 ;
 ; ZVFHMIN32-LABEL: fp2si_v3f32_v3i15:
@@ -172,9 +172,6 @@ define <3 x i15> @fp2si_v3f32_v3i15(<3 x float> %x) {
 ; ZVFHMIN32-NEXT:    vfncvt.rtz.x.f.w v9, v8
 ; ZVFHMIN32-NEXT:    vslidedown.vi v8, v9, 2
 ; ZVFHMIN32-NEXT:    vmv.x.s a1, v8
-; ZVFHMIN32-NEXT:    slli a2, a1, 17
-; ZVFHMIN32-NEXT:    srli a2, a2, 19
-; ZVFHMIN32-NEXT:    sh a2, 4(a0)
 ; ZVFHMIN32-NEXT:    vmv.x.s a2, v9
 ; ZVFHMIN32-NEXT:    lui a3, 8
 ; ZVFHMIN32-NEXT:    addi a3, a3, -1
@@ -182,11 +179,14 @@ define <3 x i15> @fp2si_v3f32_v3i15(<3 x float> %x) {
 ; ZVFHMIN32-NEXT:    vslidedown.vi v8, v9, 1
 ; ZVFHMIN32-NEXT:    vmv.x.s a4, v8
 ; ZVFHMIN32-NEXT:    and a3, a4, a3
+; ZVFHMIN32-NEXT:    slli a4, a1, 17
+; ZVFHMIN32-NEXT:    srli a4, a4, 19
 ; ZVFHMIN32-NEXT:    slli a3, a3, 15
 ; ZVFHMIN32-NEXT:    slli a1, a1, 30
 ; ZVFHMIN32-NEXT:    or a1, a2, a1
 ; ZVFHMIN32-NEXT:    or a1, a1, a3
 ; ZVFHMIN32-NEXT:    sw a1, 0(a0)
+; ZVFHMIN32-NEXT:    sh a4, 4(a0)
 ; ZVFHMIN32-NEXT:    ret
 ;
 ; ZVFHMIN64-LABEL: fp2si_v3f32_v3i15:
@@ -206,10 +206,10 @@ define <3 x i15> @fp2si_v3f32_v3i15(<3 x float> %x) {
 ; ZVFHMIN64-NEXT:    slli a3, a3, 30
 ; ZVFHMIN64-NEXT:    or a1, a1, a3
 ; ZVFHMIN64-NEXT:    or a1, a1, a2
+; ZVFHMIN64-NEXT:    slli a2, a1, 19
+; ZVFHMIN64-NEXT:    srli a2, a2, 51
 ; ZVFHMIN64-NEXT:    sw a1, 0(a0)
-; ZVFHMIN64-NEXT:    slli a1, a1, 19
-; ZVFHMIN64-NEXT:    srli a1, a1, 51
-; ZVFHMIN64-NEXT:    sh a1, 4(a0)
+; ZVFHMIN64-NEXT:    sh a2, 4(a0)
 ; ZVFHMIN64-NEXT:    ret
   %z = fptosi <3 x float> %x to <3 x i15>
   ret <3 x i15> %z
@@ -223,9 +223,6 @@ define <3 x i15> @fp2ui_v3f32_v3i15(<3 x float> %x) {
 ; ZVFH32-NEXT:    vfncvt.rtz.x.f.w v9, v8
 ; ZVFH32-NEXT:    vslidedown.vi v8, v9, 2
 ; ZVFH32-NEXT:    vmv.x.s a1, v8
-; ZVFH32-NEXT:    slli a2, a1, 17
-; ZVFH32-NEXT:    srli a2, a2, 19
-; ZVFH32-NEXT:    sh a2, 4(a0)
 ; ZVFH32-NEXT:    vmv.x.s a2, v9
 ; ZVFH32-NEXT:    lui a3, 16
 ; ZVFH32-NEXT:    addi a3, a3, -1
@@ -233,11 +230,14 @@ define <3 x i15> @fp2ui_v3f32_v3i15(<3 x float> %x) {
 ; ZVFH32-NEXT:    vslidedown.vi v8, v9, 1
 ; ZVFH32-NEXT:    vmv.x.s a4, v8
 ; ZVFH32-NEXT:    and a3, a4, a3
+; ZVFH32-NEXT:    slli a4, a1, 17
+; ZVFH32-NEXT:    srli a4, a4, 19
 ; ZVFH32-NEXT:    slli a3, a3, 15
 ; ZVFH32-NEXT:    slli a1, a1, 30
 ; ZVFH32-NEXT:    or a1, a2, a1
 ; ZVFH32-NEXT:    or a1, a1, a3
 ; ZVFH32-NEXT:    sw a1, 0(a0)
+; ZVFH32-NEXT:    sh a4, 4(a0)
 ; ZVFH32-NEXT:    ret
 ;
 ; ZVFH64-LABEL: fp2ui_v3f32_v3i15:
@@ -257,10 +257,10 @@ define <3 x i15> @fp2ui_v3f32_v3i15(<3 x float> %x) {
 ; ZVFH64-NEXT:    slli a3, a3, 30
 ; ZVFH64-NEXT:    or a1, a1, a3
 ; ZVFH64-NEXT:    or a1, a1, a2
+; ZVFH64-NEXT:    slli a2, a1, 19
+; ZVFH64-NEXT:    srli a2, a2, 51
 ; ZVFH64-NEXT:    sw a1, 0(a0)
-; ZVFH64-NEXT:    slli a1, a1, 19
-; ZVFH64-NEXT:    srli a1, a1, 51
-; ZVFH64-NEXT:    sh a1, 4(a0)
+; ZVFH64-NEXT:    sh a2, 4(a0)
 ; ZVFH64-NEXT:    ret
 ;
 ; ZVFHMIN32-LABEL: fp2ui_v3f32_v3i15:
@@ -269,9 +269,6 @@ define <3 x i15> @fp2ui_v3f32_v3i15(<3 x float> %x) {
 ; ZVFHMIN32-NEXT:    vfncvt.rtz.x.f.w v9, v8
 ; ZVFHMIN32-NEXT:    vslidedown.vi v8, v9, 2
 ; ZVFHMIN32-NEXT:    vmv.x.s a1, v8
-; ZVFHMIN32-NEXT:    slli a2, a1, 17
-; ZVFHMIN32-NEXT:    srli a2, a2, 19
-; ZVFHMIN32-NEXT:    sh a2, 4(a0)
 ; ZVFHMIN32-NEXT:    vmv.x.s a2, v9
 ; ZVFHMIN32-NEXT:    lui a3, 16
 ; ZVFHMIN32-NEXT:    addi a3, a3, -1
@@ -279,11 +276,14 @@ define <3 x i15> @fp2ui_v3f32_v3i15(<3 x float> %x) {
 ; ZVFHMIN32-NEXT:    vslidedown.vi v8, v9, 1
 ; ZVFHMIN32-NEXT:    vmv.x.s a4, v8
 ; ZVFHMIN32-NEXT:    and a3, a4, a3
+; ZVFHMIN32-NEXT:    slli a4, a1, 17
+; ZVFHMIN32-NEXT:    srli a4, a4, 19
 ; ZVFHMIN32-NEXT:    slli a3, a3, 15
 ; ZVFHMIN32-NEXT:    slli a1, a1, 30
 ; ZVFHMIN32-NEXT:    or a1, a2, a1
 ; ZVFHMIN32-NEXT:    or a1, a1, a3
 ; ZVFHMIN32-NEXT:    sw a1, 0(a0)
+; ZVFHMIN32-NEXT:    sh a4, 4(a0)
 ; ZVFHMIN32-NEXT:    ret
 ;
 ; ZVFHMIN64-LABEL: fp2ui_v3f32_v3i15:
@@ -303,10 +303,10 @@ define <3 x i15> @fp2ui_v3f32_v3i15(<3 x float> %x) {
 ; ZVFHMIN64-NEXT:    slli a3, a3, 30
 ; ZVFHMIN64-NEXT:    or a1, a1, a3
 ; ZVFHMIN64-NEXT:    or a1, a1, a2
+; ZVFHMIN64-NEXT:    slli a2, a1, 19
+; ZVFHMIN64-NEXT:    srli a2, a2, 51
 ; ZVFHMIN64-NEXT:    sw a1, 0(a0)
-; ZVFHMIN64-NEXT:    slli a1, a1, 19
-; ZVFHMIN64-NEXT:    srli a1, a1, 51
-; ZVFHMIN64-NEXT:    sh a1, 4(a0)
+; ZVFHMIN64-NEXT:    sh a2, 4(a0)
 ; ZVFHMIN64-NEXT:    ret
   %z = fptoui <3 x float> %x to <3 x i15>
   ret <3 x i15> %z
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
index 81fb86cd81cd35d36987ade050dc13836c1aa46b..7853e91ca53a391c2e89c93d2e81503af0f0deb4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
@@ -195,8 +195,8 @@ define <4 x i64> @insertelt_v4i64(<4 x i64> %a, i64 %y) {
 define void @insertelt_v4i64_store(ptr %x, i64 %y) {
 ; RV32-LABEL: insertelt_v4i64_store:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw a2, 28(a0)
 ; RV32-NEXT:    sw a1, 24(a0)
+; RV32-NEXT:    sw a2, 28(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v4i64_store:
@@ -253,8 +253,8 @@ define <3 x i64> @insertelt_v3i64(<3 x i64> %a, i64 %y) {
 define void @insertelt_v3i64_store(ptr %x, i64 %y) {
 ; RV32-LABEL: insertelt_v3i64_store:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw a2, 20(a0)
 ; RV32-NEXT:    sw a1, 16(a0)
+; RV32-NEXT:    sw a2, 20(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v3i64_store:
@@ -382,8 +382,8 @@ define void @insertelt_v8i64_0_store(ptr %x) {
 ; RV32-LABEL: insertelt_v8i64_0_store:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    li a1, -1
-; RV32-NEXT:    sw a1, 4(a0)
 ; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    sw a1, 4(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v8i64_0_store:
@@ -428,8 +428,8 @@ define void @insertelt_v8i64_store(ptr %x, i32 %idx) {
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    add a0, a0, a1
 ; RV32-NEXT:    li a1, -1
-; RV32-NEXT:    sw a1, 4(a0)
 ; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    sw a1, 4(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v8i64_store:
@@ -460,9 +460,9 @@ define <8 x i64> @insertelt_c6_v8i64_0(<8 x i64> %a, ptr %x) {
 define void @insertelt_c6_v8i64_0_store(ptr %x) {
 ; RV32-LABEL: insertelt_c6_v8i64_0_store:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    li a1, 6
 ; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_c6_v8i64_0_store:
@@ -506,9 +506,9 @@ define void @insertelt_c6_v8i64_store(ptr %x, i32 %idx) {
 ; RV32-NEXT:    andi a1, a1, 7
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    add a0, a0, a1
-; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    li a1, 6
 ; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    sw zero, 4(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_c6_v8i64_store:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index 43cee6610e7872f8be20f6f2666dba06d81b7249..7bf47d42de3b951fa9320499a516df8ce4c916f6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -310,12 +310,12 @@ define <4 x i64> @buildvec_vid_step1_add0_v4i64() {
 ; RV64ZVE32-LABEL: buildvec_vid_step1_add0_v4i64:
 ; RV64ZVE32:       # %bb.0:
 ; RV64ZVE32-NEXT:    li a1, 3
-; RV64ZVE32-NEXT:    sd a1, 24(a0)
-; RV64ZVE32-NEXT:    li a1, 2
-; RV64ZVE32-NEXT:    sd a1, 16(a0)
-; RV64ZVE32-NEXT:    li a1, 1
-; RV64ZVE32-NEXT:    sd a1, 8(a0)
+; RV64ZVE32-NEXT:    li a2, 2
+; RV64ZVE32-NEXT:    li a3, 1
 ; RV64ZVE32-NEXT:    sd zero, 0(a0)
+; RV64ZVE32-NEXT:    sd a3, 8(a0)
+; RV64ZVE32-NEXT:    sd a2, 16(a0)
+; RV64ZVE32-NEXT:    sd a1, 24(a0)
 ; RV64ZVE32-NEXT:    ret
   ret <4 x i64> <i64 0, i64 1, i64 2, i64 3>
 }
@@ -340,12 +340,12 @@ define <4 x i64> @buildvec_vid_step2_add0_v4i64() {
 ; RV64ZVE32-LABEL: buildvec_vid_step2_add0_v4i64:
 ; RV64ZVE32:       # %bb.0:
 ; RV64ZVE32-NEXT:    li a1, 6
-; RV64ZVE32-NEXT:    sd a1, 24(a0)
-; RV64ZVE32-NEXT:    li a1, 4
-; RV64ZVE32-NEXT:    sd a1, 16(a0)
-; RV64ZVE32-NEXT:    li a1, 2
-; RV64ZVE32-NEXT:    sd a1, 8(a0)
+; RV64ZVE32-NEXT:    li a2, 4
+; RV64ZVE32-NEXT:    li a3, 2
 ; RV64ZVE32-NEXT:    sd zero, 0(a0)
+; RV64ZVE32-NEXT:    sd a3, 8(a0)
+; RV64ZVE32-NEXT:    sd a2, 16(a0)
+; RV64ZVE32-NEXT:    sd a1, 24(a0)
 ; RV64ZVE32-NEXT:    ret
   ret <4 x i64> <i64 0, i64 2, i64 4, i64 6>
 }
@@ -517,8 +517,8 @@ define void @buildvec_dominant0_v2i32(ptr %x) {
 ; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI38_0)
 ; RV64ZVE32-NEXT:    ld a1, %lo(.LCPI38_0)(a1)
 ; RV64ZVE32-NEXT:    li a2, -1
-; RV64ZVE32-NEXT:    sd a2, 8(a0)
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
+; RV64ZVE32-NEXT:    sd a2, 8(a0)
 ; RV64ZVE32-NEXT:    ret
   store <2 x i64> <i64 2049638230412172402, i64 -1>, ptr %x
   ret void
@@ -548,8 +548,8 @@ define void @buildvec_dominant1_optsize_v2i32(ptr %x) optsize {
 ; RV64ZVE32-NEXT:    lui a1, %hi(.LCPI39_0)
 ; RV64ZVE32-NEXT:    ld a1, %lo(.LCPI39_0)(a1)
 ; RV64ZVE32-NEXT:    li a2, -1
-; RV64ZVE32-NEXT:    sd a2, 8(a0)
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
+; RV64ZVE32-NEXT:    sd a2, 8(a0)
 ; RV64ZVE32-NEXT:    ret
   store <2 x i64> <i64 2049638230412172402, i64 -1>, ptr %x
   ret void
@@ -1130,10 +1130,10 @@ define <4 x i64> @v4xi64_exact(i64 %a, i64 %b, i64 %c, i64 %d) vscale_range(2,2)
 ;
 ; RV64ZVE32-LABEL: v4xi64_exact:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    sd a4, 24(a0)
-; RV64ZVE32-NEXT:    sd a3, 16(a0)
-; RV64ZVE32-NEXT:    sd a2, 8(a0)
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
+; RV64ZVE32-NEXT:    sd a2, 8(a0)
+; RV64ZVE32-NEXT:    sd a3, 16(a0)
+; RV64ZVE32-NEXT:    sd a4, 24(a0)
 ; RV64ZVE32-NEXT:    ret
   %v1 = insertelement <4 x i64> poison, i64 %a, i32 0
   %v2 = insertelement <4 x i64> %v1, i64 %b, i32 1
@@ -1194,14 +1194,14 @@ define <8 x i64> @v8xi64_exact(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f, i
 ; RV64ZVE32-LABEL: v8xi64_exact:
 ; RV64ZVE32:       # %bb.0:
 ; RV64ZVE32-NEXT:    ld t0, 0(sp)
-; RV64ZVE32-NEXT:    sd t0, 56(a0)
-; RV64ZVE32-NEXT:    sd a7, 48(a0)
-; RV64ZVE32-NEXT:    sd a6, 40(a0)
 ; RV64ZVE32-NEXT:    sd a5, 32(a0)
-; RV64ZVE32-NEXT:    sd a4, 24(a0)
-; RV64ZVE32-NEXT:    sd a3, 16(a0)
-; RV64ZVE32-NEXT:    sd a2, 8(a0)
+; RV64ZVE32-NEXT:    sd a6, 40(a0)
+; RV64ZVE32-NEXT:    sd a7, 48(a0)
+; RV64ZVE32-NEXT:    sd t0, 56(a0)
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
+; RV64ZVE32-NEXT:    sd a2, 8(a0)
+; RV64ZVE32-NEXT:    sd a3, 16(a0)
+; RV64ZVE32-NEXT:    sd a4, 24(a0)
 ; RV64ZVE32-NEXT:    ret
   %v1 = insertelement <8 x i64> poison, i64 %a, i32 0
   %v2 = insertelement <8 x i64> %v1, i64 %b, i32 1
@@ -1243,14 +1243,14 @@ define <8 x i64> @v8xi64_exact_equal_halves(i64 %a, i64 %b, i64 %c, i64 %d) vsca
 ;
 ; RV64ZVE32-LABEL: v8xi64_exact_equal_halves:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    sd a4, 56(a0)
-; RV64ZVE32-NEXT:    sd a3, 48(a0)
-; RV64ZVE32-NEXT:    sd a2, 40(a0)
 ; RV64ZVE32-NEXT:    sd a1, 32(a0)
-; RV64ZVE32-NEXT:    sd a4, 24(a0)
-; RV64ZVE32-NEXT:    sd a3, 16(a0)
-; RV64ZVE32-NEXT:    sd a2, 8(a0)
+; RV64ZVE32-NEXT:    sd a2, 40(a0)
+; RV64ZVE32-NEXT:    sd a3, 48(a0)
+; RV64ZVE32-NEXT:    sd a4, 56(a0)
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
+; RV64ZVE32-NEXT:    sd a2, 8(a0)
+; RV64ZVE32-NEXT:    sd a3, 16(a0)
+; RV64ZVE32-NEXT:    sd a4, 24(a0)
 ; RV64ZVE32-NEXT:    ret
   %v1 = insertelement <8 x i64> poison, i64 %a, i32 0
   %v2 = insertelement <8 x i64> %v1, i64 %b, i32 1
@@ -1288,10 +1288,10 @@ define <8 x i64> @v8xi64_exact_undef_suffix(i64 %a, i64 %b, i64 %c, i64 %d) vsca
 ;
 ; RV64ZVE32-LABEL: v8xi64_exact_undef_suffix:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    sd a4, 24(a0)
-; RV64ZVE32-NEXT:    sd a3, 16(a0)
-; RV64ZVE32-NEXT:    sd a2, 8(a0)
 ; RV64ZVE32-NEXT:    sd a1, 0(a0)
+; RV64ZVE32-NEXT:    sd a2, 8(a0)
+; RV64ZVE32-NEXT:    sd a3, 16(a0)
+; RV64ZVE32-NEXT:    sd a4, 24(a0)
 ; RV64ZVE32-NEXT:    ret
   %v1 = insertelement <8 x i64> poison, i64 %a, i32 0
   %v2 = insertelement <8 x i64> %v1, i64 %b, i32 1
@@ -1325,10 +1325,10 @@ define <8 x i64> @v8xi64_exact_undef_prefix(i64 %a, i64 %b, i64 %c, i64 %d) vsca
 ;
 ; RV64ZVE32-LABEL: v8xi64_exact_undef_prefix:
 ; RV64ZVE32:       # %bb.0:
-; RV64ZVE32-NEXT:    sd a4, 56(a0)
-; RV64ZVE32-NEXT:    sd a3, 48(a0)
-; RV64ZVE32-NEXT:    sd a2, 40(a0)
 ; RV64ZVE32-NEXT:    sd a1, 32(a0)
+; RV64ZVE32-NEXT:    sd a2, 40(a0)
+; RV64ZVE32-NEXT:    sd a3, 48(a0)
+; RV64ZVE32-NEXT:    sd a4, 56(a0)
 ; RV64ZVE32-NEXT:    ret
   %v1 = insertelement <8 x i64> poison, i64 %a, i32 4
   %v2 = insertelement <8 x i64> %v1, i64 %b, i32 5
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll
index 336a64b1b89ca838e936961d8206811e741b13f8..f6354d3e2f619a728b4115291fc42b32d17dfdd0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll
@@ -46,8 +46,8 @@ define void @splat_v2i64(ptr %x, i64 %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a1), zero
@@ -112,8 +112,8 @@ define void @splat_v4i64(ptr %x, i64 %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a1), zero
@@ -405,8 +405,8 @@ define void @vadd_vx_v16i64(ptr %a, i64 %b, ptr %c) {
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
 ; RV32-NEXT:    vadd.vv v8, v8, v16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
index 70bda8c2da0f27e1c9faee2380718171f43d1e7f..f9992a5de793d07f28fd6668e07fe0984d05ae9a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
@@ -5527,9 +5527,9 @@ define void @mulhu_vx_v2i64(ptr %x) {
 ; RV32-NEXT:    vle64.v v8, (a0)
 ; RV32-NEXT:    lui a1, 699051
 ; RV32-NEXT:    addi a2, a1, -1366
-; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    addi a1, a1, -1365
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vlse64.v v9, (a1), zero
 ; RV32-NEXT:    vmulhu.vv v8, v8, v9
@@ -5632,9 +5632,9 @@ define void @mulhs_vx_v2i64(ptr %x) {
 ; RV32-NEXT:    vle64.v v8, (a0)
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a2, a1, 1365
-; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    addi a1, a1, 1366
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vlse64.v v9, (a1), zero
 ; RV32-NEXT:    vmulh.vv v8, v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
index 805b371f1e3d56b06e1409562013452d14fa92a5..7f1493544eabcfecdf97ccaaf743c8675ecec0e2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
@@ -14,8 +14,8 @@ define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a0, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    mv a0, sp
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -332,64 +332,64 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 68(sp)
 ; RV32-NEXT:    sw a0, 64(sp)
+; RV32-NEXT:    sw a1, 68(sp)
 ; RV32-NEXT:    addi a0, sp, 192
 ; RV32-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 7
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 124(sp)
 ; RV32-NEXT:    sw a0, 120(sp)
+; RV32-NEXT:    sw a1, 124(sp)
 ; RV32-NEXT:    addi a0, sp, 192
 ; RV32-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 6
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 116(sp)
 ; RV32-NEXT:    sw a0, 112(sp)
+; RV32-NEXT:    sw a1, 116(sp)
 ; RV32-NEXT:    addi a0, sp, 192
 ; RV32-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 5
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 108(sp)
 ; RV32-NEXT:    sw a0, 104(sp)
+; RV32-NEXT:    sw a1, 108(sp)
 ; RV32-NEXT:    addi a0, sp, 192
 ; RV32-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 4
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 100(sp)
 ; RV32-NEXT:    sw a0, 96(sp)
+; RV32-NEXT:    sw a1, 100(sp)
 ; RV32-NEXT:    addi a0, sp, 192
 ; RV32-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 3
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 92(sp)
 ; RV32-NEXT:    sw a0, 88(sp)
+; RV32-NEXT:    sw a1, 92(sp)
 ; RV32-NEXT:    addi a0, sp, 192
 ; RV32-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 84(sp)
 ; RV32-NEXT:    sw a0, 80(sp)
+; RV32-NEXT:    sw a1, 84(sp)
 ; RV32-NEXT:    addi a0, sp, 192
 ; RV32-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 76(sp)
 ; RV32-NEXT:    sw a0, 72(sp)
+; RV32-NEXT:    sw a1, 76(sp)
 ; RV32-NEXT:    addi a0, sp, 64
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
@@ -413,36 +413,36 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64-NEXT:    vfmv.f.s fa5, v8
 ; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 0(sp)
 ; RV64-NEXT:    vslidedown.vi v10, v8, 7
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 56(sp)
+; RV64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-NEXT:    vslidedown.vi v10, v8, 6
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 48(sp)
+; RV64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-NEXT:    vslidedown.vi v10, v8, 5
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 40(sp)
+; RV64-NEXT:    fcvt.l.s a3, fa5
 ; RV64-NEXT:    vslidedown.vi v10, v8, 4
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 32(sp)
+; RV64-NEXT:    fcvt.l.s a4, fa5
+; RV64-NEXT:    sd a4, 32(sp)
+; RV64-NEXT:    sd a3, 40(sp)
+; RV64-NEXT:    sd a2, 48(sp)
+; RV64-NEXT:    sd a1, 56(sp)
 ; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v9, v8, 3
 ; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 24(sp)
+; RV64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-NEXT:    vslidedown.vi v9, v8, 2
 ; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 16(sp)
+; RV64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 8(sp)
+; RV64-NEXT:    fcvt.l.s a3, fa5
+; RV64-NEXT:    sd a0, 0(sp)
+; RV64-NEXT:    sd a3, 8(sp)
+; RV64-NEXT:    sd a2, 16(sp)
+; RV64-NEXT:    sd a1, 24(sp)
 ; RV64-NEXT:    mv a0, sp
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
@@ -478,99 +478,99 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; RV32-NEXT:    vse32.v v8, (a0)
 ; RV32-NEXT:    flw fa0, 124(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 252(sp)
 ; RV32-NEXT:    sw a0, 248(sp)
+; RV32-NEXT:    sw a1, 252(sp)
 ; RV32-NEXT:    flw fa0, 120(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 244(sp)
 ; RV32-NEXT:    sw a0, 240(sp)
+; RV32-NEXT:    sw a1, 244(sp)
 ; RV32-NEXT:    flw fa0, 116(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 236(sp)
 ; RV32-NEXT:    sw a0, 232(sp)
+; RV32-NEXT:    sw a1, 236(sp)
 ; RV32-NEXT:    flw fa0, 112(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 228(sp)
 ; RV32-NEXT:    sw a0, 224(sp)
+; RV32-NEXT:    sw a1, 228(sp)
 ; RV32-NEXT:    flw fa0, 108(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 220(sp)
 ; RV32-NEXT:    sw a0, 216(sp)
+; RV32-NEXT:    sw a1, 220(sp)
 ; RV32-NEXT:    flw fa0, 104(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 212(sp)
 ; RV32-NEXT:    sw a0, 208(sp)
+; RV32-NEXT:    sw a1, 212(sp)
 ; RV32-NEXT:    flw fa0, 100(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 204(sp)
 ; RV32-NEXT:    sw a0, 200(sp)
+; RV32-NEXT:    sw a1, 204(sp)
 ; RV32-NEXT:    flw fa0, 96(sp)
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 196(sp)
 ; RV32-NEXT:    sw a0, 192(sp)
+; RV32-NEXT:    sw a1, 196(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 132(sp)
 ; RV32-NEXT:    sw a0, 128(sp)
+; RV32-NEXT:    sw a1, 132(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 3
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 156(sp)
 ; RV32-NEXT:    sw a0, 152(sp)
+; RV32-NEXT:    sw a1, 156(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 148(sp)
 ; RV32-NEXT:    sw a0, 144(sp)
+; RV32-NEXT:    sw a1, 148(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 140(sp)
 ; RV32-NEXT:    sw a0, 136(sp)
+; RV32-NEXT:    sw a1, 140(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 7
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 188(sp)
 ; RV32-NEXT:    sw a0, 184(sp)
+; RV32-NEXT:    sw a1, 188(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 6
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 180(sp)
 ; RV32-NEXT:    sw a0, 176(sp)
+; RV32-NEXT:    sw a1, 180(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 5
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 172(sp)
 ; RV32-NEXT:    sw a0, 168(sp)
+; RV32-NEXT:    sw a1, 172(sp)
 ; RV32-NEXT:    addi a0, sp, 384
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 4
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a1, 164(sp)
 ; RV32-NEXT:    sw a0, 160(sp)
+; RV32-NEXT:    sw a1, 164(sp)
 ; RV32-NEXT:    li a0, 32
 ; RV32-NEXT:    addi a1, sp, 128
 ; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
@@ -621,37 +621,37 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; RV64-NEXT:    sd a0, 192(sp)
 ; RV64-NEXT:    vfmv.f.s fa5, v8
 ; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 128(sp)
 ; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v10, v8, 3
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 152(sp)
+; RV64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-NEXT:    vslidedown.vi v10, v8, 2
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 144(sp)
+; RV64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-NEXT:    vslidedown.vi v10, v8, 1
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 136(sp)
+; RV64-NEXT:    fcvt.l.s a3, fa5
+; RV64-NEXT:    sd a0, 128(sp)
+; RV64-NEXT:    sd a3, 136(sp)
+; RV64-NEXT:    sd a2, 144(sp)
+; RV64-NEXT:    sd a1, 152(sp)
 ; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64-NEXT:    vslidedown.vi v10, v8, 7
 ; RV64-NEXT:    vfmv.f.s fa5, v10
 ; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 184(sp)
 ; RV64-NEXT:    vslidedown.vi v10, v8, 6
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 176(sp)
+; RV64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-NEXT:    vslidedown.vi v10, v8, 5
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 168(sp)
+; RV64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 160(sp)
+; RV64-NEXT:    fcvt.l.s a3, fa5
+; RV64-NEXT:    sd a3, 160(sp)
+; RV64-NEXT:    sd a2, 168(sp)
+; RV64-NEXT:    sd a1, 176(sp)
+; RV64-NEXT:    sd a0, 184(sp)
 ; RV64-NEXT:    addi a0, sp, 128
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
@@ -675,8 +675,8 @@ define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    sw a0, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
 ; RV32-NEXT:    mv a0, sp
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -879,51 +879,51 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 ; RV32-NEXT:    vse64.v v8, (a0)
 ; RV32-NEXT:    fld fa0, 120(sp)
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 188(sp)
 ; RV32-NEXT:    sw a0, 184(sp)
+; RV32-NEXT:    sw a1, 188(sp)
 ; RV32-NEXT:    fld fa0, 112(sp)
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 180(sp)
 ; RV32-NEXT:    sw a0, 176(sp)
+; RV32-NEXT:    sw a1, 180(sp)
 ; RV32-NEXT:    fld fa0, 104(sp)
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 172(sp)
 ; RV32-NEXT:    sw a0, 168(sp)
+; RV32-NEXT:    sw a1, 172(sp)
 ; RV32-NEXT:    fld fa0, 96(sp)
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 164(sp)
 ; RV32-NEXT:    sw a0, 160(sp)
+; RV32-NEXT:    sw a1, 164(sp)
 ; RV32-NEXT:    addi a0, sp, 256
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 132(sp)
 ; RV32-NEXT:    sw a0, 128(sp)
+; RV32-NEXT:    sw a1, 132(sp)
 ; RV32-NEXT:    addi a0, sp, 256
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 140(sp)
 ; RV32-NEXT:    sw a0, 136(sp)
+; RV32-NEXT:    sw a1, 140(sp)
 ; RV32-NEXT:    addi a0, sp, 256
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 3
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 156(sp)
 ; RV32-NEXT:    sw a0, 152(sp)
+; RV32-NEXT:    sw a1, 156(sp)
 ; RV32-NEXT:    addi a0, sp, 256
 ; RV32-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32-NEXT:    vfmv.f.s fa0, v8
 ; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a1, 148(sp)
 ; RV32-NEXT:    sw a0, 144(sp)
+; RV32-NEXT:    sw a1, 148(sp)
 ; RV32-NEXT:    addi a0, sp, 128
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
@@ -961,21 +961,21 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 ; RV64-NEXT:    sd a0, 96(sp)
 ; RV64-NEXT:    vfmv.f.s fa5, v8
 ; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    sd a0, 64(sp)
 ; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v10, v8, 1
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    sd a0, 72(sp)
+; RV64-NEXT:    fcvt.l.d a1, fa5
 ; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
 ; RV64-NEXT:    vslidedown.vi v10, v8, 3
 ; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    sd a0, 88(sp)
+; RV64-NEXT:    fcvt.l.d a2, fa5
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    sd a0, 80(sp)
+; RV64-NEXT:    fcvt.l.d a3, fa5
+; RV64-NEXT:    sd a0, 64(sp)
+; RV64-NEXT:    sd a1, 72(sp)
+; RV64-NEXT:    sd a3, 80(sp)
+; RV64-NEXT:    sd a2, 88(sp)
 ; RV64-NEXT:    addi a0, sp, 64
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
index a28d9d993dbbed72fd8f516854101ff6fe1b701f..3ce649409adb402464b4895dd770d9c04998f672 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
@@ -315,36 +315,36 @@ define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
 ; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v8
 ; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 0(sp)
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 56(sp)
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 6
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 48(sp)
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 40(sp)
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 4
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 32(sp)
+; RV64-i64-NEXT:    fcvt.l.s a4, fa5
+; RV64-i64-NEXT:    sd a4, 32(sp)
+; RV64-i64-NEXT:    sd a3, 40(sp)
+; RV64-i64-NEXT:    sd a2, 48(sp)
+; RV64-i64-NEXT:    sd a1, 56(sp)
 ; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-i64-NEXT:    vslidedown.vi v9, v8, 3
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 24(sp)
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v9, v8, 2
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 16(sp)
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 8(sp)
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5
+; RV64-i64-NEXT:    sd a0, 0(sp)
+; RV64-i64-NEXT:    sd a3, 8(sp)
+; RV64-i64-NEXT:    sd a2, 16(sp)
+; RV64-i64-NEXT:    sd a1, 24(sp)
 ; RV64-i64-NEXT:    mv a0, sp
 ; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-i64-NEXT:    vle64.v v8, (a0)
@@ -399,37 +399,37 @@ define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 ; RV32-NEXT:    sw a0, 96(sp)
 ; RV32-NEXT:    vfmv.f.s fa5, v8
 ; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 64(sp)
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v10, v8, 3
 ; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 76(sp)
+; RV32-NEXT:    fcvt.w.s a1, fa5
 ; RV32-NEXT:    vslidedown.vi v10, v8, 2
 ; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 72(sp)
+; RV32-NEXT:    fcvt.w.s a2, fa5
 ; RV32-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 68(sp)
+; RV32-NEXT:    fcvt.w.s a3, fa5
+; RV32-NEXT:    sw a0, 64(sp)
+; RV32-NEXT:    sw a3, 68(sp)
+; RV32-NEXT:    sw a2, 72(sp)
+; RV32-NEXT:    sw a1, 76(sp)
 ; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v10, v8, 7
 ; RV32-NEXT:    vfmv.f.s fa5, v10
 ; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 92(sp)
 ; RV32-NEXT:    vslidedown.vi v10, v8, 6
 ; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 88(sp)
+; RV32-NEXT:    fcvt.w.s a1, fa5
 ; RV32-NEXT:    vslidedown.vi v10, v8, 5
 ; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 84(sp)
+; RV32-NEXT:    fcvt.w.s a2, fa5
 ; RV32-NEXT:    vslidedown.vi v8, v8, 4
 ; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 80(sp)
+; RV32-NEXT:    fcvt.w.s a3, fa5
+; RV32-NEXT:    sw a3, 80(sp)
+; RV32-NEXT:    sw a2, 84(sp)
+; RV32-NEXT:    sw a1, 88(sp)
+; RV32-NEXT:    sw a0, 92(sp)
 ; RV32-NEXT:    addi a0, sp, 64
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
@@ -479,37 +479,37 @@ define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 ; RV64-i32-NEXT:    sw a0, 96(sp)
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v8
 ; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 64(sp)
 ; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-i32-NEXT:    vslidedown.vi v10, v8, 3
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 76(sp)
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5
 ; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 72(sp)
+; RV64-i32-NEXT:    fcvt.w.s a2, fa5
 ; RV64-i32-NEXT:    vslidedown.vi v10, v8, 1
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 68(sp)
+; RV64-i32-NEXT:    fcvt.w.s a3, fa5
+; RV64-i32-NEXT:    sw a0, 64(sp)
+; RV64-i32-NEXT:    sw a3, 68(sp)
+; RV64-i32-NEXT:    sw a2, 72(sp)
+; RV64-i32-NEXT:    sw a1, 76(sp)
 ; RV64-i32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64-i32-NEXT:    vslidedown.vi v10, v8, 7
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v10
 ; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 92(sp)
 ; RV64-i32-NEXT:    vslidedown.vi v10, v8, 6
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 88(sp)
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5
 ; RV64-i32-NEXT:    vslidedown.vi v10, v8, 5
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 84(sp)
+; RV64-i32-NEXT:    fcvt.w.s a2, fa5
 ; RV64-i32-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.w.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 80(sp)
+; RV64-i32-NEXT:    fcvt.w.s a3, fa5
+; RV64-i32-NEXT:    sw a3, 80(sp)
+; RV64-i32-NEXT:    sw a2, 84(sp)
+; RV64-i32-NEXT:    sw a1, 88(sp)
+; RV64-i32-NEXT:    sw a0, 92(sp)
 ; RV64-i32-NEXT:    addi a0, sp, 64
 ; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV64-i32-NEXT:    vle32.v v8, (a0)
@@ -559,37 +559,37 @@ define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 ; RV64-i64-NEXT:    sd a0, 192(sp)
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v8
 ; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 128(sp)
 ; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 152(sp)
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 144(sp)
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 1
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 136(sp)
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5
+; RV64-i64-NEXT:    sd a0, 128(sp)
+; RV64-i64-NEXT:    sd a3, 136(sp)
+; RV64-i64-NEXT:    sd a2, 144(sp)
+; RV64-i64-NEXT:    sd a1, 152(sp)
 ; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
 ; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 184(sp)
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 6
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 176(sp)
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 168(sp)
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 160(sp)
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5
+; RV64-i64-NEXT:    sd a3, 160(sp)
+; RV64-i64-NEXT:    sd a2, 168(sp)
+; RV64-i64-NEXT:    sd a1, 176(sp)
+; RV64-i64-NEXT:    sd a0, 184(sp)
 ; RV64-i64-NEXT:    addi a0, sp, 128
 ; RV64-i64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-i64-NEXT:    vle64.v v8, (a0)
@@ -880,21 +880,21 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 ; RV64-i64-NEXT:    sd a0, 96(sp)
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v8
 ; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    sd a0, 64(sp)
 ; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 1
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    sd a0, 72(sp)
+; RV64-i64-NEXT:    fcvt.l.d a1, fa5
 ; RV64-i64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
 ; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    sd a0, 88(sp)
+; RV64-i64-NEXT:    fcvt.l.d a2, fa5
 ; RV64-i64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    sd a0, 80(sp)
+; RV64-i64-NEXT:    fcvt.l.d a3, fa5
+; RV64-i64-NEXT:    sd a0, 64(sp)
+; RV64-i64-NEXT:    sd a1, 72(sp)
+; RV64-i64-NEXT:    sd a3, 80(sp)
+; RV64-i64-NEXT:    sd a2, 88(sp)
 ; RV64-i64-NEXT:    addi a0, sp, 64
 ; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-i64-NEXT:    vle64.v v8, (a0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 9cd38056364494159473beea3dbff3af75fab560..9c6ec6aef6034764b1a2d964dff7feaffce9e1f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -341,8 +341,8 @@ define <2 x i64> @mgather_v2i8_sextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2 x
 ; RV32ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV32ZVE32F-NEXT:    srai a4, a3, 31
 ; RV32ZVE32F-NEXT:    sw a3, 0(a0)
-; RV32ZVE32F-NEXT:    sw a1, 8(a0)
 ; RV32ZVE32F-NEXT:    sw a4, 4(a0)
+; RV32ZVE32F-NEXT:    sw a1, 8(a0)
 ; RV32ZVE32F-NEXT:    sw a2, 12(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
@@ -402,10 +402,10 @@ define <2 x i64> @mgather_v2i8_zextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2 x
 ; RV32ZVE32F-NEXT:    andi a1, a1, 255
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV32ZVE32F-NEXT:    andi a2, a2, 255
-; RV32ZVE32F-NEXT:    sw zero, 12(a0)
-; RV32ZVE32F-NEXT:    sw zero, 4(a0)
 ; RV32ZVE32F-NEXT:    sw a2, 0(a0)
+; RV32ZVE32F-NEXT:    sw zero, 4(a0)
 ; RV32ZVE32F-NEXT:    sw a1, 8(a0)
+; RV32ZVE32F-NEXT:    sw zero, 12(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_v2i8_zextload_v2i64:
@@ -1034,8 +1034,8 @@ define <2 x i64> @mgather_v2i16_sextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV32ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV32ZVE32F-NEXT:    srai a4, a3, 31
 ; RV32ZVE32F-NEXT:    sw a3, 0(a0)
-; RV32ZVE32F-NEXT:    sw a1, 8(a0)
 ; RV32ZVE32F-NEXT:    sw a4, 4(a0)
+; RV32ZVE32F-NEXT:    sw a1, 8(a0)
 ; RV32ZVE32F-NEXT:    sw a2, 12(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
@@ -1097,10 +1097,10 @@ define <2 x i64> @mgather_v2i16_zextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV32ZVE32F-NEXT:    and a1, a1, a2
 ; RV32ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV32ZVE32F-NEXT:    and a2, a3, a2
-; RV32ZVE32F-NEXT:    sw zero, 12(a0)
-; RV32ZVE32F-NEXT:    sw zero, 4(a0)
 ; RV32ZVE32F-NEXT:    sw a2, 0(a0)
+; RV32ZVE32F-NEXT:    sw zero, 4(a0)
 ; RV32ZVE32F-NEXT:    sw a1, 8(a0)
+; RV32ZVE32F-NEXT:    sw zero, 12(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_v2i16_zextload_v2i64:
@@ -2154,8 +2154,8 @@ define <2 x i64> @mgather_v2i32_zextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
-; RV32ZVE32F-NEXT:    sw zero, 12(a0)
 ; RV32ZVE32F-NEXT:    sw zero, 4(a0)
+; RV32ZVE32F-NEXT:    sw zero, 12(a0)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vse32.v v9, (a0)
 ; RV32ZVE32F-NEXT:    addi a0, a0, 8
@@ -3775,36 +3775,36 @@ define <4 x i64> @mgather_truemask_v4i64(<4 x ptr> %ptrs, <4 x i64> %passthru) {
 ; RV32ZVE32F-NEXT:    lw a3, 4(a3)
 ; RV32ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
 ; RV32ZVE32F-NEXT:    vmv.x.s a5, v9
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a6, v8
-; RV32ZVE32F-NEXT:    lw a7, 0(a6)
-; RV32ZVE32F-NEXT:    lw a6, 4(a6)
-; RV32ZVE32F-NEXT:    lw t0, 0(a5)
+; RV32ZVE32F-NEXT:    lw a6, 0(a5)
 ; RV32ZVE32F-NEXT:    lw a5, 4(a5)
-; RV32ZVE32F-NEXT:    sw a1, 4(a0)
-; RV32ZVE32F-NEXT:    sw a2, 0(a0)
-; RV32ZVE32F-NEXT:    sw a6, 28(a0)
-; RV32ZVE32F-NEXT:    sw a7, 24(a0)
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 3
+; RV32ZVE32F-NEXT:    vmv.x.s a7, v8
+; RV32ZVE32F-NEXT:    lw t0, 0(a7)
+; RV32ZVE32F-NEXT:    lw a7, 4(a7)
+; RV32ZVE32F-NEXT:    sw a6, 16(a0)
 ; RV32ZVE32F-NEXT:    sw a5, 20(a0)
-; RV32ZVE32F-NEXT:    sw t0, 16(a0)
-; RV32ZVE32F-NEXT:    sw a3, 12(a0)
+; RV32ZVE32F-NEXT:    sw t0, 24(a0)
+; RV32ZVE32F-NEXT:    sw a7, 28(a0)
+; RV32ZVE32F-NEXT:    sw a2, 0(a0)
+; RV32ZVE32F-NEXT:    sw a1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw a4, 8(a0)
+; RV32ZVE32F-NEXT:    sw a3, 12(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_truemask_v4i64:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    ld a2, 24(a1)
-; RV64ZVE32F-NEXT:    ld a3, 16(a1)
-; RV64ZVE32F-NEXT:    ld a4, 8(a1)
-; RV64ZVE32F-NEXT:    ld a1, 0(a1)
+; RV64ZVE32F-NEXT:    ld a2, 0(a1)
+; RV64ZVE32F-NEXT:    ld a3, 8(a1)
+; RV64ZVE32F-NEXT:    ld a4, 16(a1)
+; RV64ZVE32F-NEXT:    ld a1, 24(a1)
 ; RV64ZVE32F-NEXT:    ld a2, 0(a2)
 ; RV64ZVE32F-NEXT:    ld a3, 0(a3)
 ; RV64ZVE32F-NEXT:    ld a4, 0(a4)
 ; RV64ZVE32F-NEXT:    ld a1, 0(a1)
-; RV64ZVE32F-NEXT:    sd a2, 24(a0)
-; RV64ZVE32F-NEXT:    sd a3, 16(a0)
-; RV64ZVE32F-NEXT:    sd a4, 8(a0)
-; RV64ZVE32F-NEXT:    sd a1, 0(a0)
+; RV64ZVE32F-NEXT:    sd a2, 0(a0)
+; RV64ZVE32F-NEXT:    sd a3, 8(a0)
+; RV64ZVE32F-NEXT:    sd a4, 16(a0)
+; RV64ZVE32F-NEXT:    sd a1, 24(a0)
 ; RV64ZVE32F-NEXT:    ret
   %v = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> %ptrs, i32 8, <4 x i1> splat (i1 1), <4 x i64> %passthru)
   ret <4 x i64> %v
@@ -3823,34 +3823,34 @@ define <4 x i64> @mgather_falsemask_v4i64(<4 x ptr> %ptrs, <4 x i64> %passthru)
 ;
 ; RV32ZVE32F-LABEL: mgather_falsemask_v4i64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    lw a2, 16(a1)
-; RV32ZVE32F-NEXT:    lw a3, 20(a1)
-; RV32ZVE32F-NEXT:    lw a4, 24(a1)
-; RV32ZVE32F-NEXT:    lw a5, 28(a1)
-; RV32ZVE32F-NEXT:    lw a6, 0(a1)
-; RV32ZVE32F-NEXT:    lw a7, 4(a1)
-; RV32ZVE32F-NEXT:    lw t0, 8(a1)
-; RV32ZVE32F-NEXT:    lw a1, 12(a1)
-; RV32ZVE32F-NEXT:    sw a5, 28(a0)
-; RV32ZVE32F-NEXT:    sw a4, 24(a0)
-; RV32ZVE32F-NEXT:    sw a3, 20(a0)
-; RV32ZVE32F-NEXT:    sw a2, 16(a0)
-; RV32ZVE32F-NEXT:    sw a1, 12(a0)
-; RV32ZVE32F-NEXT:    sw t0, 8(a0)
-; RV32ZVE32F-NEXT:    sw a7, 4(a0)
-; RV32ZVE32F-NEXT:    sw a6, 0(a0)
+; RV32ZVE32F-NEXT:    lw a2, 0(a1)
+; RV32ZVE32F-NEXT:    lw a3, 4(a1)
+; RV32ZVE32F-NEXT:    lw a4, 8(a1)
+; RV32ZVE32F-NEXT:    lw a5, 12(a1)
+; RV32ZVE32F-NEXT:    lw a6, 16(a1)
+; RV32ZVE32F-NEXT:    lw a7, 20(a1)
+; RV32ZVE32F-NEXT:    lw t0, 24(a1)
+; RV32ZVE32F-NEXT:    lw a1, 28(a1)
+; RV32ZVE32F-NEXT:    sw a6, 16(a0)
+; RV32ZVE32F-NEXT:    sw a7, 20(a0)
+; RV32ZVE32F-NEXT:    sw t0, 24(a0)
+; RV32ZVE32F-NEXT:    sw a1, 28(a0)
+; RV32ZVE32F-NEXT:    sw a2, 0(a0)
+; RV32ZVE32F-NEXT:    sw a3, 4(a0)
+; RV32ZVE32F-NEXT:    sw a4, 8(a0)
+; RV32ZVE32F-NEXT:    sw a5, 12(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_falsemask_v4i64:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    ld a1, 24(a2)
-; RV64ZVE32F-NEXT:    ld a3, 16(a2)
-; RV64ZVE32F-NEXT:    ld a4, 8(a2)
-; RV64ZVE32F-NEXT:    ld a2, 0(a2)
-; RV64ZVE32F-NEXT:    sd a1, 24(a0)
-; RV64ZVE32F-NEXT:    sd a3, 16(a0)
-; RV64ZVE32F-NEXT:    sd a4, 8(a0)
-; RV64ZVE32F-NEXT:    sd a2, 0(a0)
+; RV64ZVE32F-NEXT:    ld a1, 0(a2)
+; RV64ZVE32F-NEXT:    ld a3, 8(a2)
+; RV64ZVE32F-NEXT:    ld a4, 16(a2)
+; RV64ZVE32F-NEXT:    ld a2, 24(a2)
+; RV64ZVE32F-NEXT:    sd a1, 0(a0)
+; RV64ZVE32F-NEXT:    sd a3, 8(a0)
+; RV64ZVE32F-NEXT:    sd a4, 16(a0)
+; RV64ZVE32F-NEXT:    sd a2, 24(a0)
 ; RV64ZVE32F-NEXT:    ret
   %v = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> %ptrs, i32 8, <4 x i1> zeroinitializer, <4 x i64> %passthru)
   ret <4 x i64> %v
@@ -9433,35 +9433,35 @@ define <4 x double> @mgather_truemask_v4f64(<4 x ptr> %ptrs, <4 x double> %passt
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    fld fa5, 0(a1)
-; RV32ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV32ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV32ZVE32F-NEXT:    fld fa4, 0(a1)
+; RV32ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v9
+; RV32ZVE32F-NEXT:    fld fa3, 0(a1)
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 3
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV32ZVE32F-NEXT:    fld fa3, 0(a1)
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV32ZVE32F-NEXT:    fld fa2, 0(a1)
 ; RV32ZVE32F-NEXT:    fsd fa5, 0(a0)
-; RV32ZVE32F-NEXT:    fsd fa3, 24(a0)
-; RV32ZVE32F-NEXT:    fsd fa4, 16(a0)
-; RV32ZVE32F-NEXT:    fsd fa2, 8(a0)
+; RV32ZVE32F-NEXT:    fsd fa4, 8(a0)
+; RV32ZVE32F-NEXT:    fsd fa3, 16(a0)
+; RV32ZVE32F-NEXT:    fsd fa2, 24(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_truemask_v4f64:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    ld a2, 24(a1)
-; RV64ZVE32F-NEXT:    ld a3, 16(a1)
-; RV64ZVE32F-NEXT:    ld a4, 8(a1)
-; RV64ZVE32F-NEXT:    ld a1, 0(a1)
+; RV64ZVE32F-NEXT:    ld a2, 0(a1)
+; RV64ZVE32F-NEXT:    ld a3, 8(a1)
+; RV64ZVE32F-NEXT:    ld a4, 16(a1)
+; RV64ZVE32F-NEXT:    ld a1, 24(a1)
 ; RV64ZVE32F-NEXT:    fld fa5, 0(a2)
 ; RV64ZVE32F-NEXT:    fld fa4, 0(a3)
 ; RV64ZVE32F-NEXT:    fld fa3, 0(a4)
 ; RV64ZVE32F-NEXT:    fld fa2, 0(a1)
-; RV64ZVE32F-NEXT:    fsd fa5, 24(a0)
-; RV64ZVE32F-NEXT:    fsd fa4, 16(a0)
-; RV64ZVE32F-NEXT:    fsd fa3, 8(a0)
-; RV64ZVE32F-NEXT:    fsd fa2, 0(a0)
+; RV64ZVE32F-NEXT:    fsd fa5, 0(a0)
+; RV64ZVE32F-NEXT:    fsd fa4, 8(a0)
+; RV64ZVE32F-NEXT:    fsd fa3, 16(a0)
+; RV64ZVE32F-NEXT:    fsd fa2, 24(a0)
 ; RV64ZVE32F-NEXT:    ret
   %v = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %ptrs, i32 8, <4 x i1> splat (i1 1), <4 x double> %passthru)
   ret <4 x double> %v
@@ -9480,18 +9480,18 @@ define <4 x double> @mgather_falsemask_v4f64(<4 x ptr> %ptrs, <4 x double> %pass
 ;
 ; RV32ZVE32F-LABEL: mgather_falsemask_v4f64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    fsd fa3, 24(a0)
-; RV32ZVE32F-NEXT:    fsd fa2, 16(a0)
-; RV32ZVE32F-NEXT:    fsd fa1, 8(a0)
 ; RV32ZVE32F-NEXT:    fsd fa0, 0(a0)
+; RV32ZVE32F-NEXT:    fsd fa1, 8(a0)
+; RV32ZVE32F-NEXT:    fsd fa2, 16(a0)
+; RV32ZVE32F-NEXT:    fsd fa3, 24(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_falsemask_v4f64:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    fsd fa3, 24(a0)
-; RV64ZVE32F-NEXT:    fsd fa2, 16(a0)
-; RV64ZVE32F-NEXT:    fsd fa1, 8(a0)
 ; RV64ZVE32F-NEXT:    fsd fa0, 0(a0)
+; RV64ZVE32F-NEXT:    fsd fa1, 8(a0)
+; RV64ZVE32F-NEXT:    fsd fa2, 16(a0)
+; RV64ZVE32F-NEXT:    fsd fa3, 24(a0)
 ; RV64ZVE32F-NEXT:    ret
   %v = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %ptrs, i32 8, <4 x i1> zeroinitializer, <4 x double> %passthru)
   ret <4 x double> %v
@@ -13489,41 +13489,41 @@ define <32 x i64> @mgather_strided_split(ptr %base) {
 ; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    lw a3, 0(a1)
-; RV32ZVE32F-NEXT:    sw a3, 216(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a3, 252(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    lw a1, 4(a1)
-; RV32ZVE32F-NEXT:    sw a1, 208(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a1, 248(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v16, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v16
 ; RV32ZVE32F-NEXT:    lw a3, 0(a1)
-; RV32ZVE32F-NEXT:    sw a3, 252(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a3, 244(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    lw a1, 4(a1)
-; RV32ZVE32F-NEXT:    sw a1, 248(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a1, 240(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    vslidedown.vi v16, v8, 2
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v16
 ; RV32ZVE32F-NEXT:    lw a3, 0(a1)
-; RV32ZVE32F-NEXT:    sw a3, 244(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a3, 220(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    lw a1, 4(a1)
-; RV32ZVE32F-NEXT:    sw a1, 236(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a1, 216(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    vslidedown.vi v16, v8, 3
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v16
 ; RV32ZVE32F-NEXT:    lw a3, 0(a1)
-; RV32ZVE32F-NEXT:    sw a3, 228(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a3, 212(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    lw a1, 4(a1)
-; RV32ZVE32F-NEXT:    sw a1, 220(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a1, 208(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v16, v8, 4
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v16
 ; RV32ZVE32F-NEXT:    lw a3, 0(a1)
-; RV32ZVE32F-NEXT:    sw a3, 240(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a3, 236(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    lw a1, 4(a1)
 ; RV32ZVE32F-NEXT:    sw a1, 232(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    vslidedown.vi v16, v8, 5
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v16
 ; RV32ZVE32F-NEXT:    lw a3, 0(a1)
-; RV32ZVE32F-NEXT:    sw a3, 224(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a3, 228(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    lw a1, 4(a1)
-; RV32ZVE32F-NEXT:    sw a1, 212(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw a1, 224(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    vslidedown.vi v16, v8, 6
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v16
 ; RV32ZVE32F-NEXT:    lw a3, 0(a1)
@@ -13583,160 +13583,160 @@ define <32 x i64> @mgather_strided_split(ptr %base) {
 ; RV32ZVE32F-NEXT:    lw a2, 324(sp)
 ; RV32ZVE32F-NEXT:    lw a3, 328(sp)
 ; RV32ZVE32F-NEXT:    lw a4, 332(sp)
-; RV32ZVE32F-NEXT:    lw a5, 0(a1)
-; RV32ZVE32F-NEXT:    sw a5, 124(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    lw a1, 4(a1)
-; RV32ZVE32F-NEXT:    sw a1, 120(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    lw a1, 0(a2)
-; RV32ZVE32F-NEXT:    sw a1, 116(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    lw a1, 4(a2)
-; RV32ZVE32F-NEXT:    sw a1, 112(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    lw a1, 0(a3)
-; RV32ZVE32F-NEXT:    sw a1, 104(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    lw ra, 4(a3)
-; RV32ZVE32F-NEXT:    lw s10, 0(a4)
-; RV32ZVE32F-NEXT:    lw s8, 4(a4)
+; RV32ZVE32F-NEXT:    lw s8, 0(a1)
+; RV32ZVE32F-NEXT:    lw s9, 4(a1)
+; RV32ZVE32F-NEXT:    lw s10, 0(a2)
+; RV32ZVE32F-NEXT:    lw s11, 4(a2)
+; RV32ZVE32F-NEXT:    lw t5, 0(a3)
+; RV32ZVE32F-NEXT:    lw t6, 4(a3)
+; RV32ZVE32F-NEXT:    lw s2, 0(a4)
+; RV32ZVE32F-NEXT:    lw s3, 4(a4)
 ; RV32ZVE32F-NEXT:    lw a2, 336(sp)
 ; RV32ZVE32F-NEXT:    lw a4, 340(sp)
-; RV32ZVE32F-NEXT:    lw a6, 344(sp)
-; RV32ZVE32F-NEXT:    lw t0, 348(sp)
-; RV32ZVE32F-NEXT:    lw s5, 0(a2)
-; RV32ZVE32F-NEXT:    lw s4, 4(a2)
-; RV32ZVE32F-NEXT:    lw t6, 0(a4)
-; RV32ZVE32F-NEXT:    lw t5, 4(a4)
-; RV32ZVE32F-NEXT:    lw t3, 0(a6)
-; RV32ZVE32F-NEXT:    lw t2, 4(a6)
-; RV32ZVE32F-NEXT:    lw t1, 0(t0)
-; RV32ZVE32F-NEXT:    lw a7, 4(t0)
-; RV32ZVE32F-NEXT:    lw a6, 352(sp)
-; RV32ZVE32F-NEXT:    lw t0, 356(sp)
-; RV32ZVE32F-NEXT:    lw t4, 360(sp)
-; RV32ZVE32F-NEXT:    lw a1, 364(sp)
+; RV32ZVE32F-NEXT:    lw a5, 344(sp)
+; RV32ZVE32F-NEXT:    lw a6, 348(sp)
+; RV32ZVE32F-NEXT:    lw a7, 0(a2)
+; RV32ZVE32F-NEXT:    lw t0, 4(a2)
+; RV32ZVE32F-NEXT:    lw t1, 0(a4)
+; RV32ZVE32F-NEXT:    lw t2, 4(a4)
+; RV32ZVE32F-NEXT:    lw a1, 0(a5)
+; RV32ZVE32F-NEXT:    lw a2, 4(a5)
+; RV32ZVE32F-NEXT:    lw a3, 0(a6)
+; RV32ZVE32F-NEXT:    lw a4, 4(a6)
+; RV32ZVE32F-NEXT:    lw a5, 352(sp)
+; RV32ZVE32F-NEXT:    lw a6, 356(sp)
+; RV32ZVE32F-NEXT:    lw t3, 360(sp)
+; RV32ZVE32F-NEXT:    lw t4, 364(sp)
+; RV32ZVE32F-NEXT:    lw s4, 0(a5)
+; RV32ZVE32F-NEXT:    sw s4, 116(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    lw a5, 4(a5)
+; RV32ZVE32F-NEXT:    sw a5, 112(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    lw a5, 0(a6)
-; RV32ZVE32F-NEXT:    lw a6, 4(a6)
-; RV32ZVE32F-NEXT:    lw a2, 0(t0)
-; RV32ZVE32F-NEXT:    sw a2, 108(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    lw a2, 4(t0)
-; RV32ZVE32F-NEXT:    sw a2, 100(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    lw s11, 0(t4)
-; RV32ZVE32F-NEXT:    lw s9, 4(t4)
-; RV32ZVE32F-NEXT:    lw s7, 0(a1)
-; RV32ZVE32F-NEXT:    lw s6, 4(a1)
-; RV32ZVE32F-NEXT:    lw a4, 368(sp)
-; RV32ZVE32F-NEXT:    lw a3, 372(sp)
-; RV32ZVE32F-NEXT:    lw a2, 376(sp)
-; RV32ZVE32F-NEXT:    lw a1, 380(sp)
-; RV32ZVE32F-NEXT:    lw s3, 0(a4)
-; RV32ZVE32F-NEXT:    lw s2, 4(a4)
-; RV32ZVE32F-NEXT:    lw t4, 0(a3)
-; RV32ZVE32F-NEXT:    lw t0, 4(a3)
-; RV32ZVE32F-NEXT:    lw a4, 0(a2)
-; RV32ZVE32F-NEXT:    lw a3, 4(a2)
-; RV32ZVE32F-NEXT:    lw a2, 0(a1)
-; RV32ZVE32F-NEXT:    lw a1, 4(a1)
-; RV32ZVE32F-NEXT:    sw a6, 196(a0)
-; RV32ZVE32F-NEXT:    sw a5, 192(a0)
-; RV32ZVE32F-NEXT:    sw a7, 188(a0)
-; RV32ZVE32F-NEXT:    sw t1, 184(a0)
-; RV32ZVE32F-NEXT:    sw t2, 180(a0)
-; RV32ZVE32F-NEXT:    sw t3, 176(a0)
-; RV32ZVE32F-NEXT:    sw t5, 172(a0)
-; RV32ZVE32F-NEXT:    sw t6, 168(a0)
-; RV32ZVE32F-NEXT:    sw s4, 164(a0)
-; RV32ZVE32F-NEXT:    sw s5, 160(a0)
-; RV32ZVE32F-NEXT:    sw s8, 156(a0)
-; RV32ZVE32F-NEXT:    sw s10, 152(a0)
-; RV32ZVE32F-NEXT:    sw ra, 148(a0)
-; RV32ZVE32F-NEXT:    lw a5, 104(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 144(a0)
-; RV32ZVE32F-NEXT:    lw a5, 112(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 140(a0)
-; RV32ZVE32F-NEXT:    lw a5, 116(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 136(a0)
-; RV32ZVE32F-NEXT:    lw a5, 120(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 132(a0)
-; RV32ZVE32F-NEXT:    lw a5, 124(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 128(a0)
-; RV32ZVE32F-NEXT:    lw a5, 128(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 124(a0)
-; RV32ZVE32F-NEXT:    lw a5, 132(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 120(a0)
-; RV32ZVE32F-NEXT:    lw a5, 136(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 116(a0)
-; RV32ZVE32F-NEXT:    lw a5, 140(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 112(a0)
-; RV32ZVE32F-NEXT:    lw a5, 144(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 108(a0)
-; RV32ZVE32F-NEXT:    lw a5, 148(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 104(a0)
-; RV32ZVE32F-NEXT:    lw a5, 152(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 100(a0)
-; RV32ZVE32F-NEXT:    lw a5, 156(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 96(a0)
-; RV32ZVE32F-NEXT:    lw a5, 160(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 92(a0)
-; RV32ZVE32F-NEXT:    lw a5, 164(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 88(a0)
-; RV32ZVE32F-NEXT:    lw a5, 168(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 84(a0)
-; RV32ZVE32F-NEXT:    lw a5, 172(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 80(a0)
-; RV32ZVE32F-NEXT:    lw a5, 176(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 76(a0)
-; RV32ZVE32F-NEXT:    lw a5, 180(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 72(a0)
-; RV32ZVE32F-NEXT:    lw a5, 184(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 68(a0)
-; RV32ZVE32F-NEXT:    lw a5, 188(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 64(a0)
-; RV32ZVE32F-NEXT:    lw a5, 208(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 4(a0)
-; RV32ZVE32F-NEXT:    lw a5, 216(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a5, 0(a0)
-; RV32ZVE32F-NEXT:    sw a1, 252(a0)
-; RV32ZVE32F-NEXT:    sw a2, 248(a0)
-; RV32ZVE32F-NEXT:    sw a3, 244(a0)
-; RV32ZVE32F-NEXT:    sw a4, 240(a0)
-; RV32ZVE32F-NEXT:    sw t0, 236(a0)
-; RV32ZVE32F-NEXT:    sw t4, 232(a0)
-; RV32ZVE32F-NEXT:    sw s2, 228(a0)
-; RV32ZVE32F-NEXT:    sw s3, 224(a0)
-; RV32ZVE32F-NEXT:    sw s6, 220(a0)
-; RV32ZVE32F-NEXT:    sw s7, 216(a0)
-; RV32ZVE32F-NEXT:    sw s9, 212(a0)
-; RV32ZVE32F-NEXT:    sw s11, 208(a0)
-; RV32ZVE32F-NEXT:    lw a1, 100(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 204(a0)
+; RV32ZVE32F-NEXT:    sw a5, 124(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    lw a5, 4(a6)
+; RV32ZVE32F-NEXT:    sw a5, 120(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    lw ra, 0(t3)
+; RV32ZVE32F-NEXT:    lw a5, 4(t3)
+; RV32ZVE32F-NEXT:    sw a5, 108(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    lw a5, 0(t4)
+; RV32ZVE32F-NEXT:    sw a5, 104(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    lw a5, 4(t4)
+; RV32ZVE32F-NEXT:    sw a5, 100(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    lw a5, 368(sp)
+; RV32ZVE32F-NEXT:    lw a6, 372(sp)
+; RV32ZVE32F-NEXT:    lw t3, 376(sp)
+; RV32ZVE32F-NEXT:    lw t4, 380(sp)
+; RV32ZVE32F-NEXT:    lw s4, 0(a5)
+; RV32ZVE32F-NEXT:    lw s5, 4(a5)
+; RV32ZVE32F-NEXT:    lw s6, 0(a6)
+; RV32ZVE32F-NEXT:    lw s7, 4(a6)
+; RV32ZVE32F-NEXT:    lw a5, 0(t3)
+; RV32ZVE32F-NEXT:    lw a6, 4(t3)
+; RV32ZVE32F-NEXT:    lw t3, 0(t4)
+; RV32ZVE32F-NEXT:    lw t4, 4(t4)
+; RV32ZVE32F-NEXT:    sw a1, 176(a0)
+; RV32ZVE32F-NEXT:    sw a2, 180(a0)
+; RV32ZVE32F-NEXT:    sw a3, 184(a0)
+; RV32ZVE32F-NEXT:    sw a4, 188(a0)
+; RV32ZVE32F-NEXT:    sw a7, 160(a0)
+; RV32ZVE32F-NEXT:    sw t0, 164(a0)
+; RV32ZVE32F-NEXT:    sw t1, 168(a0)
+; RV32ZVE32F-NEXT:    sw t2, 172(a0)
+; RV32ZVE32F-NEXT:    sw t5, 144(a0)
+; RV32ZVE32F-NEXT:    sw t6, 148(a0)
+; RV32ZVE32F-NEXT:    sw s2, 152(a0)
+; RV32ZVE32F-NEXT:    sw s3, 156(a0)
+; RV32ZVE32F-NEXT:    sw s8, 128(a0)
+; RV32ZVE32F-NEXT:    sw s9, 132(a0)
+; RV32ZVE32F-NEXT:    sw s10, 136(a0)
+; RV32ZVE32F-NEXT:    sw s11, 140(a0)
+; RV32ZVE32F-NEXT:    lw a1, 140(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 112(a0)
+; RV32ZVE32F-NEXT:    lw a1, 136(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 116(a0)
+; RV32ZVE32F-NEXT:    lw a1, 132(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 120(a0)
+; RV32ZVE32F-NEXT:    lw a1, 128(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 124(a0)
+; RV32ZVE32F-NEXT:    lw a1, 156(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 96(a0)
+; RV32ZVE32F-NEXT:    lw a1, 152(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 100(a0)
+; RV32ZVE32F-NEXT:    lw a1, 148(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 104(a0)
+; RV32ZVE32F-NEXT:    lw a1, 144(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 108(a0)
+; RV32ZVE32F-NEXT:    lw a1, 172(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 80(a0)
+; RV32ZVE32F-NEXT:    lw a1, 168(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 84(a0)
+; RV32ZVE32F-NEXT:    lw a1, 164(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 88(a0)
+; RV32ZVE32F-NEXT:    lw a1, 160(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 92(a0)
+; RV32ZVE32F-NEXT:    lw a1, 188(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 64(a0)
+; RV32ZVE32F-NEXT:    lw a1, 184(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 68(a0)
+; RV32ZVE32F-NEXT:    lw a1, 180(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 72(a0)
+; RV32ZVE32F-NEXT:    lw a1, 176(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 76(a0)
+; RV32ZVE32F-NEXT:    sw a5, 240(a0)
+; RV32ZVE32F-NEXT:    sw a6, 244(a0)
+; RV32ZVE32F-NEXT:    sw t3, 248(a0)
+; RV32ZVE32F-NEXT:    sw t4, 252(a0)
+; RV32ZVE32F-NEXT:    sw s4, 224(a0)
+; RV32ZVE32F-NEXT:    sw s5, 228(a0)
+; RV32ZVE32F-NEXT:    sw s6, 232(a0)
+; RV32ZVE32F-NEXT:    sw s7, 236(a0)
+; RV32ZVE32F-NEXT:    sw ra, 208(a0)
 ; RV32ZVE32F-NEXT:    lw a1, 108(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 212(a0)
+; RV32ZVE32F-NEXT:    lw a1, 104(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 216(a0)
+; RV32ZVE32F-NEXT:    lw a1, 100(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 220(a0)
+; RV32ZVE32F-NEXT:    lw a1, 116(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 192(a0)
+; RV32ZVE32F-NEXT:    lw a1, 112(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 196(a0)
+; RV32ZVE32F-NEXT:    lw a1, 124(sp) # 4-byte Folded Reload
 ; RV32ZVE32F-NEXT:    sw a1, 200(a0)
+; RV32ZVE32F-NEXT:    lw a1, 120(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 204(a0)
 ; RV32ZVE32F-NEXT:    lw a1, 220(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 28(a0)
-; RV32ZVE32F-NEXT:    lw a1, 228(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 24(a0)
-; RV32ZVE32F-NEXT:    lw a1, 236(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 20(a0)
-; RV32ZVE32F-NEXT:    lw a1, 244(sp) # 4-byte Folded Reload
 ; RV32ZVE32F-NEXT:    sw a1, 16(a0)
-; RV32ZVE32F-NEXT:    lw a1, 248(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 12(a0)
+; RV32ZVE32F-NEXT:    lw a1, 216(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 20(a0)
+; RV32ZVE32F-NEXT:    lw a1, 212(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 24(a0)
+; RV32ZVE32F-NEXT:    lw a1, 208(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 28(a0)
 ; RV32ZVE32F-NEXT:    lw a1, 252(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 248(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 4(a0)
+; RV32ZVE32F-NEXT:    lw a1, 244(sp) # 4-byte Folded Reload
 ; RV32ZVE32F-NEXT:    sw a1, 8(a0)
-; RV32ZVE32F-NEXT:    lw a1, 192(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 60(a0)
-; RV32ZVE32F-NEXT:    lw a1, 196(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 56(a0)
-; RV32ZVE32F-NEXT:    lw a1, 200(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 52(a0)
+; RV32ZVE32F-NEXT:    lw a1, 240(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 12(a0)
 ; RV32ZVE32F-NEXT:    lw a1, 204(sp) # 4-byte Folded Reload
 ; RV32ZVE32F-NEXT:    sw a1, 48(a0)
-; RV32ZVE32F-NEXT:    lw a1, 212(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 44(a0)
-; RV32ZVE32F-NEXT:    lw a1, 224(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 40(a0)
+; RV32ZVE32F-NEXT:    lw a1, 200(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 52(a0)
+; RV32ZVE32F-NEXT:    lw a1, 196(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 56(a0)
+; RV32ZVE32F-NEXT:    lw a1, 192(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 60(a0)
+; RV32ZVE32F-NEXT:    lw a1, 236(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 32(a0)
 ; RV32ZVE32F-NEXT:    lw a1, 232(sp) # 4-byte Folded Reload
 ; RV32ZVE32F-NEXT:    sw a1, 36(a0)
-; RV32ZVE32F-NEXT:    lw a1, 240(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    sw a1, 32(a0)
+; RV32ZVE32F-NEXT:    lw a1, 228(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 40(a0)
+; RV32ZVE32F-NEXT:    lw a1, 224(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    sw a1, 44(a0)
 ; RV32ZVE32F-NEXT:    addi sp, s0, -512
 ; RV32ZVE32F-NEXT:    lw ra, 508(sp) # 4-byte Folded Reload
 ; RV32ZVE32F-NEXT:    lw s0, 504(sp) # 4-byte Folded Reload
@@ -13812,51 +13812,51 @@ define <32 x i64> @mgather_strided_split(ptr %base) {
 ; RV64ZVE32F-NEXT:    ld s8, 336(a1)
 ; RV64ZVE32F-NEXT:    ld s9, 352(a1)
 ; RV64ZVE32F-NEXT:    ld s10, 368(a1)
-; RV64ZVE32F-NEXT:    ld s11, 448(a1)
-; RV64ZVE32F-NEXT:    ld ra, 464(a1)
-; RV64ZVE32F-NEXT:    ld a3, 480(a1)
-; RV64ZVE32F-NEXT:    ld a2, 496(a1)
-; RV64ZVE32F-NEXT:    ld a6, 384(a1)
-; RV64ZVE32F-NEXT:    ld a5, 400(a1)
-; RV64ZVE32F-NEXT:    ld a4, 416(a1)
-; RV64ZVE32F-NEXT:    ld a1, 432(a1)
-; RV64ZVE32F-NEXT:    sd a2, 248(a0)
-; RV64ZVE32F-NEXT:    sd a3, 240(a0)
-; RV64ZVE32F-NEXT:    sd ra, 232(a0)
-; RV64ZVE32F-NEXT:    sd s11, 224(a0)
-; RV64ZVE32F-NEXT:    sd a1, 216(a0)
-; RV64ZVE32F-NEXT:    sd a4, 208(a0)
-; RV64ZVE32F-NEXT:    sd a5, 200(a0)
-; RV64ZVE32F-NEXT:    sd a6, 192(a0)
-; RV64ZVE32F-NEXT:    sd s10, 184(a0)
-; RV64ZVE32F-NEXT:    sd s9, 176(a0)
-; RV64ZVE32F-NEXT:    sd s8, 168(a0)
+; RV64ZVE32F-NEXT:    ld s11, 384(a1)
+; RV64ZVE32F-NEXT:    ld ra, 400(a1)
+; RV64ZVE32F-NEXT:    ld a6, 416(a1)
+; RV64ZVE32F-NEXT:    ld a5, 432(a1)
+; RV64ZVE32F-NEXT:    ld a2, 448(a1)
+; RV64ZVE32F-NEXT:    ld a3, 464(a1)
+; RV64ZVE32F-NEXT:    ld a4, 480(a1)
+; RV64ZVE32F-NEXT:    ld a1, 496(a1)
+; RV64ZVE32F-NEXT:    sd a2, 224(a0)
+; RV64ZVE32F-NEXT:    sd a3, 232(a0)
+; RV64ZVE32F-NEXT:    sd a4, 240(a0)
+; RV64ZVE32F-NEXT:    sd a1, 248(a0)
+; RV64ZVE32F-NEXT:    sd s11, 192(a0)
+; RV64ZVE32F-NEXT:    sd ra, 200(a0)
+; RV64ZVE32F-NEXT:    sd a6, 208(a0)
+; RV64ZVE32F-NEXT:    sd a5, 216(a0)
 ; RV64ZVE32F-NEXT:    sd s7, 160(a0)
-; RV64ZVE32F-NEXT:    sd s6, 152(a0)
-; RV64ZVE32F-NEXT:    sd s5, 144(a0)
-; RV64ZVE32F-NEXT:    sd s4, 136(a0)
+; RV64ZVE32F-NEXT:    sd s8, 168(a0)
+; RV64ZVE32F-NEXT:    sd s9, 176(a0)
+; RV64ZVE32F-NEXT:    sd s10, 184(a0)
 ; RV64ZVE32F-NEXT:    sd s3, 128(a0)
-; RV64ZVE32F-NEXT:    sd s2, 120(a0)
-; RV64ZVE32F-NEXT:    sd s1, 112(a0)
-; RV64ZVE32F-NEXT:    sd s0, 104(a0)
+; RV64ZVE32F-NEXT:    sd s4, 136(a0)
+; RV64ZVE32F-NEXT:    sd s5, 144(a0)
+; RV64ZVE32F-NEXT:    sd s6, 152(a0)
 ; RV64ZVE32F-NEXT:    sd t6, 96(a0)
-; RV64ZVE32F-NEXT:    sd t5, 88(a0)
-; RV64ZVE32F-NEXT:    sd t4, 80(a0)
-; RV64ZVE32F-NEXT:    sd t3, 72(a0)
+; RV64ZVE32F-NEXT:    sd s0, 104(a0)
+; RV64ZVE32F-NEXT:    sd s1, 112(a0)
+; RV64ZVE32F-NEXT:    sd s2, 120(a0)
 ; RV64ZVE32F-NEXT:    sd t2, 64(a0)
-; RV64ZVE32F-NEXT:    sd t1, 56(a0)
-; RV64ZVE32F-NEXT:    sd t0, 48(a0)
-; RV64ZVE32F-NEXT:    sd a7, 40(a0)
+; RV64ZVE32F-NEXT:    sd t3, 72(a0)
+; RV64ZVE32F-NEXT:    sd t4, 80(a0)
+; RV64ZVE32F-NEXT:    sd t5, 88(a0)
 ; RV64ZVE32F-NEXT:    ld a1, 0(sp) # 8-byte Folded Reload
 ; RV64ZVE32F-NEXT:    sd a1, 32(a0)
-; RV64ZVE32F-NEXT:    ld a1, 8(sp) # 8-byte Folded Reload
-; RV64ZVE32F-NEXT:    sd a1, 24(a0)
-; RV64ZVE32F-NEXT:    ld a1, 16(sp) # 8-byte Folded Reload
-; RV64ZVE32F-NEXT:    sd a1, 16(a0)
-; RV64ZVE32F-NEXT:    ld a1, 24(sp) # 8-byte Folded Reload
-; RV64ZVE32F-NEXT:    sd a1, 8(a0)
+; RV64ZVE32F-NEXT:    sd a7, 40(a0)
+; RV64ZVE32F-NEXT:    sd t0, 48(a0)
+; RV64ZVE32F-NEXT:    sd t1, 56(a0)
 ; RV64ZVE32F-NEXT:    ld a1, 32(sp) # 8-byte Folded Reload
 ; RV64ZVE32F-NEXT:    sd a1, 0(a0)
+; RV64ZVE32F-NEXT:    ld a1, 24(sp) # 8-byte Folded Reload
+; RV64ZVE32F-NEXT:    sd a1, 8(a0)
+; RV64ZVE32F-NEXT:    ld a1, 16(sp) # 8-byte Folded Reload
+; RV64ZVE32F-NEXT:    sd a1, 16(a0)
+; RV64ZVE32F-NEXT:    ld a1, 8(sp) # 8-byte Folded Reload
+; RV64ZVE32F-NEXT:    sd a1, 24(a0)
 ; RV64ZVE32F-NEXT:    ld ra, 136(sp) # 8-byte Folded Reload
 ; RV64ZVE32F-NEXT:    ld s0, 128(sp) # 8-byte Folded Reload
 ; RV64ZVE32F-NEXT:    ld s1, 120(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
index bc7758717c1c152b5f22171912838dfee08c0fa1..323f08acac28e36feb557c5f81ddbe7fe3691b63 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
@@ -2863,8 +2863,8 @@ define void @mscatter_v1i64(<1 x i64> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(a2)
 ; RV32ZVE32F-NEXT:    sw a0, 0(a2)
+; RV32ZVE32F-NEXT:    sw a1, 4(a2)
 ; RV32ZVE32F-NEXT:  .LBB36_2: # %else
 ; RV32ZVE32F-NEXT:    ret
 ;
@@ -2910,20 +2910,20 @@ define void @mscatter_v2i64(<2 x i64> %val, <2 x ptr> %ptrs, <2 x i1> %m) {
 ; RV32ZVE32F-NEXT:  .LBB37_2: # %else2
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB37_3: # %cond.store
-; RV32ZVE32F-NEXT:    lw a4, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a4, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a5, v8
-; RV32ZVE32F-NEXT:    sw a4, 4(a5)
-; RV32ZVE32F-NEXT:    sw a0, 0(a5)
+; RV32ZVE32F-NEXT:    sw a4, 0(a5)
+; RV32ZVE32F-NEXT:    sw a0, 4(a5)
 ; RV32ZVE32F-NEXT:    andi a3, a3, 2
 ; RV32ZVE32F-NEXT:    beqz a3, .LBB37_2
 ; RV32ZVE32F-NEXT:  .LBB37_4: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
-; RV32ZVE32F-NEXT:    sw a2, 4(a0)
 ; RV32ZVE32F-NEXT:    sw a1, 0(a0)
+; RV32ZVE32F-NEXT:    sw a2, 4(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mscatter_v2i64:
@@ -2987,20 +2987,20 @@ define void @mscatter_v4i64(<4 x i64> %val, <4 x ptr> %ptrs, <4 x i1> %m) {
 ; RV32ZVE32F-NEXT:  .LBB38_4: # %else6
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB38_5: # %cond.store
-; RV32ZVE32F-NEXT:    lw t0, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw t0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s t1, v8
-; RV32ZVE32F-NEXT:    sw t0, 4(t1)
-; RV32ZVE32F-NEXT:    sw a0, 0(t1)
+; RV32ZVE32F-NEXT:    sw t0, 0(t1)
+; RV32ZVE32F-NEXT:    sw a0, 4(t1)
 ; RV32ZVE32F-NEXT:    andi a0, a5, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB38_2
 ; RV32ZVE32F-NEXT:  .LBB38_6: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v9
-; RV32ZVE32F-NEXT:    sw a7, 4(a0)
 ; RV32ZVE32F-NEXT:    sw a6, 0(a0)
+; RV32ZVE32F-NEXT:    sw a7, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, a5, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB38_3
 ; RV32ZVE32F-NEXT:  .LBB38_7: # %cond.store3
@@ -3208,20 +3208,20 @@ define void @mscatter_v8i64(<8 x i64> %val, <8 x ptr> %ptrs, <8 x i1> %m) {
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB41_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw s1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw s1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw s1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw s1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, a7, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB41_2
 ; RV32ZVE32F-NEXT:  .LBB41_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s0, 4(a0)
 ; RV32ZVE32F-NEXT:    sw t6, 0(a0)
+; RV32ZVE32F-NEXT:    sw s0, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, a7, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB41_3
 ; RV32ZVE32F-NEXT:  .LBB41_12: # %cond.store3
@@ -3443,19 +3443,19 @@ define void @mscatter_baseidx_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8> %id
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB42_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB42_2
 ; RV32ZVE32F-NEXT:  .LBB42_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB42_3
 ; RV32ZVE32F-NEXT:  .LBB42_12: # %cond.store3
@@ -3687,19 +3687,19 @@ define void @mscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB43_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB43_2
 ; RV32ZVE32F-NEXT:  .LBB43_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB43_3
 ; RV32ZVE32F-NEXT:  .LBB43_12: # %cond.store3
@@ -3933,19 +3933,19 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB44_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB44_2
 ; RV32ZVE32F-NEXT:  .LBB44_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB44_3
 ; RV32ZVE32F-NEXT:  .LBB44_12: # %cond.store3
@@ -4186,19 +4186,19 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB45_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_2
 ; RV32ZVE32F-NEXT:  .LBB45_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB45_3
 ; RV32ZVE32F-NEXT:  .LBB45_12: # %cond.store3
@@ -4431,19 +4431,19 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB46_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_2
 ; RV32ZVE32F-NEXT:  .LBB46_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB46_3
 ; RV32ZVE32F-NEXT:  .LBB46_12: # %cond.store3
@@ -4678,19 +4678,19 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB47_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_2
 ; RV32ZVE32F-NEXT:  .LBB47_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB47_3
 ; RV32ZVE32F-NEXT:  .LBB47_12: # %cond.store3
@@ -4932,19 +4932,19 @@ define void @mscatter_baseidx_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i32> %
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB48_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB48_2
 ; RV32ZVE32F-NEXT:  .LBB48_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB48_3
 ; RV32ZVE32F-NEXT:  .LBB48_12: # %cond.store3
@@ -5175,19 +5175,19 @@ define void @mscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB49_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB49_2
 ; RV32ZVE32F-NEXT:  .LBB49_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB49_3
 ; RV32ZVE32F-NEXT:  .LBB49_12: # %cond.store3
@@ -5419,19 +5419,19 @@ define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV32ZVE32F-NEXT:    addi sp, sp, 16
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB50_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB50_2
 ; RV32ZVE32F-NEXT:  .LBB50_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, t0, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB50_3
 ; RV32ZVE32F-NEXT:  .LBB50_12: # %cond.store3
@@ -5705,19 +5705,19 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ; RV32ZVE32F-NEXT:    addi sp, sp, 48
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB51_10: # %cond.store
-; RV32ZVE32F-NEXT:    lw a1, 4(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a0)
+; RV32ZVE32F-NEXT:    lw a1, 0(a0)
+; RV32ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
-; RV32ZVE32F-NEXT:    sw a1, 4(s2)
-; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    sw a1, 0(s2)
+; RV32ZVE32F-NEXT:    sw a0, 4(s2)
 ; RV32ZVE32F-NEXT:    andi a0, a2, 2
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB51_2
 ; RV32ZVE32F-NEXT:  .LBB51_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
-; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
 ; RV32ZVE32F-NEXT:    andi a0, a2, 4
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB51_3
 ; RV32ZVE32F-NEXT:  .LBB51_12: # %cond.store3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
index 016f95bfef7e71956310d1c0db57fd6248bf862b..080ffe1a78b511f143b45ca91f6f1c3f92a35918 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
@@ -833,8 +833,8 @@ define signext i64 @vpreduce_add_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %m
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -867,8 +867,8 @@ define signext i64 @vpreduce_umax_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -901,8 +901,8 @@ define signext i64 @vpreduce_smax_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -935,8 +935,8 @@ define signext i64 @vpreduce_umin_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -969,8 +969,8 @@ define signext i64 @vpreduce_smin_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1003,8 +1003,8 @@ define signext i64 @vpreduce_and_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %m
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1037,8 +1037,8 @@ define signext i64 @vpreduce_or_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %m,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1071,8 +1071,8 @@ define signext i64 @vpreduce_xor_v2i64(i64 signext %s, <2 x i64> %v, <2 x i1> %m
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1105,8 +1105,8 @@ define signext i64 @vpreduce_add_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %m
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1139,8 +1139,8 @@ define signext i64 @vpreduce_umax_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1173,8 +1173,8 @@ define signext i64 @vpreduce_smax_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1207,8 +1207,8 @@ define signext i64 @vpreduce_umin_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1241,8 +1241,8 @@ define signext i64 @vpreduce_smin_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1275,8 +1275,8 @@ define signext i64 @vpreduce_and_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %m
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1309,8 +1309,8 @@ define signext i64 @vpreduce_or_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %m,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1343,8 +1343,8 @@ define signext i64 @vpreduce_xor_v4i64(i64 signext %s, <4 x i64> %v, <4 x i1> %m
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
index 5f3847e085055b36d90583db6ed25ed11dfb593b..0cecec31e2bda38346d196fd42ff48b1dabca86a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
@@ -1376,8 +1376,8 @@ define <8 x i1> @icmp_eq_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1404,8 +1404,8 @@ define <8 x i1> @icmp_eq_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 z
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1465,8 +1465,8 @@ define <8 x i1> @icmp_ne_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1493,8 +1493,8 @@ define <8 x i1> @icmp_ne_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 z
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1554,8 +1554,8 @@ define <8 x i1> @icmp_ugt_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1582,8 +1582,8 @@ define <8 x i1> @icmp_ugt_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1643,8 +1643,8 @@ define <8 x i1> @icmp_uge_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1673,8 +1673,8 @@ define <8 x i1> @icmp_uge_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1734,8 +1734,8 @@ define <8 x i1> @icmp_ult_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1762,8 +1762,8 @@ define <8 x i1> @icmp_ult_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1823,8 +1823,8 @@ define <8 x i1> @icmp_sgt_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1851,8 +1851,8 @@ define <8 x i1> @icmp_sgt_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1912,8 +1912,8 @@ define <8 x i1> @icmp_sge_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1942,8 +1942,8 @@ define <8 x i1> @icmp_sge_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2003,8 +2003,8 @@ define <8 x i1> @icmp_slt_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2031,8 +2031,8 @@ define <8 x i1> @icmp_slt_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2092,8 +2092,8 @@ define <8 x i1> @icmp_sle_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2120,8 +2120,8 @@ define <8 x i1> @icmp_sle_vx_swap_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll
index 0d8a9692442353a26300e7b7a02fb56a0d1dde79..ff5f6960ed4e4d6ddeeb307d3bacc9e1ce6277ff 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1down.ll
@@ -86,8 +86,8 @@ define <2 x i64> @vslide1down_2xi64(<2 x i64> %v, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -111,8 +111,8 @@ define <4 x i64> @vslide1down_4xi64(<4 x i64> %v, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
index d1fb30c7daa3ea07f68393adf0a2d3afa2269fe6..e0b2dd1af918357221bf18cd54ea71080997b6c9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
@@ -93,8 +93,8 @@ define <2 x i64> @vslide1up_2xi64(<2 x i64> %v, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -119,8 +119,8 @@ define <4 x i64> @vslide1up_4xi64(<4 x i64> %v, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
index 072c88f6353eaf1aae5355ad66989cc159728c1d..4ef65032469e418ebe37bb02239ea5a1bf58a989 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
@@ -259,10 +259,10 @@ define void @vnsrl_0_i64(ptr %in, ptr %out) {
 ;
 ; ZVE32F-LABEL: vnsrl_0_i64:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a2, 16(a0)
-; ZVE32F-NEXT:    ld a0, 0(a0)
-; ZVE32F-NEXT:    sd a2, 8(a1)
-; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ld a2, 0(a0)
+; ZVE32F-NEXT:    ld a0, 16(a0)
+; ZVE32F-NEXT:    sd a2, 0(a1)
+; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
 entry:
   %0 = load <4 x i64>, ptr %in, align 8
@@ -285,10 +285,10 @@ define void @vnsrl_64_i64(ptr %in, ptr %out) {
 ;
 ; ZVE32F-LABEL: vnsrl_64_i64:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a2, 24(a0)
-; ZVE32F-NEXT:    ld a0, 8(a0)
-; ZVE32F-NEXT:    sd a2, 8(a1)
-; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ld a2, 8(a0)
+; ZVE32F-NEXT:    ld a0, 24(a0)
+; ZVE32F-NEXT:    sd a2, 0(a1)
+; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
 entry:
   %0 = load <4 x i64>, ptr %in, align 8
@@ -310,10 +310,10 @@ define void @vnsrl_0_double(ptr %in, ptr %out) {
 ;
 ; ZVE32F-LABEL: vnsrl_0_double:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a2, 16(a0)
-; ZVE32F-NEXT:    ld a0, 0(a0)
-; ZVE32F-NEXT:    sd a2, 8(a1)
-; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ld a2, 0(a0)
+; ZVE32F-NEXT:    ld a0, 16(a0)
+; ZVE32F-NEXT:    sd a2, 0(a1)
+; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
 entry:
   %0 = load <4 x double>, ptr %in, align 8
@@ -336,10 +336,10 @@ define void @vnsrl_64_double(ptr %in, ptr %out) {
 ;
 ; ZVE32F-LABEL: vnsrl_64_double:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a2, 24(a0)
-; ZVE32F-NEXT:    ld a0, 8(a0)
-; ZVE32F-NEXT:    sd a2, 8(a1)
-; ZVE32F-NEXT:    sd a0, 0(a1)
+; ZVE32F-NEXT:    ld a2, 8(a0)
+; ZVE32F-NEXT:    ld a0, 24(a0)
+; ZVE32F-NEXT:    sd a2, 0(a1)
+; ZVE32F-NEXT:    sd a0, 8(a1)
 ; ZVE32F-NEXT:    ret
 entry:
   %0 = load <4 x double>, ptr %in, align 8
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
index 9385fa69b2f04900eae9829e6156289aec7ef919..28202dc07f9564c7db8357f948d7e5e410fec40e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
@@ -663,14 +663,14 @@ define void @gather_of_pointers(ptr noalias nocapture %arg, ptr noalias nocaptur
 ; ZVE32F-NEXT:    add a6, a1, a6
 ; ZVE32F-NEXT:    mul a7, a2, a5
 ; ZVE32F-NEXT:    add a7, a1, a7
-; ZVE32F-NEXT:    ld t0, 0(a6)
-; ZVE32F-NEXT:    ld t1, 0(a7)
-; ZVE32F-NEXT:    ld a6, 80(a6)
+; ZVE32F-NEXT:    ld t0, 0(a7)
+; ZVE32F-NEXT:    ld t1, 0(a6)
 ; ZVE32F-NEXT:    ld a7, 80(a7)
-; ZVE32F-NEXT:    sd t0, 8(a0)
-; ZVE32F-NEXT:    sd t1, 0(a0)
-; ZVE32F-NEXT:    sd a6, 24(a0)
+; ZVE32F-NEXT:    ld a6, 80(a6)
+; ZVE32F-NEXT:    sd t0, 0(a0)
+; ZVE32F-NEXT:    sd t1, 8(a0)
 ; ZVE32F-NEXT:    sd a7, 16(a0)
+; ZVE32F-NEXT:    sd a6, 24(a0)
 ; ZVE32F-NEXT:    addi a2, a2, 4
 ; ZVE32F-NEXT:    addi a0, a0, 32
 ; ZVE32F-NEXT:    addi a4, a4, 4
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
index eba3bd9d86dbb398c86a0ccb229b2cd24810b452..293b75dc207c86e81961373085bcc8075d409cf8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-unaligned.ll
@@ -281,9 +281,9 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV32-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32-SLOW-NEXT:    vmv.x.s a2, v9
+; RV32-SLOW-NEXT:    srli a3, a1, 8
 ; RV32-SLOW-NEXT:    sb a1, 0(a2)
-; RV32-SLOW-NEXT:    srli a1, a1, 8
-; RV32-SLOW-NEXT:    sb a1, 1(a2)
+; RV32-SLOW-NEXT:    sb a3, 1(a2)
 ; RV32-SLOW-NEXT:    andi a1, a0, 2
 ; RV32-SLOW-NEXT:    beqz a1, .LBB6_2
 ; RV32-SLOW-NEXT:  .LBB6_6: # %cond.store1
@@ -293,9 +293,9 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32-SLOW-NEXT:    vslidedown.vi v10, v9, 1
 ; RV32-SLOW-NEXT:    vmv.x.s a2, v10
+; RV32-SLOW-NEXT:    srli a3, a1, 8
 ; RV32-SLOW-NEXT:    sb a1, 0(a2)
-; RV32-SLOW-NEXT:    srli a1, a1, 8
-; RV32-SLOW-NEXT:    sb a1, 1(a2)
+; RV32-SLOW-NEXT:    sb a3, 1(a2)
 ; RV32-SLOW-NEXT:    andi a1, a0, 4
 ; RV32-SLOW-NEXT:    beqz a1, .LBB6_3
 ; RV32-SLOW-NEXT:  .LBB6_7: # %cond.store3
@@ -305,9 +305,9 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32-SLOW-NEXT:    vslidedown.vi v10, v9, 2
 ; RV32-SLOW-NEXT:    vmv.x.s a2, v10
+; RV32-SLOW-NEXT:    srli a3, a1, 8
 ; RV32-SLOW-NEXT:    sb a1, 0(a2)
-; RV32-SLOW-NEXT:    srli a1, a1, 8
-; RV32-SLOW-NEXT:    sb a1, 1(a2)
+; RV32-SLOW-NEXT:    sb a3, 1(a2)
 ; RV32-SLOW-NEXT:    andi a0, a0, 8
 ; RV32-SLOW-NEXT:    beqz a0, .LBB6_4
 ; RV32-SLOW-NEXT:  .LBB6_8: # %cond.store5
@@ -317,9 +317,9 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32-SLOW-NEXT:    vslidedown.vi v8, v9, 3
 ; RV32-SLOW-NEXT:    vmv.x.s a1, v8
+; RV32-SLOW-NEXT:    srli a2, a0, 8
 ; RV32-SLOW-NEXT:    sb a0, 0(a1)
-; RV32-SLOW-NEXT:    srli a0, a0, 8
-; RV32-SLOW-NEXT:    sb a0, 1(a1)
+; RV32-SLOW-NEXT:    sb a2, 1(a1)
 ; RV32-SLOW-NEXT:    ret
 ;
 ; RV64-SLOW-LABEL: mscatter_v4i16_align1:
@@ -345,8 +345,8 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV64-SLOW-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-SLOW-NEXT:    vmv.x.s a2, v10
 ; RV64-SLOW-NEXT:    srli a3, a1, 8
-; RV64-SLOW-NEXT:    sb a3, 1(a2)
 ; RV64-SLOW-NEXT:    sb a1, 0(a2)
+; RV64-SLOW-NEXT:    sb a3, 1(a2)
 ; RV64-SLOW-NEXT:    andi a1, a0, 2
 ; RV64-SLOW-NEXT:    beqz a1, .LBB6_2
 ; RV64-SLOW-NEXT:  .LBB6_6: # %cond.store1
@@ -357,8 +357,8 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV64-SLOW-NEXT:    vslidedown.vi v9, v10, 1
 ; RV64-SLOW-NEXT:    vmv.x.s a2, v9
 ; RV64-SLOW-NEXT:    srli a3, a1, 8
-; RV64-SLOW-NEXT:    sb a3, 1(a2)
 ; RV64-SLOW-NEXT:    sb a1, 0(a2)
+; RV64-SLOW-NEXT:    sb a3, 1(a2)
 ; RV64-SLOW-NEXT:    andi a1, a0, 4
 ; RV64-SLOW-NEXT:    beqz a1, .LBB6_3
 ; RV64-SLOW-NEXT:  .LBB6_7: # %cond.store3
@@ -369,8 +369,8 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV64-SLOW-NEXT:    vslidedown.vi v12, v10, 2
 ; RV64-SLOW-NEXT:    vmv.x.s a2, v12
 ; RV64-SLOW-NEXT:    srli a3, a1, 8
-; RV64-SLOW-NEXT:    sb a3, 1(a2)
 ; RV64-SLOW-NEXT:    sb a1, 0(a2)
+; RV64-SLOW-NEXT:    sb a3, 1(a2)
 ; RV64-SLOW-NEXT:    andi a0, a0, 8
 ; RV64-SLOW-NEXT:    beqz a0, .LBB6_4
 ; RV64-SLOW-NEXT:  .LBB6_8: # %cond.store5
@@ -381,8 +381,8 @@ define void @mscatter_v4i16_align1(<4 x i16> %val, <4 x ptr> %ptrs, <4 x i1> %m)
 ; RV64-SLOW-NEXT:    vslidedown.vi v8, v10, 3
 ; RV64-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV64-SLOW-NEXT:    srli a2, a0, 8
-; RV64-SLOW-NEXT:    sb a2, 1(a1)
 ; RV64-SLOW-NEXT:    sb a0, 0(a1)
+; RV64-SLOW-NEXT:    sb a2, 1(a1)
 ; RV64-SLOW-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: mscatter_v4i16_align1:
@@ -418,9 +418,9 @@ define void @mscatter_v2i32_align2(<2 x i32> %val, <2 x ptr> %ptrs, <2 x i1> %m)
 ; RV32-SLOW-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV32-SLOW-NEXT:    vmv.x.s a2, v9
+; RV32-SLOW-NEXT:    srli a3, a1, 16
 ; RV32-SLOW-NEXT:    sh a1, 0(a2)
-; RV32-SLOW-NEXT:    srli a1, a1, 16
-; RV32-SLOW-NEXT:    sh a1, 2(a2)
+; RV32-SLOW-NEXT:    sh a3, 2(a2)
 ; RV32-SLOW-NEXT:    andi a0, a0, 2
 ; RV32-SLOW-NEXT:    beqz a0, .LBB7_2
 ; RV32-SLOW-NEXT:  .LBB7_4: # %cond.store1
@@ -429,9 +429,9 @@ define void @mscatter_v2i32_align2(<2 x i32> %val, <2 x ptr> %ptrs, <2 x i1> %m)
 ; RV32-SLOW-NEXT:    vmv.x.s a0, v8
 ; RV32-SLOW-NEXT:    vslidedown.vi v8, v9, 1
 ; RV32-SLOW-NEXT:    vmv.x.s a1, v8
+; RV32-SLOW-NEXT:    srli a2, a0, 16
 ; RV32-SLOW-NEXT:    sh a0, 0(a1)
-; RV32-SLOW-NEXT:    srli a0, a0, 16
-; RV32-SLOW-NEXT:    sh a0, 2(a1)
+; RV32-SLOW-NEXT:    sh a2, 2(a1)
 ; RV32-SLOW-NEXT:    ret
 ;
 ; RV64-SLOW-LABEL: mscatter_v2i32_align2:
@@ -450,9 +450,9 @@ define void @mscatter_v2i32_align2(<2 x i32> %val, <2 x ptr> %ptrs, <2 x i1> %m)
 ; RV64-SLOW-NEXT:    vmv.x.s a1, v8
 ; RV64-SLOW-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-SLOW-NEXT:    vmv.x.s a2, v9
+; RV64-SLOW-NEXT:    srli a3, a1, 16
 ; RV64-SLOW-NEXT:    sh a1, 0(a2)
-; RV64-SLOW-NEXT:    srli a1, a1, 16
-; RV64-SLOW-NEXT:    sh a1, 2(a2)
+; RV64-SLOW-NEXT:    sh a3, 2(a2)
 ; RV64-SLOW-NEXT:    andi a0, a0, 2
 ; RV64-SLOW-NEXT:    beqz a0, .LBB7_2
 ; RV64-SLOW-NEXT:  .LBB7_4: # %cond.store1
@@ -462,9 +462,9 @@ define void @mscatter_v2i32_align2(<2 x i32> %val, <2 x ptr> %ptrs, <2 x i1> %m)
 ; RV64-SLOW-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; RV64-SLOW-NEXT:    vslidedown.vi v8, v9, 1
 ; RV64-SLOW-NEXT:    vmv.x.s a1, v8
+; RV64-SLOW-NEXT:    srli a2, a0, 16
 ; RV64-SLOW-NEXT:    sh a0, 0(a1)
-; RV64-SLOW-NEXT:    srli a0, a0, 16
-; RV64-SLOW-NEXT:    sh a0, 2(a1)
+; RV64-SLOW-NEXT:    sh a2, 2(a1)
 ; RV64-SLOW-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: mscatter_v2i32_align2:
@@ -605,18 +605,18 @@ define void @masked_store_v2i32_align2(<2 x i32> %val, ptr %a, <2 x i32> %m) nou
 ; SLOW-NEXT:  .LBB9_3: # %cond.store
 ; SLOW-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; SLOW-NEXT:    vmv.x.s a2, v8
+; SLOW-NEXT:    srli a3, a2, 16
 ; SLOW-NEXT:    sh a2, 0(a0)
-; SLOW-NEXT:    srli a2, a2, 16
-; SLOW-NEXT:    sh a2, 2(a0)
+; SLOW-NEXT:    sh a3, 2(a0)
 ; SLOW-NEXT:    andi a1, a1, 2
 ; SLOW-NEXT:    beqz a1, .LBB9_2
 ; SLOW-NEXT:  .LBB9_4: # %cond.store1
 ; SLOW-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; SLOW-NEXT:    vslidedown.vi v8, v8, 1
 ; SLOW-NEXT:    vmv.x.s a1, v8
+; SLOW-NEXT:    srli a2, a1, 16
 ; SLOW-NEXT:    sh a1, 4(a0)
-; SLOW-NEXT:    srli a1, a1, 16
-; SLOW-NEXT:    sh a1, 6(a0)
+; SLOW-NEXT:    sh a2, 6(a0)
 ; SLOW-NEXT:    ret
 ;
 ; FAST-LABEL: masked_store_v2i32_align2:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll
index ea7f6beb22a7cb1e52592763c2ed252568e37693..da34396d72a79479fde3cec8e900ed60dea18ba1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll
@@ -185,8 +185,8 @@ define <8 x i64> @vaaddu_vx_v8i64_floor(<8 x i64> %x, i64 %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -428,8 +428,8 @@ define <8 x i64> @vaaddu_vx_v8i64_ceil(<8 x i64> %x, i64 %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll
index 6246ef7db0cb3394945345f37aee6bb3d1e18a43..2f7b7b6807ce212d33b3590a6ce191e15e32939e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll
@@ -992,8 +992,8 @@ define <2 x i64> @vadd_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1018,8 +1018,8 @@ define <2 x i64> @vadd_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1086,8 +1086,8 @@ define <4 x i64> @vadd_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1112,8 +1112,8 @@ define <4 x i64> @vadd_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1180,8 +1180,8 @@ define <8 x i64> @vadd_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1206,8 +1206,8 @@ define <8 x i64> @vadd_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1274,8 +1274,8 @@ define <16 x i64> @vadd_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1300,8 +1300,8 @@ define <16 x i64> @vadd_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vand-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vand-vp.ll
index c413dd86f37128c9c07f8f20ffb854c351e53c52..fb7816ce254583eb0319848945a5f1bb488a2e02 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vand-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vand-vp.ll
@@ -859,8 +859,8 @@ define <2 x i64> @vand_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -885,8 +885,8 @@ define <2 x i64> @vand_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -953,8 +953,8 @@ define <4 x i64> @vand_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -979,8 +979,8 @@ define <4 x i64> @vand_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1047,8 +1047,8 @@ define <8 x i64> @vand_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1073,8 +1073,8 @@ define <8 x i64> @vand_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1141,8 +1141,8 @@ define <11 x i64> @vand_vx_v11i64(<11 x i64> %va, i64 %b, <11 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1167,8 +1167,8 @@ define <11 x i64> @vand_vx_v11i64_unmasked(<11 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1235,8 +1235,8 @@ define <16 x i64> @vand_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1261,8 +1261,8 @@ define <16 x i64> @vand_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdiv-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdiv-vp.ll
index e626727ffb8b4cc79226984695ae5c145405b08c..d1f77bf275ab5c6ae601d29066d3d514704f1412 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdiv-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdiv-vp.ll
@@ -611,8 +611,8 @@ define <2 x i64> @vdiv_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -637,8 +637,8 @@ define <2 x i64> @vdiv_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -685,8 +685,8 @@ define <4 x i64> @vdiv_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -711,8 +711,8 @@ define <4 x i64> @vdiv_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -759,8 +759,8 @@ define <8 x i64> @vdiv_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -785,8 +785,8 @@ define <8 x i64> @vdiv_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -833,8 +833,8 @@ define <16 x i64> @vdiv_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -859,8 +859,8 @@ define <16 x i64> @vdiv_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdivu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdivu-vp.ll
index 3715449ef27f0647ec63fd829dd8f568e22f5076..0219ebf503aa5999542fc79faa3fdccb87ad9c08 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdivu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vdivu-vp.ll
@@ -610,8 +610,8 @@ define <2 x i64> @vdivu_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -636,8 +636,8 @@ define <2 x i64> @vdivu_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -684,8 +684,8 @@ define <4 x i64> @vdivu_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -710,8 +710,8 @@ define <4 x i64> @vdivu_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -758,8 +758,8 @@ define <8 x i64> @vdivu_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -784,8 +784,8 @@ define <8 x i64> @vdivu_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -832,8 +832,8 @@ define <16 x i64> @vdivu_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zer
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -858,8 +858,8 @@ define <16 x i64> @vdivu_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmacc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmacc-vp.ll
index 4805d6782a3b934e2dd0a34b843b1de7b696c0b8..382101cbdffb33c2dcce6b656eb6eec39b543ea5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmacc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmacc-vp.ll
@@ -1375,8 +1375,8 @@ define <2 x i64> @vmacc_vx_nxv2i64(<2 x i64> %a, i64 %b, <2 x i64> %c,  <2 x i1>
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1405,8 +1405,8 @@ define <2 x i64> @vmacc_vx_nxv2i64_unmasked(<2 x i64> %a, i64 %b, <2 x i64> %c,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1448,8 +1448,8 @@ define <2 x i64> @vmacc_vx_nxv2i64_ta(<2 x i64> %a, i64 %b, <2 x i64> %c,  <2 x
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1509,8 +1509,8 @@ define <4 x i64> @vmacc_vx_nxv4i64(<4 x i64> %a, i64 %b, <4 x i64> %c,  <4 x i1>
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1539,8 +1539,8 @@ define <4 x i64> @vmacc_vx_nxv4i64_unmasked(<4 x i64> %a, i64 %b, <4 x i64> %c,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1582,8 +1582,8 @@ define <4 x i64> @vmacc_vx_nxv4i64_ta(<4 x i64> %a, i64 %b, <4 x i64> %c,  <4 x
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1643,8 +1643,8 @@ define <8 x i64> @vmacc_vx_nxv8i64(<8 x i64> %a, i64 %b, <8 x i64> %c,  <8 x i1>
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1673,8 +1673,8 @@ define <8 x i64> @vmacc_vx_nxv8i64_unmasked(<8 x i64> %a, i64 %b, <8 x i64> %c,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1716,8 +1716,8 @@ define <8 x i64> @vmacc_vx_nxv8i64_ta(<8 x i64> %a, i64 %b, <8 x i64> %c,  <8 x
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll
index 6adc6ba9621a8fea6656865f87e2448c67c84960..9a3a513167d87958e8e4c21cd21324cb2e870735 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll
@@ -744,8 +744,8 @@ define <2 x i64> @vmax_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -770,8 +770,8 @@ define <2 x i64> @vmax_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -818,8 +818,8 @@ define <4 x i64> @vmax_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -844,8 +844,8 @@ define <4 x i64> @vmax_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -892,8 +892,8 @@ define <8 x i64> @vmax_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -918,8 +918,8 @@ define <8 x i64> @vmax_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -966,8 +966,8 @@ define <16 x i64> @vmax_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -992,8 +992,8 @@ define <16 x i64> @vmax_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll
index baeb372c017e2e408a83d9bdeff793590e8022c1..5090cb2899eb703082752225563757c43225bcd5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll
@@ -743,8 +743,8 @@ define <2 x i64> @vmaxu_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -769,8 +769,8 @@ define <2 x i64> @vmaxu_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -817,8 +817,8 @@ define <4 x i64> @vmaxu_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -843,8 +843,8 @@ define <4 x i64> @vmaxu_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -891,8 +891,8 @@ define <8 x i64> @vmaxu_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -917,8 +917,8 @@ define <8 x i64> @vmaxu_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -965,8 +965,8 @@ define <16 x i64> @vmaxu_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zer
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -991,8 +991,8 @@ define <16 x i64> @vmaxu_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll
index d0c21ce05c025455d0b124a0c7070b7ba849bc59..a25eddc8d973fccbe0090354e1cba2361e97b13b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll
@@ -744,8 +744,8 @@ define <2 x i64> @vmin_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -770,8 +770,8 @@ define <2 x i64> @vmin_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -818,8 +818,8 @@ define <4 x i64> @vmin_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -844,8 +844,8 @@ define <4 x i64> @vmin_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -892,8 +892,8 @@ define <8 x i64> @vmin_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -918,8 +918,8 @@ define <8 x i64> @vmin_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -966,8 +966,8 @@ define <16 x i64> @vmin_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -992,8 +992,8 @@ define <16 x i64> @vmin_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll
index a730ba4729d252a3ab48196f5add3cfb6d0343a8..70df4d000ba925d0a877b64276feb4e17218590b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll
@@ -743,8 +743,8 @@ define <2 x i64> @vminu_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -769,8 +769,8 @@ define <2 x i64> @vminu_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -817,8 +817,8 @@ define <4 x i64> @vminu_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -843,8 +843,8 @@ define <4 x i64> @vminu_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -891,8 +891,8 @@ define <8 x i64> @vminu_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -917,8 +917,8 @@ define <8 x i64> @vminu_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -965,8 +965,8 @@ define <16 x i64> @vminu_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zer
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -991,8 +991,8 @@ define <16 x i64> @vminu_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmul-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmul-vp.ll
index 8970fbf740d2356ec4eea6ca25f910a21242bc71..143ba263267fa24b505ff48e3e248a301561bcb2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmul-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmul-vp.ll
@@ -653,8 +653,8 @@ define <2 x i64> @vmul_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -679,8 +679,8 @@ define <2 x i64> @vmul_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -727,8 +727,8 @@ define <4 x i64> @vmul_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -753,8 +753,8 @@ define <4 x i64> @vmul_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -801,8 +801,8 @@ define <8 x i64> @vmul_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -827,8 +827,8 @@ define <8 x i64> @vmul_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -875,8 +875,8 @@ define <16 x i64> @vmul_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -901,8 +901,8 @@ define <16 x i64> @vmul_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vnmsac-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vnmsac-vp.ll
index 805e2e2e6bd35d10c1d853543710c73e3c517180..7540b493aeded055fbc84059113ef76ac880a40d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vnmsac-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vnmsac-vp.ll
@@ -1375,8 +1375,8 @@ define <2 x i64> @vnmsac_vx_nxv2i64(<2 x i64> %a, i64 %b, <2 x i64> %c,  <2 x i1
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1405,8 +1405,8 @@ define <2 x i64> @vnmsac_vx_nxv2i64_unmasked(<2 x i64> %a, i64 %b, <2 x i64> %c,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1448,8 +1448,8 @@ define <2 x i64> @vnmsac_vx_nxv2i64_ta(<2 x i64> %a, i64 %b, <2 x i64> %c,  <2 x
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1509,8 +1509,8 @@ define <4 x i64> @vnmsac_vx_nxv4i64(<4 x i64> %a, i64 %b, <4 x i64> %c,  <4 x i1
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1539,8 +1539,8 @@ define <4 x i64> @vnmsac_vx_nxv4i64_unmasked(<4 x i64> %a, i64 %b, <4 x i64> %c,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1582,8 +1582,8 @@ define <4 x i64> @vnmsac_vx_nxv4i64_ta(<4 x i64> %a, i64 %b, <4 x i64> %c,  <4 x
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1643,8 +1643,8 @@ define <8 x i64> @vnmsac_vx_nxv8i64(<8 x i64> %a, i64 %b, <8 x i64> %c,  <8 x i1
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1673,8 +1673,8 @@ define <8 x i64> @vnmsac_vx_nxv8i64_unmasked(<8 x i64> %a, i64 %b, <8 x i64> %c,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1716,8 +1716,8 @@ define <8 x i64> @vnmsac_vx_nxv8i64_ta(<8 x i64> %a, i64 %b, <8 x i64> %c,  <8 x
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vor-vp.ll
index 09c281b525a643960a17c1db3e0e5a9a65aca079..e8a10fb88642491fc49e5ece8dcec01b7057933c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vor-vp.ll
@@ -913,8 +913,8 @@ define <2 x i64> @vor_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -939,8 +939,8 @@ define <2 x i64> @vor_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl)
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1007,8 +1007,8 @@ define <4 x i64> @vor_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1033,8 +1033,8 @@ define <4 x i64> @vor_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl)
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1101,8 +1101,8 @@ define <8 x i64> @vor_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1127,8 +1127,8 @@ define <8 x i64> @vor_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl)
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1195,8 +1195,8 @@ define <16 x i64> @vor_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1221,8 +1221,8 @@ define <16 x i64> @vor_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll
index 2913cbdf0fffd220542da46230cec0201194cfce..a31405f75e8a4e685c9f46bdbfd7f3df7ffaa426 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splat.ll
@@ -187,8 +187,8 @@ define <1 x i64> @vp_splat_v1i64(i64 %val, <1 x i1> %m, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -209,8 +209,8 @@ define <2 x i64> @vp_splat_v2i64(i64 %val, <2 x i1> %m, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -231,8 +231,8 @@ define <4 x i64> @vp_splat_v4i64(i64 %val, <4 x i1> %m, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -253,8 +253,8 @@ define <8 x i64> @vp_splat_v8i64(i64 %val, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
index bdf76dc63ddd8563f4c84daf372f84d8cada0d39..df1c84a9e05d816e24dd8c7b749db28fa335d496 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpmerge.ll
@@ -566,8 +566,8 @@ define <2 x i64> @vpmerge_vx_v2i64(i64 %a, <2 x i64> %vb, <2 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -586,8 +586,8 @@ define <2 x i64> @vpmerge_vx_v2i64(i64 %a, <2 x i64> %vb, <2 x i1> %m, i32 zeroe
 ; RV32ZVFHMIN:       # %bb.0:
 ; RV32ZVFHMIN-NEXT:    addi sp, sp, -16
 ; RV32ZVFHMIN-NEXT:    .cfi_def_cfa_offset 16
-; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    sw a0, 8(sp)
+; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    addi a0, sp, 8
 ; RV32ZVFHMIN-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32ZVFHMIN-NEXT:    vlse64.v v9, (a0), zero
@@ -635,8 +635,8 @@ define <4 x i64> @vpmerge_vx_v4i64(i64 %a, <4 x i64> %vb, <4 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -655,8 +655,8 @@ define <4 x i64> @vpmerge_vx_v4i64(i64 %a, <4 x i64> %vb, <4 x i1> %m, i32 zeroe
 ; RV32ZVFHMIN:       # %bb.0:
 ; RV32ZVFHMIN-NEXT:    addi sp, sp, -16
 ; RV32ZVFHMIN-NEXT:    .cfi_def_cfa_offset 16
-; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    sw a0, 8(sp)
+; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    addi a0, sp, 8
 ; RV32ZVFHMIN-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32ZVFHMIN-NEXT:    vlse64.v v10, (a0), zero
@@ -704,8 +704,8 @@ define <8 x i64> @vpmerge_vx_v8i64(i64 %a, <8 x i64> %vb, <8 x i1> %m, i32 zeroe
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -724,8 +724,8 @@ define <8 x i64> @vpmerge_vx_v8i64(i64 %a, <8 x i64> %vb, <8 x i1> %m, i32 zeroe
 ; RV32ZVFHMIN:       # %bb.0:
 ; RV32ZVFHMIN-NEXT:    addi sp, sp, -16
 ; RV32ZVFHMIN-NEXT:    .cfi_def_cfa_offset 16
-; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    sw a0, 8(sp)
+; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    addi a0, sp, 8
 ; RV32ZVFHMIN-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32ZVFHMIN-NEXT:    vlse64.v v12, (a0), zero
@@ -773,8 +773,8 @@ define <16 x i64> @vpmerge_vx_v16i64(i64 %a, <16 x i64> %vb, <16 x i1> %m, i32 z
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -793,8 +793,8 @@ define <16 x i64> @vpmerge_vx_v16i64(i64 %a, <16 x i64> %vb, <16 x i1> %m, i32 z
 ; RV32ZVFHMIN:       # %bb.0:
 ; RV32ZVFHMIN-NEXT:    addi sp, sp, -16
 ; RV32ZVFHMIN-NEXT:    .cfi_def_cfa_offset 16
-; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    sw a0, 8(sp)
+; RV32ZVFHMIN-NEXT:    sw a1, 12(sp)
 ; RV32ZVFHMIN-NEXT:    addi a0, sp, 8
 ; RV32ZVFHMIN-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32ZVFHMIN-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrem-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrem-vp.ll
index aa76324f3804f1b9b21b851b87e45d1700b76a3c..2ce6df5ce197ea6cad7b2d8fca4292931ea0faf9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrem-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrem-vp.ll
@@ -611,8 +611,8 @@ define <2 x i64> @vrem_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -637,8 +637,8 @@ define <2 x i64> @vrem_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -685,8 +685,8 @@ define <4 x i64> @vrem_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -711,8 +711,8 @@ define <4 x i64> @vrem_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -759,8 +759,8 @@ define <8 x i64> @vrem_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -785,8 +785,8 @@ define <8 x i64> @vrem_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -833,8 +833,8 @@ define <16 x i64> @vrem_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -859,8 +859,8 @@ define <16 x i64> @vrem_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vremu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vremu-vp.ll
index 24fa9357f91660c06ac0533ec2aa2e38565963bc..08ae37e13caaea92c6785f4020a70cecda96713a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vremu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vremu-vp.ll
@@ -610,8 +610,8 @@ define <2 x i64> @vremu_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -636,8 +636,8 @@ define <2 x i64> @vremu_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -684,8 +684,8 @@ define <4 x i64> @vremu_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -710,8 +710,8 @@ define <4 x i64> @vremu_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -758,8 +758,8 @@ define <8 x i64> @vremu_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -784,8 +784,8 @@ define <8 x i64> @vremu_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -832,8 +832,8 @@ define <16 x i64> @vremu_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zer
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -858,8 +858,8 @@ define <16 x i64> @vremu_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrsub-vp.ll
index 563482b88e8bdbc60d2a4c8b23b66a4cccfa92ae..bc2e0e5833f5cb6634694fcb495cca40798c72e3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vrsub-vp.ll
@@ -563,8 +563,8 @@ define <2 x i64> @vrsub_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -589,8 +589,8 @@ define <2 x i64> @vrsub_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -637,8 +637,8 @@ define <4 x i64> @vrsub_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -663,8 +663,8 @@ define <4 x i64> @vrsub_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -711,8 +711,8 @@ define <8 x i64> @vrsub_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -737,8 +737,8 @@ define <8 x i64> @vrsub_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -785,8 +785,8 @@ define <16 x i64> @vrsub_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zer
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -811,8 +811,8 @@ define <16 x i64> @vrsub_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
index 5030fda9dea3313a8faec1bf17ddca5d039e976a..056a7f44c9da2dfaab827929082bed997d9e978b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
@@ -1005,8 +1005,8 @@ define <2 x i64> @vsadd_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1031,8 +1031,8 @@ define <2 x i64> @vsadd_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1099,8 +1099,8 @@ define <4 x i64> @vsadd_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1125,8 +1125,8 @@ define <4 x i64> @vsadd_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1193,8 +1193,8 @@ define <8 x i64> @vsadd_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1219,8 +1219,8 @@ define <8 x i64> @vsadd_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1287,8 +1287,8 @@ define <16 x i64> @vsadd_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zer
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1313,8 +1313,8 @@ define <16 x i64> @vsadd_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd.ll
index 741699289e0271e4fd9a0a84f7b377029b1a1b9d..f69675b4a24b0130ad9bba468e59272a4e2fb4e1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd.ll
@@ -441,8 +441,8 @@ define <2 x i64> @sadd_v2i64_vx(<2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -488,8 +488,8 @@ define <4 x i64> @sadd_v4i64_vx(<4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -535,8 +535,8 @@ define <8 x i64> @sadd_v8i64_vx(<8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -582,8 +582,8 @@ define <16 x i64> @sadd_v16i64_vx(<16 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
index 562399ea33e7a82f7002a7a074cbdd9473d0c749..b79ec290e3d3e1284e9ad7f336b7d94b2399e2aa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
@@ -1001,8 +1001,8 @@ define <2 x i64> @vsaddu_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1027,8 +1027,8 @@ define <2 x i64> @vsaddu_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1095,8 +1095,8 @@ define <4 x i64> @vsaddu_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1121,8 +1121,8 @@ define <4 x i64> @vsaddu_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1189,8 +1189,8 @@ define <8 x i64> @vsaddu_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1215,8 +1215,8 @@ define <8 x i64> @vsaddu_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1283,8 +1283,8 @@ define <16 x i64> @vsaddu_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 ze
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1309,8 +1309,8 @@ define <16 x i64> @vsaddu_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu.ll
index 7b2cab294aa4965d439f528b9d93e6c70d9b8c1a..7397147a7580c15bee5e56d3d43b26ce5468dc46 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu.ll
@@ -441,8 +441,8 @@ define <2 x i64> @uadd_v2i64_vx(<2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -488,8 +488,8 @@ define <4 x i64> @uadd_v4i64_vx(<4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -535,8 +535,8 @@ define <8 x i64> @uadd_v8i64_vx(<8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -582,8 +582,8 @@ define <16 x i64> @uadd_v16i64_vx(<16 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
index 549c6ca11e320e3a2396d59c8a8e863267cb5628..c989736378479e2e89778bf7f705aa9d6051a17a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
@@ -1037,8 +1037,8 @@ define <2 x i64> @vssub_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1063,8 +1063,8 @@ define <2 x i64> @vssub_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1133,8 +1133,8 @@ define <4 x i64> @vssub_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1159,8 +1159,8 @@ define <4 x i64> @vssub_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1229,8 +1229,8 @@ define <8 x i64> @vssub_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1255,8 +1255,8 @@ define <8 x i64> @vssub_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %ev
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1325,8 +1325,8 @@ define <16 x i64> @vssub_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zer
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1351,8 +1351,8 @@ define <16 x i64> @vssub_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub.ll
index efe28eb9021ce23a3d42564907890355f5d3f1e0..e1746a31847da5839141f09fd085b79b84df4e1d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub.ll
@@ -441,8 +441,8 @@ define <2 x i64> @ssub_v2i64_vx(<2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -489,8 +489,8 @@ define <4 x i64> @ssub_v4i64_vx(<4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -537,8 +537,8 @@ define <8 x i64> @ssub_v8i64_vx(<8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -585,8 +585,8 @@ define <16 x i64> @ssub_v16i64_vx(<16 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
index 683f1150310b395c9416f92e49a6346aafab35f1..1fc2a18631001b6faf6cb22d613a6676d1432d41 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
@@ -1032,8 +1032,8 @@ define <2 x i64> @vssubu_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1058,8 +1058,8 @@ define <2 x i64> @vssubu_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1128,8 +1128,8 @@ define <4 x i64> @vssubu_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1154,8 +1154,8 @@ define <4 x i64> @vssubu_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1224,8 +1224,8 @@ define <8 x i64> @vssubu_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroex
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1250,8 +1250,8 @@ define <8 x i64> @vssubu_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %e
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1320,8 +1320,8 @@ define <16 x i64> @vssubu_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 ze
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1346,8 +1346,8 @@ define <16 x i64> @vssubu_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu.ll
index dc9279f6e7fa09c8c1c296920013c3186afb4042..28c29d2960b2686c99a1837bb90a397a03c3ca2a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu.ll
@@ -441,8 +441,8 @@ define <2 x i64> @usub_v2i64_vx(<2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -489,8 +489,8 @@ define <4 x i64> @usub_v4i64_vx(<4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -537,8 +537,8 @@ define <8 x i64> @usub_v8i64_vx(<8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -585,8 +585,8 @@ define <16 x i64> @usub_v16i64_vx(<16 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsub-vp.ll
index 6052c9ee20fe102b672a0b5d58c65c449e33ef03..7f1ad26bcda7a30d2f4d3212e351649d6266d967 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsub-vp.ll
@@ -641,8 +641,8 @@ define <2 x i64> @vsub_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -667,8 +667,8 @@ define <2 x i64> @vsub_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -715,8 +715,8 @@ define <4 x i64> @vsub_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -741,8 +741,8 @@ define <4 x i64> @vsub_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -789,8 +789,8 @@ define <8 x i64> @vsub_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -815,8 +815,8 @@ define <8 x i64> @vsub_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -863,8 +863,8 @@ define <16 x i64> @vsub_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -889,8 +889,8 @@ define <16 x i64> @vsub_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll
index 3a222e95566a48f7cd13053cd136f16439897fa1..50184796b38f534182089eed1b22357585d0d7ee 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll
@@ -824,12 +824,12 @@ define <2 x i64> @vwadd_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwadd_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    lw a2, 4(a1)
-; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lw a2, 0(a1)
+; RV32-NEXT:    lw a1, 4(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw a2, 12(sp)
-; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwadd.wv v8, v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
index 1fc6af2d4cc1c63a338664825dce25015f8cc9e6..98f246b8741dcc88bd57beae0dabcd75c6e0a190 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
@@ -773,8 +773,8 @@ define <2 x i64> @vwaddu_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 ; RV32-NEXT:    lbu a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwaddu.wv v8, v8, v9
@@ -805,8 +805,8 @@ define <2 x i64> @vwaddu_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 ; RV32-NEXT:    lhu a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwaddu.wv v8, v8, v9
@@ -837,8 +837,8 @@ define <2 x i64> @vwaddu_vx_v2i64_i32(ptr %x, ptr %y) nounwind {
 ; RV32-NEXT:    lw a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwaddu.wv v8, v8, v9
@@ -866,12 +866,12 @@ define <2 x i64> @vwaddu_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwaddu_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    lw a2, 4(a1)
-; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lw a2, 0(a1)
+; RV32-NEXT:    lw a1, 4(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw a2, 12(sp)
-; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwaddu.wv v8, v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
index 97c7f101c258275f402274701696fd3099aabe1b..01f2fe506e85f77b00a4874a04b67419621eef55 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
@@ -853,12 +853,12 @@ define <2 x i64> @vwmul_vx_v2i64_i64(ptr %x, ptr %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    lw a2, 4(a1)
-; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lw a2, 0(a1)
+; RV32-NEXT:    lw a1, 4(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    sw a2, 12(sp)
-; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
 ; RV32-NEXT:    vsext.vf2 v10, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll
index c73b3a0dce6be4b02d228fda7794e04a0129d368..db2f544ab30669650412760b2855344e5ff46c3d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll
@@ -794,8 +794,8 @@ define <2 x i64> @vwmulsu_vx_v2i64_i8(ptr %x, ptr %y) {
 ; RV32-NEXT:    lbu a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
 ; RV32-NEXT:    vsext.vf2 v10, v8
@@ -828,8 +828,8 @@ define <2 x i64> @vwmulsu_vx_v2i64_i16(ptr %x, ptr %y) {
 ; RV32-NEXT:    lhu a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
 ; RV32-NEXT:    vsext.vf2 v10, v8
@@ -862,8 +862,8 @@ define <2 x i64> @vwmulsu_vx_v2i64_i32(ptr %x, ptr %y) {
 ; RV32-NEXT:    lw a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
 ; RV32-NEXT:    vsext.vf2 v10, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll
index d632dc4c2a30d27a783edbb75e05f2cc30525f01..7a925165d98163968c7ec1489ef971a59fa21315 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll
@@ -830,12 +830,12 @@ define <2 x i64> @vwsub_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsub_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    lw a2, 4(a1)
-; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lw a2, 0(a1)
+; RV32-NEXT:    lw a1, 4(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw a2, 12(sp)
-; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwsub.wv v8, v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll
index 9b5f4a5012f4ed6e7799a5164cc559eeaf2af231..4c08a8c15a388e0a4b5c8f21157a396480ff18c7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll
@@ -776,8 +776,8 @@ define <2 x i64> @vwsubu_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 ; RV32-NEXT:    lbu a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwsubu.wv v8, v8, v9
@@ -809,8 +809,8 @@ define <2 x i64> @vwsubu_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 ; RV32-NEXT:    lhu a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwsubu.wv v8, v8, v9
@@ -842,8 +842,8 @@ define <2 x i64> @vwsubu_vx_v2i64_i32(ptr %x, ptr %y) nounwind {
 ; RV32-NEXT:    lw a1, 0(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwsubu.wv v8, v8, v9
@@ -872,12 +872,12 @@ define <2 x i64> @vwsubu_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsubu_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    lw a2, 4(a1)
-; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lw a2, 0(a1)
+; RV32-NEXT:    lw a1, 4(a1)
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
-; RV32-NEXT:    sw a2, 12(sp)
-; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a2, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
 ; RV32-NEXT:    vwsubu.wv v8, v8, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vxor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vxor-vp.ll
index 16487a0784125e85227545b70b6c97b1900ea58b..677b4ea6e91eca66ab554a72e89331f84c55af20 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vxor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vxor-vp.ll
@@ -1173,8 +1173,8 @@ define <2 x i64> @vxor_vx_v2i64(<2 x i64> %va, i64 %b, <2 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1199,8 +1199,8 @@ define <2 x i64> @vxor_vx_v2i64_unmasked(<2 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1287,8 +1287,8 @@ define <4 x i64> @vxor_vx_v4i64(<4 x i64> %va, i64 %b, <4 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1313,8 +1313,8 @@ define <4 x i64> @vxor_vx_v4i64_unmasked(<4 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1401,8 +1401,8 @@ define <8 x i64> @vxor_vx_v8i64(<8 x i64> %va, i64 %b, <8 x i1> %m, i32 zeroext
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1427,8 +1427,8 @@ define <8 x i64> @vxor_vx_v8i64_unmasked(<8 x i64> %va, i64 %b, i32 zeroext %evl
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1515,8 +1515,8 @@ define <16 x i64> @vxor_vx_v16i64(<16 x i64> %va, i64 %b, <16 x i1> %m, i32 zero
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1541,8 +1541,8 @@ define <16 x i64> @vxor_vx_v16i64_unmasked(<16 x i64> %va, i64 %b, i32 zeroext %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-xsfvcp-xvv.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-xsfvcp-xvv.ll
index 44ffffc7e59d5b2bbf647e76bebaa751206543dd..e44ff31406f4af9fa2aed2d88647873558a9c050 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-xsfvcp-xvv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-xsfvcp-xvv.ll
@@ -293,7 +293,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv8i64.nxv8i64.iXLen(iXLen, <8 x i6
 define <1 x i8> @test_sf_vc_v_vvv_se_e8mf8(<1 x i8> %vd, <1 x i8> %vs2, <1 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -306,7 +306,7 @@ declare <1 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv1i8.iXLen.nxv1i8.iXLen(iXLen, <1
 define <2 x i8> @test_sf_vc_v_vvv_se_e8mf4(<2 x i8> %vd, <2 x i8> %vs2, <2 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -319,7 +319,7 @@ declare <2 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv2i8.iXLen.nxv2i8.iXLen(iXLen, <2
 define <4 x i8> @test_sf_vc_v_vvv_se_e8mf2(<4 x i8> %vd, <4 x i8> %vs2, <4 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -332,7 +332,7 @@ declare <4 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv4i8.iXLen.nxv4i8.iXLen(iXLen, <4
 define <8 x i8> @test_sf_vc_v_vvv_se_e8m1(<8 x i8> %vd, <8 x i8> %vs2, <8 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -345,7 +345,7 @@ declare <8 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv8i8.iXLen.nxv8i8.iXLen(iXLen, <8
 define <16 x i8> @test_sf_vc_v_vvv_se_e8m2(<16 x i8> %vd, <16 x i8> %vs2, <16 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -358,7 +358,7 @@ declare <16 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv16i8.iXLen.nxv16i8.iXLen(iXLen,
 define <32 x i8> @test_sf_vc_v_vvv_se_e8m4(<32 x i8> %vd, <32 x i8> %vs2, <32 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -371,7 +371,7 @@ declare <32 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv32i8.iXLen.nxv32i8.iXLen(iXLen,
 define <64 x i8> @test_sf_vc_v_vvv_se_e8m8(<64 x i8> %vd, <64 x i8> %vs2, <64 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -384,7 +384,7 @@ declare <64 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv64i8.iXLen.nxv64i8.iXLen(iXLen,
 define <1 x i16> @test_sf_vc_v_vvv_se_e16mf4(<1 x i16> %vd, <1 x i16> %vs2, <1 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -397,7 +397,7 @@ declare <1 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv1i16.iXLen.nxv1i16.iXLen(iXLen,
 define <2 x i16> @test_sf_vc_v_vvv_se_e16mf2(<2 x i16> %vd, <2 x i16> %vs2, <2 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -410,7 +410,7 @@ declare <2 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv2i16.iXLen.nxv2i16.iXLen(iXLen,
 define <4 x i16> @test_sf_vc_v_vvv_se_e16m1(<4 x i16> %vd, <4 x i16> %vs2, <4 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -423,7 +423,7 @@ declare <4 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv4i16.iXLen.nxv4i16.iXLen(iXLen,
 define <8 x i16> @test_sf_vc_v_vvv_se_e16m2(<8 x i16> %vd, <8 x i16> %vs2, <8 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -436,7 +436,7 @@ declare <8 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv8i16.iXLen.nxv8i16.iXLen(iXLen,
 define <16 x i16> @test_sf_vc_v_vvv_se_e16m4(<16 x i16> %vd, <16 x i16> %vs2, <16 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -449,7 +449,7 @@ declare <16 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv16i16.iXLen.nxv16i16.iXLen(iXLe
 define <32 x i16> @test_sf_vc_v_vvv_se_e16m8(<32 x i16> %vd, <32 x i16> %vs2, <32 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -462,7 +462,7 @@ declare <32 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv32i16.iXLen.nxv32i16.iXLen(iXLe
 define <1 x i32> @test_sf_vc_v_vvv_se_e32mf2(<1 x i32> %vd, <1 x i32> %vs2, <1 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -475,7 +475,7 @@ declare <1 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv1i32.iXLen.nxv1i32.iXLen(iXLen,
 define <2 x i32> @test_sf_vc_v_vvv_se_e32m1(<2 x i32> %vd, <2 x i32> %vs2, <2 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -488,7 +488,7 @@ declare <2 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv2i32.iXLen.nxv2i32.iXLen(iXLen,
 define <4 x i32> @test_sf_vc_v_vvv_se_e32m2(<4 x i32> %vd, <4 x i32> %vs2, <4 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -501,7 +501,7 @@ declare <4 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv4i32.iXLen.nxv4i32.iXLen(iXLen,
 define <8 x i32> @test_sf_vc_v_vvv_se_e32m4(<8 x i32> %vd, <8 x i32> %vs2, <8 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -514,7 +514,7 @@ declare <8 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv8i32.iXLen.nxv8i32.iXLen(iXLen,
 define <16 x i32> @test_sf_vc_v_vvv_se_e32m8(<16 x i32> %vd, <16 x i32> %vs2, <16 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -527,7 +527,7 @@ declare <16 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv16i32.iXLen.nxv16i32.iXLen(iXLe
 define <1 x i64> @test_sf_vc_v_vvv_se_e64m1(<1 x i64> %vd, <1 x i64> %vs2, <1 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -540,7 +540,7 @@ declare <1 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv1i64.iXLen.nxv1i64.iXLen(iXLen,
 define <2 x i64> @test_sf_vc_v_vvv_se_e64m2(<2 x i64> %vd, <2 x i64> %vs2, <2 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -553,7 +553,7 @@ declare <2 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv2i64.iXLen.nxv2i64.iXLen(iXLen,
 define <4 x i64> @test_sf_vc_v_vvv_se_e64m4(<4 x i64> %vd, <4 x i64> %vs2, <4 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -566,7 +566,7 @@ declare <4 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv4i64.iXLen.nxv4i64.iXLen(iXLen,
 define <8 x i64> @test_sf_vc_v_vvv_se_e64m8(<8 x i64> %vd, <8 x i64> %vs2, <8 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -579,7 +579,7 @@ declare <8 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv8i64.iXLen.nxv8i64.iXLen(iXLen,
 define <1 x i8> @test_sf_vc_v_vvv_e8mf8(<1 x i8> %vd, <1 x i8> %vs2, <1 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -592,7 +592,7 @@ declare <1 x i8> @llvm.riscv.sf.vc.v.vvv.nxv1i8.iXLen.nxv1i8.iXLen(iXLen, <1 x i
 define <2 x i8> @test_sf_vc_v_vvv_e8mf4(<2 x i8> %vd, <2 x i8> %vs2, <2 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -605,7 +605,7 @@ declare <2 x i8> @llvm.riscv.sf.vc.v.vvv.nxv2i8.iXLen.nxv2i8.iXLen(iXLen, <2 x i
 define <4 x i8> @test_sf_vc_v_vvv_e8mf2(<4 x i8> %vd, <4 x i8> %vs2, <4 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -618,7 +618,7 @@ declare <4 x i8> @llvm.riscv.sf.vc.v.vvv.nxv4i8.iXLen.nxv4i8.iXLen(iXLen, <4 x i
 define <8 x i8> @test_sf_vc_v_vvv_e8m1(<8 x i8> %vd, <8 x i8> %vs2, <8 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -631,7 +631,7 @@ declare <8 x i8> @llvm.riscv.sf.vc.v.vvv.nxv8i8.iXLen.nxv8i8.iXLen(iXLen, <8 x i
 define <16 x i8> @test_sf_vc_v_vvv_e8m2(<16 x i8> %vd, <16 x i8> %vs2, <16 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -644,7 +644,7 @@ declare <16 x i8> @llvm.riscv.sf.vc.v.vvv.nxv16i8.iXLen.nxv16i8.iXLen(iXLen, <16
 define <32 x i8> @test_sf_vc_v_vvv_e8m4(<32 x i8> %vd, <32 x i8> %vs2, <32 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -657,7 +657,7 @@ declare <32 x i8> @llvm.riscv.sf.vc.v.vvv.nxv32i8.iXLen.nxv32i8.iXLen(iXLen, <32
 define <64 x i8> @test_sf_vc_v_vvv_e8m8(<64 x i8> %vd, <64 x i8> %vs2, <64 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -670,7 +670,7 @@ declare <64 x i8> @llvm.riscv.sf.vc.v.vvv.nxv64i8.iXLen.nxv64i8.iXLen(iXLen, <64
 define <1 x i16> @test_sf_vc_v_vvv_e16mf4(<1 x i16> %vd, <1 x i16> %vs2, <1 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -683,7 +683,7 @@ declare <1 x i16> @llvm.riscv.sf.vc.v.vvv.nxv1i16.iXLen.nxv1i16.iXLen(iXLen, <1
 define <2 x i16> @test_sf_vc_v_vvv_e16mf2(<2 x i16> %vd, <2 x i16> %vs2, <2 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -696,7 +696,7 @@ declare <2 x i16> @llvm.riscv.sf.vc.v.vvv.nxv2i16.iXLen.nxv2i16.iXLen(iXLen, <2
 define <4 x i16> @test_sf_vc_v_vvv_e16m1(<4 x i16> %vd, <4 x i16> %vs2, <4 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -709,7 +709,7 @@ declare <4 x i16> @llvm.riscv.sf.vc.v.vvv.nxv4i16.iXLen.nxv4i16.iXLen(iXLen, <4
 define <8 x i16> @test_sf_vc_v_vvv_e16m2(<8 x i16> %vd, <8 x i16> %vs2, <8 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -722,7 +722,7 @@ declare <8 x i16> @llvm.riscv.sf.vc.v.vvv.nxv8i16.iXLen.nxv8i16.iXLen(iXLen, <8
 define <16 x i16> @test_sf_vc_v_vvv_e16m4(<16 x i16> %vd, <16 x i16> %vs2, <16 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -735,7 +735,7 @@ declare <16 x i16> @llvm.riscv.sf.vc.v.vvv.nxv16i16.iXLen.nxv16i16.iXLen(iXLen,
 define <32 x i16> @test_sf_vc_v_vvv_e16m8(<32 x i16> %vd, <32 x i16> %vs2, <32 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -748,7 +748,7 @@ declare <32 x i16> @llvm.riscv.sf.vc.v.vvv.nxv32i16.iXLen.nxv32i16.iXLen(iXLen,
 define <1 x i32> @test_sf_vc_v_vvv_e32mf2(<1 x i32> %vd, <1 x i32> %vs2, <1 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -761,7 +761,7 @@ declare <1 x i32> @llvm.riscv.sf.vc.v.vvv.nxv1i32.iXLen.nxv1i32.iXLen(iXLen, <1
 define <2 x i32> @test_sf_vc_v_vvv_e32m1(<2 x i32> %vd, <2 x i32> %vs2, <2 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -774,7 +774,7 @@ declare <2 x i32> @llvm.riscv.sf.vc.v.vvv.nxv2i32.iXLen.nxv2i32.iXLen(iXLen, <2
 define <4 x i32> @test_sf_vc_v_vvv_e32m2(<4 x i32> %vd, <4 x i32> %vs2, <4 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -787,7 +787,7 @@ declare <4 x i32> @llvm.riscv.sf.vc.v.vvv.nxv4i32.iXLen.nxv4i32.iXLen(iXLen, <4
 define <8 x i32> @test_sf_vc_v_vvv_e32m4(<8 x i32> %vd, <8 x i32> %vs2, <8 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -800,7 +800,7 @@ declare <8 x i32> @llvm.riscv.sf.vc.v.vvv.nxv8i32.iXLen.nxv8i32.iXLen(iXLen, <8
 define <16 x i32> @test_sf_vc_v_vvv_e32m8(<16 x i32> %vd, <16 x i32> %vs2, <16 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -813,7 +813,7 @@ declare <16 x i32> @llvm.riscv.sf.vc.v.vvv.nxv16i32.iXLen.nxv16i32.iXLen(iXLen,
 define <1 x i64> @test_sf_vc_v_vvv_e64m1(<1 x i64> %vd, <1 x i64> %vs2, <1 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -826,7 +826,7 @@ declare <1 x i64> @llvm.riscv.sf.vc.v.vvv.nxv1i64.iXLen.nxv1i64.iXLen(iXLen, <1
 define <2 x i64> @test_sf_vc_v_vvv_e64m2(<2 x i64> %vd, <2 x i64> %vs2, <2 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -839,7 +839,7 @@ declare <2 x i64> @llvm.riscv.sf.vc.v.vvv.nxv2i64.iXLen.nxv2i64.iXLen(iXLen, <2
 define <4 x i64> @test_sf_vc_v_vvv_e64m4(<4 x i64> %vd, <4 x i64> %vs2, <4 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -852,7 +852,7 @@ declare <4 x i64> @llvm.riscv.sf.vc.v.vvv.nxv4i64.iXLen.nxv4i64.iXLen(iXLen, <4
 define <8 x i64> @test_sf_vc_v_vvv_e64m8(<8 x i64> %vd, <8 x i64> %vs2, <8 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1099,7 +1099,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.i32.nxv16i32.iXLen.iXLen(iXLen, <16 x i32>
 define <1 x i8> @test_sf_vc_v_xvv_se_e8mf8(<1 x i8> %vd, <1 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1112,7 +1112,7 @@ declare <1 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv1i8.iXLen.i8.iXLen(iXLen, <1 x i8
 define <2 x i8> @test_sf_vc_v_xvv_se_e8mf4(<2 x i8> %vd, <2 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1125,7 +1125,7 @@ declare <2 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv2i8.iXLen.i8.iXLen(iXLen, <2 x i8
 define <4 x i8> @test_sf_vc_v_xvv_se_e8mf2(<4 x i8> %vd, <4 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1138,7 +1138,7 @@ declare <4 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv4i8.iXLen.i8.iXLen(iXLen, <4 x i8
 define <8 x i8> @test_sf_vc_v_xvv_se_e8m1(<8 x i8> %vd, <8 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1151,7 +1151,7 @@ declare <8 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv8i8.iXLen.i8.iXLen(iXLen, <8 x i8
 define <16 x i8> @test_sf_vc_v_xvv_se_e8m2(<16 x i8> %vd, <16 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1164,7 +1164,7 @@ declare <16 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv16i8.iXLen.i8.iXLen(iXLen, <16 x
 define <32 x i8> @test_sf_vc_v_xvv_se_e8m4(<32 x i8> %vd, <32 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1177,7 +1177,7 @@ declare <32 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv32i8.iXLen.i8.iXLen(iXLen, <32 x
 define <64 x i8> @test_sf_vc_v_xvv_se_e8m8(<64 x i8> %vd, <64 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1190,7 +1190,7 @@ declare <64 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv64i8.iXLen.i8.iXLen(iXLen, <64 x
 define <1 x i16> @test_sf_vc_v_xvv_se_e16mf4(<1 x i16> %vd, <1 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1203,7 +1203,7 @@ declare <1 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv1i16.iXLen.i16.iXLen(iXLen, <1 x
 define <2 x i16> @test_sf_vc_v_xvv_se_e16mf2(<2 x i16> %vd, <2 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1216,7 +1216,7 @@ declare <2 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv2i16.iXLen.i16.iXLen(iXLen, <2 x
 define <4 x i16> @test_sf_vc_v_xvv_se_e16m1(<4 x i16> %vd, <4 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1229,7 +1229,7 @@ declare <4 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv4i16.iXLen.i16.iXLen(iXLen, <4 x
 define <8 x i16> @test_sf_vc_v_xvv_se_e16m2(<8 x i16> %vd, <8 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1242,7 +1242,7 @@ declare <8 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv8i16.iXLen.i16.iXLen(iXLen, <8 x
 define <16 x i16> @test_sf_vc_v_xvv_se_e16m4(<16 x i16> %vd, <16 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1255,7 +1255,7 @@ declare <16 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv16i16.iXLen.i16.iXLen(iXLen, <1
 define <32 x i16> @test_sf_vc_v_xvv_se_e16m8(<32 x i16> %vd, <32 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1268,7 +1268,7 @@ declare <32 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv32i16.iXLen.i16.iXLen(iXLen, <3
 define <1 x i32> @test_sf_vc_v_xvv_se_e32mf2(<1 x i32> %vd, <1 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1281,7 +1281,7 @@ declare <1 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv1i32.iXLen.i32.iXLen(iXLen, <1 x
 define <2 x i32> @test_sf_vc_v_xvv_se_e32m1(<2 x i32> %vd, <2 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1294,7 +1294,7 @@ declare <2 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv2i32.iXLen.i32.iXLen(iXLen, <2 x
 define <4 x i32> @test_sf_vc_v_xvv_se_e32m2(<4 x i32> %vd, <4 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1307,7 +1307,7 @@ declare <4 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv4i32.iXLen.i32.iXLen(iXLen, <4 x
 define <8 x i32> @test_sf_vc_v_xvv_se_e32m4(<8 x i32> %vd, <8 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1320,7 +1320,7 @@ declare <8 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv8i32.iXLen.i32.iXLen(iXLen, <8 x
 define <16 x i32> @test_sf_vc_v_xvv_se_e32m8(<16 x i32> %vd, <16 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1333,7 +1333,7 @@ declare <16 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv16i32.iXLen.i32.iXLen(iXLen, <1
 define <1 x i8> @test_sf_vc_v_xvv_e8mf8(<1 x i8> %vd, <1 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1346,7 +1346,7 @@ declare <1 x i8> @llvm.riscv.sf.vc.v.xvv.nxv1i8.iXLen.i8.iXLen(iXLen, <1 x i8>,
 define <2 x i8> @test_sf_vc_v_xvv_e8mf4(<2 x i8> %vd, <2 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1359,7 +1359,7 @@ declare <2 x i8> @llvm.riscv.sf.vc.v.xvv.nxv2i8.iXLen.i8.iXLen(iXLen, <2 x i8>,
 define <4 x i8> @test_sf_vc_v_xvv_e8mf2(<4 x i8> %vd, <4 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1372,7 +1372,7 @@ declare <4 x i8> @llvm.riscv.sf.vc.v.xvv.nxv4i8.iXLen.i8.iXLen(iXLen, <4 x i8>,
 define <8 x i8> @test_sf_vc_v_xvv_e8m1(<8 x i8> %vd, <8 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1385,7 +1385,7 @@ declare <8 x i8> @llvm.riscv.sf.vc.v.xvv.nxv8i8.iXLen.i8.iXLen(iXLen, <8 x i8>,
 define <16 x i8> @test_sf_vc_v_xvv_e8m2(<16 x i8> %vd, <16 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1398,7 +1398,7 @@ declare <16 x i8> @llvm.riscv.sf.vc.v.xvv.nxv16i8.iXLen.i8.iXLen(iXLen, <16 x i8
 define <32 x i8> @test_sf_vc_v_xvv_e8m4(<32 x i8> %vd, <32 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1411,7 +1411,7 @@ declare <32 x i8> @llvm.riscv.sf.vc.v.xvv.nxv32i8.iXLen.i8.iXLen(iXLen, <32 x i8
 define <64 x i8> @test_sf_vc_v_xvv_e8m8(<64 x i8> %vd, <64 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1424,7 +1424,7 @@ declare <64 x i8> @llvm.riscv.sf.vc.v.xvv.nxv64i8.iXLen.i8.iXLen(iXLen, <64 x i8
 define <1 x i16> @test_sf_vc_v_xvv_e16mf4(<1 x i16> %vd, <1 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1437,7 +1437,7 @@ declare <1 x i16> @llvm.riscv.sf.vc.v.xvv.nxv1i16.iXLen.i16.iXLen(iXLen, <1 x i1
 define <2 x i16> @test_sf_vc_v_xvv_e16mf2(<2 x i16> %vd, <2 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1450,7 +1450,7 @@ declare <2 x i16> @llvm.riscv.sf.vc.v.xvv.nxv2i16.iXLen.i16.iXLen(iXLen, <2 x i1
 define <4 x i16> @test_sf_vc_v_xvv_e16m1(<4 x i16> %vd, <4 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1463,7 +1463,7 @@ declare <4 x i16> @llvm.riscv.sf.vc.v.xvv.nxv4i16.iXLen.i16.iXLen(iXLen, <4 x i1
 define <8 x i16> @test_sf_vc_v_xvv_e16m2(<8 x i16> %vd, <8 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1476,7 +1476,7 @@ declare <8 x i16> @llvm.riscv.sf.vc.v.xvv.nxv8i16.iXLen.i16.iXLen(iXLen, <8 x i1
 define <16 x i16> @test_sf_vc_v_xvv_e16m4(<16 x i16> %vd, <16 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1489,7 +1489,7 @@ declare <16 x i16> @llvm.riscv.sf.vc.v.xvv.nxv16i16.iXLen.i16.iXLen(iXLen, <16 x
 define <32 x i16> @test_sf_vc_v_xvv_e16m8(<32 x i16> %vd, <32 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1502,7 +1502,7 @@ declare <32 x i16> @llvm.riscv.sf.vc.v.xvv.nxv32i16.iXLen.i16.iXLen(iXLen, <32 x
 define <1 x i32> @test_sf_vc_v_xvv_e32mf2(<1 x i32> %vd, <1 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1515,7 +1515,7 @@ declare <1 x i32> @llvm.riscv.sf.vc.v.xvv.nxv1i32.iXLen.i32.iXLen(iXLen, <1 x i3
 define <2 x i32> @test_sf_vc_v_xvv_e32m1(<2 x i32> %vd, <2 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1528,7 +1528,7 @@ declare <2 x i32> @llvm.riscv.sf.vc.v.xvv.nxv2i32.iXLen.i32.iXLen(iXLen, <2 x i3
 define <4 x i32> @test_sf_vc_v_xvv_e32m2(<4 x i32> %vd, <4 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1541,7 +1541,7 @@ declare <4 x i32> @llvm.riscv.sf.vc.v.xvv.nxv4i32.iXLen.i32.iXLen(iXLen, <4 x i3
 define <8 x i32> @test_sf_vc_v_xvv_e32m4(<8 x i32> %vd, <8 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1554,7 +1554,7 @@ declare <8 x i32> @llvm.riscv.sf.vc.v.xvv.nxv8i32.iXLen.i32.iXLen(iXLen, <8 x i3
 define <16 x i32> @test_sf_vc_v_xvv_e32m8(<16 x i32> %vd, <16 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1853,7 +1853,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv8i64.iXLen.iXLen(iXLen, <8 x i64>
 define <1 x i8> @test_sf_vc_v_ivv_se_e8mf8(<1 x i8> %vd, <1 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1866,7 +1866,7 @@ declare <1 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv1i8.iXLen.iXLen.iXLen(iXLen, <1 x
 define <2 x i8> @test_sf_vc_v_ivv_se_e8mf4(<2 x i8> %vd, <2 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1879,7 +1879,7 @@ declare <2 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv2i8.iXLen.iXLen.iXLen(iXLen, <2 x
 define <4 x i8> @test_sf_vc_v_ivv_se_e8mf2(<4 x i8> %vd, <4 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1892,7 +1892,7 @@ declare <4 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv4i8.iXLen.iXLen.iXLen(iXLen, <4 x
 define <8 x i8> @test_sf_vc_v_ivv_se_e8m1(<8 x i8> %vd, <8 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1905,7 +1905,7 @@ declare <8 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv8i8.iXLen.iXLen.iXLen(iXLen, <8 x
 define <16 x i8> @test_sf_vc_v_ivv_se_e8m2(<16 x i8> %vd, <16 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1918,7 +1918,7 @@ declare <16 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv16i8.iXLen.iXLen.iXLen(iXLen, <1
 define <32 x i8> @test_sf_vc_v_ivv_se_e8m4(<32 x i8> %vd, <32 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1931,7 +1931,7 @@ declare <32 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv32i8.iXLen.iXLen.iXLen(iXLen, <3
 define <64 x i8> @test_sf_vc_v_ivv_se_e8m8(<64 x i8> %vd, <64 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1944,7 +1944,7 @@ declare <64 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv64i8.iXLen.iXLen.iXLen(iXLen, <6
 define <1 x i16> @test_sf_vc_v_ivv_se_e16mf4(<1 x i16> %vd, <1 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1957,7 +1957,7 @@ declare <1 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv1i16.iXLen.iXLen.iXLen(iXLen, <1
 define <2 x i16> @test_sf_vc_v_ivv_se_e16mf2(<2 x i16> %vd, <2 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1970,7 +1970,7 @@ declare <2 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv2i16.iXLen.iXLen.iXLen(iXLen, <2
 define <4 x i16> @test_sf_vc_v_ivv_se_e16m1(<4 x i16> %vd, <4 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1983,7 +1983,7 @@ declare <4 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv4i16.iXLen.iXLen.iXLen(iXLen, <4
 define <8 x i16> @test_sf_vc_v_ivv_se_e16m2(<8 x i16> %vd, <8 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1996,7 +1996,7 @@ declare <8 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv8i16.iXLen.iXLen.iXLen(iXLen, <8
 define <16 x i16> @test_sf_vc_v_ivv_se_e16m4(<16 x i16> %vd, <16 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2009,7 +2009,7 @@ declare <16 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv16i16.iXLen.iXLen.iXLen(iXLen,
 define <32 x i16> @test_sf_vc_v_ivv_se_e16m8(<32 x i16> %vd, <32 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2022,7 +2022,7 @@ declare <32 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv32i16.iXLen.iXLen.iXLen(iXLen,
 define <1 x i32> @test_sf_vc_v_ivv_se_e32mf2(<1 x i32> %vd, <1 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2035,7 +2035,7 @@ declare <1 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv1i32.iXLen.iXLen.iXLen(iXLen, <1
 define <2 x i32> @test_sf_vc_v_ivv_se_e32m1(<2 x i32> %vd, <2 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2048,7 +2048,7 @@ declare <2 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv2i32.iXLen.iXLen.iXLen(iXLen, <2
 define <4 x i32> @test_sf_vc_v_ivv_se_e32m2(<4 x i32> %vd, <4 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2061,7 +2061,7 @@ declare <4 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv4i32.iXLen.iXLen.iXLen(iXLen, <4
 define <8 x i32> @test_sf_vc_v_ivv_se_e32m4(<8 x i32> %vd, <8 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2074,7 +2074,7 @@ declare <8 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv8i32.iXLen.iXLen.iXLen(iXLen, <8
 define <16 x i32> @test_sf_vc_v_ivv_se_e32m8(<16 x i32> %vd, <16 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2087,7 +2087,7 @@ declare <16 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv16i32.iXLen.iXLen.iXLen(iXLen,
 define <1 x i64> @test_sf_vc_v_ivv_se_e64m1(<1 x i64> %vd, <1 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2100,7 +2100,7 @@ declare <1 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv1i64.iXLen.iXLen.iXLen(iXLen, <1
 define <2 x i64> @test_sf_vc_v_ivv_se_e64m2(<2 x i64> %vd, <2 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2113,7 +2113,7 @@ declare <2 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv2i64.iXLen.iXLen.iXLen(iXLen, <2
 define <4 x i64> @test_sf_vc_v_ivv_se_e64m4(<4 x i64> %vd, <4 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2126,7 +2126,7 @@ declare <4 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv4i64.iXLen.iXLen.iXLen(iXLen, <4
 define <8 x i64> @test_sf_vc_v_ivv_se_e64m8(<8 x i64> %vd, <8 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2139,7 +2139,7 @@ declare <8 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv8i64.iXLen.iXLen.iXLen(iXLen, <8
 define <1 x i8> @test_sf_vc_v_ivv_e8mf8(<1 x i8> %vd, <1 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2152,7 +2152,7 @@ declare <1 x i8> @llvm.riscv.sf.vc.v.ivv.nxv1i8.iXLen.iXLen.iXLen(iXLen, <1 x i8
 define <2 x i8> @test_sf_vc_v_ivv_e8mf4(<2 x i8> %vd, <2 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2165,7 +2165,7 @@ declare <2 x i8> @llvm.riscv.sf.vc.v.ivv.nxv2i8.iXLen.iXLen.iXLen(iXLen, <2 x i8
 define <4 x i8> @test_sf_vc_v_ivv_e8mf2(<4 x i8> %vd, <4 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2178,7 +2178,7 @@ declare <4 x i8> @llvm.riscv.sf.vc.v.ivv.nxv4i8.iXLen.iXLen.iXLen(iXLen, <4 x i8
 define <8 x i8> @test_sf_vc_v_ivv_e8m1(<8 x i8> %vd, <8 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2191,7 +2191,7 @@ declare <8 x i8> @llvm.riscv.sf.vc.v.ivv.nxv8i8.iXLen.iXLen.iXLen(iXLen, <8 x i8
 define <16 x i8> @test_sf_vc_v_ivv_e8m2(<16 x i8> %vd, <16 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2204,7 +2204,7 @@ declare <16 x i8> @llvm.riscv.sf.vc.v.ivv.nxv16i8.iXLen.iXLen.iXLen(iXLen, <16 x
 define <32 x i8> @test_sf_vc_v_ivv_e8m4(<32 x i8> %vd, <32 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2217,7 +2217,7 @@ declare <32 x i8> @llvm.riscv.sf.vc.v.ivv.nxv32i8.iXLen.iXLen.iXLen(iXLen, <32 x
 define <64 x i8> @test_sf_vc_v_ivv_e8m8(<64 x i8> %vd, <64 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2230,7 +2230,7 @@ declare <64 x i8> @llvm.riscv.sf.vc.v.ivv.nxv64i8.iXLen.iXLen.iXLen(iXLen, <64 x
 define <1 x i16> @test_sf_vc_v_ivv_e16mf4(<1 x i16> %vd, <1 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2243,7 +2243,7 @@ declare <1 x i16> @llvm.riscv.sf.vc.v.ivv.nxv1i16.iXLen.iXLen.iXLen(iXLen, <1 x
 define <2 x i16> @test_sf_vc_v_ivv_e16mf2(<2 x i16> %vd, <2 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2256,7 +2256,7 @@ declare <2 x i16> @llvm.riscv.sf.vc.v.ivv.nxv2i16.iXLen.iXLen.iXLen(iXLen, <2 x
 define <4 x i16> @test_sf_vc_v_ivv_e16m1(<4 x i16> %vd, <4 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2269,7 +2269,7 @@ declare <4 x i16> @llvm.riscv.sf.vc.v.ivv.nxv4i16.iXLen.iXLen.iXLen(iXLen, <4 x
 define <8 x i16> @test_sf_vc_v_ivv_e16m2(<8 x i16> %vd, <8 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2282,7 +2282,7 @@ declare <8 x i16> @llvm.riscv.sf.vc.v.ivv.nxv8i16.iXLen.iXLen.iXLen(iXLen, <8 x
 define <16 x i16> @test_sf_vc_v_ivv_e16m4(<16 x i16> %vd, <16 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2295,7 +2295,7 @@ declare <16 x i16> @llvm.riscv.sf.vc.v.ivv.nxv16i16.iXLen.iXLen.iXLen(iXLen, <16
 define <32 x i16> @test_sf_vc_v_ivv_e16m8(<32 x i16> %vd, <32 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2308,7 +2308,7 @@ declare <32 x i16> @llvm.riscv.sf.vc.v.ivv.nxv32i16.iXLen.iXLen.iXLen(iXLen, <32
 define <1 x i32> @test_sf_vc_v_ivv_e32mf2(<1 x i32> %vd, <1 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2321,7 +2321,7 @@ declare <1 x i32> @llvm.riscv.sf.vc.v.ivv.nxv1i32.iXLen.iXLen.iXLen(iXLen, <1 x
 define <2 x i32> @test_sf_vc_v_ivv_e32m1(<2 x i32> %vd, <2 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2334,7 +2334,7 @@ declare <2 x i32> @llvm.riscv.sf.vc.v.ivv.nxv2i32.iXLen.iXLen.iXLen(iXLen, <2 x
 define <4 x i32> @test_sf_vc_v_ivv_e32m2(<4 x i32> %vd, <4 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2347,7 +2347,7 @@ declare <4 x i32> @llvm.riscv.sf.vc.v.ivv.nxv4i32.iXLen.iXLen.iXLen(iXLen, <4 x
 define <8 x i32> @test_sf_vc_v_ivv_e32m4(<8 x i32> %vd, <8 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2360,7 +2360,7 @@ declare <8 x i32> @llvm.riscv.sf.vc.v.ivv.nxv8i32.iXLen.iXLen.iXLen(iXLen, <8 x
 define <16 x i32> @test_sf_vc_v_ivv_e32m8(<16 x i32> %vd, <16 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2373,7 +2373,7 @@ declare <16 x i32> @llvm.riscv.sf.vc.v.ivv.nxv16i32.iXLen.iXLen.iXLen(iXLen, <16
 define <1 x i64> @test_sf_vc_v_ivv_e64m1(<1 x i64> %vd, <1 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2386,7 +2386,7 @@ declare <1 x i64> @llvm.riscv.sf.vc.v.ivv.nxv1i64.iXLen.iXLen.iXLen(iXLen, <1 x
 define <2 x i64> @test_sf_vc_v_ivv_e64m2(<2 x i64> %vd, <2 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2399,7 +2399,7 @@ declare <2 x i64> @llvm.riscv.sf.vc.v.ivv.nxv2i64.iXLen.iXLen.iXLen(iXLen, <2 x
 define <4 x i64> @test_sf_vc_v_ivv_e64m4(<4 x i64> %vd, <4 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2412,7 +2412,7 @@ declare <4 x i64> @llvm.riscv.sf.vc.v.ivv.nxv4i64.iXLen.iXLen.iXLen(iXLen, <4 x
 define <8 x i64> @test_sf_vc_v_ivv_e64m8(<8 x i64> %vd, <8 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2438,7 +2438,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv1f16.nxv1i16.nxv1i16.iXLen(iXLen,
 define <1 x half> @test_sf_vc_fv_fvv_se_e16mf4(<1 x half> %vd, <1 x i16> %vs2, <1 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2464,7 +2464,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv2f16.nxv2i16.nxv2i16.iXLen(iXLen,
 define <2 x half> @test_sf_vc_fv_fvv_se_e16mf2(<2 x half> %vd, <2 x i16> %vs2, <2 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2490,7 +2490,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv4f16.nxv4i16.nxv4i16.iXLen(iXLen,
 define <4 x half> @test_sf_vc_fv_fvv_se_e16m1(<4 x half> %vd, <4 x i16> %vs2, <4 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2516,7 +2516,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv8f16.nxv8i16.nxv8i16.iXLen(iXLen,
 define <8 x half> @test_sf_vc_fv_fvv_se_e16m2(<8 x half> %vd, <8 x i16> %vs2, <8 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2542,7 +2542,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv16f16.nxv16i16.nxv16i16.iXLen(iXL
 define <16 x half> @test_sf_vc_fv_fvv_se_e16m4(<16 x half> %vd, <16 x i16> %vs2, <16 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -2568,7 +2568,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv32f16.nxv32i16.nxv32i16.iXLen(iXL
 define <32 x half> @test_sf_vc_fv_fvv_se_e16m8(<32 x half> %vd, <32 x i16> %vs2, <32 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -2594,7 +2594,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv1f32.nxv1i32.nxv1i32.iXLen(iXLen,
 define <1 x float> @test_sf_vc_fv_fvv_se_e32mf2(<1 x float> %vd, <1 x i32> %vs2, <1 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2620,7 +2620,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv2f32.nxv2i32.nxv2i32.iXLen(iXLen,
 define <2 x float> @test_sf_vc_fv_fvv_se_e32m1(<2 x float> %vd, <2 x i32> %vs2, <2 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2646,7 +2646,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv4f32.nxv4i32.nxv4i32.iXLen(iXLen,
 define <4 x float> @test_sf_vc_fv_fvv_se_e32m2(<4 x float> %vd, <4 x i32> %vs2, <4 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2672,7 +2672,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv8f32.nxv8i32.nxv8i32.iXLen(iXLen,
 define <8 x float> @test_sf_vc_fv_fvv_se_e32m4(<8 x float> %vd, <8 x i32> %vs2, <8 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -2698,7 +2698,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv16f32.nxv16i32.nxv16i32.iXLen(iXL
 define <16 x float> @test_sf_vc_fv_fvv_se_e32m8(<16 x float> %vd, <16 x i32> %vs2, <16 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -2724,7 +2724,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv1f64.nxv1i64.nxv1i64.iXLen(iXLen,
 define <1 x double> @test_sf_vc_fv_fvv_se_e64m1(<1 x double> %vd, <1 x i64> %vs2, <1 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2750,7 +2750,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv2f64.nxv2i64.nxv2i64.iXLen(iXLen,
 define <2 x double> @test_sf_vc_fv_fvv_se_e64m2(<2 x double> %vd, <2 x i64> %vs2, <2 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2776,7 +2776,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv4f64.nxv4i64.nxv4i64.iXLen(iXLen,
 define <4 x double> @test_sf_vc_fv_fvv_se_e64m4(<4 x double> %vd, <4 x i64> %vs2, <4 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -2802,7 +2802,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv8f64.nxv8i64.nxv8i64.iXLen(iXLen,
 define <8 x double> @test_sf_vc_fv_fvv_se_e64m8(<8 x double> %vd, <8 x i64> %vs2, <8 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -2828,7 +2828,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv1f16.nxv1i16.i16.iXLen(iXLen, <1
 define <1 x half> @test_sf_vc_v_fvvx_se_e16mf4(<1 x half> %vd, <1 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2854,7 +2854,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv2f16.nxv2i16.i16.iXLen(iXLen, <2
 define <2 x half> @test_sf_vc_v_fvvx_se_e16mf2(<2 x half> %vd, <2 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2880,7 +2880,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv4f16.nxv4i16.i16.iXLen(iXLen, <4
 define <4 x half> @test_sf_vc_v_fvvx_se_e16m1(<4 x half> %vd, <4 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2906,7 +2906,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv8f16.nxv8i16.i16.iXLen(iXLen, <8
 define <8 x half> @test_sf_vc_v_fvvx_se_e16m2(<8 x half> %vd, <8 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2932,7 +2932,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv16f16.nxv16i16.i16.iXLen(iXLen, <
 define <16 x half> @test_sf_vc_v_fvvx_se_e16m4(<16 x half> %vd, <16 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2958,7 +2958,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv32f16.nxv32i16.i16.iXLen(iXLen, <
 define <32 x half> @test_sf_vc_v_fvvx_se_e16m8(<32 x half> %vd, <32 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2984,7 +2984,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv1f32.nxv1i32.i32.iXLen(iXLen, <1
 define <1 x float> @test_sf_vc_v_fvvx_se_e32mf2(<1 x float> %vd, <1 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3010,7 +3010,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv2f32.nxv2i32.i32.iXLen(iXLen, <2
 define <2 x float> @test_sf_vc_v_fvvx_se_e32m1(<2 x float> %vd, <2 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3036,7 +3036,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv4f32.nxv4i32.i32.iXLen(iXLen, <4
 define <4 x float> @test_sf_vc_v_fvvx_se_e32m2(<4 x float> %vd, <4 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3062,7 +3062,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv8f32.nxv8i32.i32.iXLen(iXLen, <8
 define <8 x float> @test_sf_vc_v_fvvx_se_e32m4(<8 x float> %vd, <8 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3088,7 +3088,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv16f32.nxv16i32.i32.iXLen(iXLen, <
 define <16 x float> @test_sf_vc_v_fvvx_se_e32m8(<16 x float> %vd, <16 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3114,7 +3114,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv1f16.nxv1i16.iXLen.iXLen(iXLen, <
 define <1 x half> @test_sf_vc_fv_fvvi_se_e16mf4(<1 x half> %vd, <1 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3140,7 +3140,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv2f16.nxv2i16.iXLen.iXLen(iXLen, <
 define <2 x half> @test_sf_vc_fv_fvvi_se_e16mf2(<2 x half> %vd, <2 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3166,7 +3166,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv4f16.nxv4i16.iXLen.iXLen(iXLen, <
 define <4 x half> @test_sf_vc_fv_fvvi_se_e16m1(<4 x half> %vd, <4 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3192,7 +3192,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv8f16.nxv8i16.iXLen.iXLen(iXLen, <
 define <8 x half> @test_sf_vc_fv_fvvi_se_e16m2(<8 x half> %vd, <8 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3218,7 +3218,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv16f16.nxv16i16.iXLen.iXLen(iXLen,
 define <16 x half> @test_sf_vc_fv_fvvi_se_e16m4(<16 x half> %vd, <16 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3244,7 +3244,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv32f16.nxv32i16.iXLen.iXLen(iXLen,
 define <32 x half> @test_sf_vc_fv_fvvi_se_e16m8(<32 x half> %vd, <32 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3270,7 +3270,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv1f32.nxv1i32.iXLen.iXLen(iXLen, <
 define <1 x float> @test_sf_vc_fv_fvvi_se_e32mf2(<1 x float> %vd, <1 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3296,7 +3296,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv2f32.nxv2i32.iXLen.iXLen(iXLen, <
 define <2 x float> @test_sf_vc_fv_fvvi_se_e32m1(<2 x float> %vd, <2 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3322,7 +3322,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv4f32.nxv4i32.iXLen.iXLen(iXLen, <
 define <4 x float> @test_sf_vc_fv_fvvi_se_e32m2(<4 x float> %vd, <4 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3348,7 +3348,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv8f32.nxv8i32.iXLen.iXLen(iXLen, <
 define <8 x float> @test_sf_vc_fv_fvvi_se_e32m4(<8 x float> %vd, <8 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3374,7 +3374,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv16f32.nxv16i32.iXLen.iXLen(iXLen,
 define <16 x float> @test_sf_vc_fv_fvvi_se_e32m8(<16 x float> %vd, <16 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3400,7 +3400,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv1f16.nxv1i16.f16.iXLen(iXLen, <1
 define <1 x half> @test_sf_vc_fv_fvvf_se_e16mf4(<1 x half> %vd, <1 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3426,7 +3426,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv2f16.nxv2i16.f16.iXLen(iXLen, <2
 define <2 x half> @test_sf_vc_fv_fvvf_se_e16mf2(<2 x half> %vd, <2 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3452,7 +3452,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv4f16.nxv4i16.f16.iXLen(iXLen, <4
 define <4 x half> @test_sf_vc_fv_fvvf_se_e16m1(<4 x half> %vd, <4 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3478,7 +3478,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv8f16.nxv8i16.f16.iXLen(iXLen, <8
 define <8 x half> @test_sf_vc_fv_fvvf_se_e16m2(<8 x half> %vd, <8 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3504,7 +3504,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv16f16.nxv16i16.f16.iXLen(iXLen, <
 define <16 x half> @test_sf_vc_fv_fvvf_se_e16m4(<16 x half> %vd, <16 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v10, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3530,7 +3530,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv32f16.nxv32i16.f16.iXLen(iXLen, <
 define <32 x half> @test_sf_vc_fv_fvvf_se_e16m8(<32 x half> %vd, <32 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v12, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3556,7 +3556,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv1f32.nxv1i32.f32.iXLen(iXLen, <1
 define <1 x float> @test_sf_vc_fv_fvvf_se_e32mf2(<1 x float> %vd, <1 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3582,7 +3582,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv2f32.nxv2i32.f32.iXLen(iXLen, <2
 define <2 x float> @test_sf_vc_fv_fvvf_se_e32m1(<2 x float> %vd, <2 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3608,7 +3608,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv4f32.nxv4i32.f32.iXLen(iXLen, <4
 define <4 x float> @test_sf_vc_fv_fvvf_se_e32m2(<4 x float> %vd, <4 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3634,7 +3634,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv8f32.nxv8i32.f32.iXLen(iXLen, <8
 define <8 x float> @test_sf_vc_fv_fvvf_se_e32m4(<8 x float> %vd, <8 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v10, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3660,7 +3660,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv16f32.nxv16i32.f32.iXLen(iXLen, <
 define <16 x float> @test_sf_vc_fv_fvvf_se_e32m8(<16 x float> %vd, <16 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v12, fa0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 5a880105f683795d2049275e87c8c28ab13cd7cb..5a1f7f54305846c0203de0f43534ab85210469cd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -154,10 +154,10 @@ define <4 x i32> @stest_f32i32(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB3_8: # %entry
 ; CHECK-NOV-NEXT:    lui a1, 524288
 ; CHECK-NOV-NEXT:  .LBB3_9: # %entry
-; CHECK-NOV-NEXT:    sw a1, 12(a0)
-; CHECK-NOV-NEXT:    sw a2, 8(a0)
-; CHECK-NOV-NEXT:    sw a4, 4(a0)
 ; CHECK-NOV-NEXT:    sw a5, 0(a0)
+; CHECK-NOV-NEXT:    sw a4, 4(a0)
+; CHECK-NOV-NEXT:    sw a2, 8(a0)
+; CHECK-NOV-NEXT:    sw a1, 12(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB3_10: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a6
@@ -219,10 +219,10 @@ define <4 x i32> @utest_f32i32(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB4_4: # %entry
 ; CHECK-NOV-NEXT:    mv a5, a3
 ; CHECK-NOV-NEXT:  .LBB4_5: # %entry
-; CHECK-NOV-NEXT:    sw a5, 12(a0)
-; CHECK-NOV-NEXT:    sw a4, 8(a0)
-; CHECK-NOV-NEXT:    sw a2, 4(a0)
 ; CHECK-NOV-NEXT:    sw a1, 0(a0)
+; CHECK-NOV-NEXT:    sw a2, 4(a0)
+; CHECK-NOV-NEXT:    sw a4, 8(a0)
+; CHECK-NOV-NEXT:    sw a5, 12(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB4_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a3
@@ -282,10 +282,10 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) {
 ; CHECK-NOV-NEXT:    and a2, a6, a2
 ; CHECK-NOV-NEXT:    negw a4, a4
 ; CHECK-NOV-NEXT:    and a1, a4, a1
-; CHECK-NOV-NEXT:    sw a1, 12(a0)
-; CHECK-NOV-NEXT:    sw a2, 8(a0)
-; CHECK-NOV-NEXT:    sw a3, 4(a0)
 ; CHECK-NOV-NEXT:    sw a5, 0(a0)
+; CHECK-NOV-NEXT:    sw a3, 4(a0)
+; CHECK-NOV-NEXT:    sw a2, 8(a0)
+; CHECK-NOV-NEXT:    sw a1, 12(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB5_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a4
@@ -380,10 +380,10 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB6_8: # %entry
 ; CHECK-NOV-NEXT:    lui a0, 524288
 ; CHECK-NOV-NEXT:  .LBB6_9: # %entry
-; CHECK-NOV-NEXT:    sw a0, 12(s0)
-; CHECK-NOV-NEXT:    sw s1, 8(s0)
-; CHECK-NOV-NEXT:    sw a2, 4(s0)
 ; CHECK-NOV-NEXT:    sw a3, 0(s0)
+; CHECK-NOV-NEXT:    sw a2, 4(s0)
+; CHECK-NOV-NEXT:    sw s1, 8(s0)
+; CHECK-NOV-NEXT:    sw a0, 12(s0)
 ; CHECK-NOV-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -555,10 +555,10 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB7_4: # %entry
 ; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:  .LBB7_5: # %entry
-; CHECK-NOV-NEXT:    sw a3, 12(s0)
-; CHECK-NOV-NEXT:    sw a2, 8(s0)
-; CHECK-NOV-NEXT:    sw s1, 4(s0)
 ; CHECK-NOV-NEXT:    sw a0, 0(s0)
+; CHECK-NOV-NEXT:    sw s1, 4(s0)
+; CHECK-NOV-NEXT:    sw a2, 8(s0)
+; CHECK-NOV-NEXT:    sw a3, 12(s0)
 ; CHECK-NOV-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -728,10 +728,10 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-NOV-NEXT:    and a4, a4, s1
 ; CHECK-NOV-NEXT:    negw a2, a2
 ; CHECK-NOV-NEXT:    and a0, a2, a0
-; CHECK-NOV-NEXT:    sw a0, 12(s0)
-; CHECK-NOV-NEXT:    sw a4, 8(s0)
-; CHECK-NOV-NEXT:    sw a1, 4(s0)
 ; CHECK-NOV-NEXT:    sw a3, 0(s0)
+; CHECK-NOV-NEXT:    sw a1, 4(s0)
+; CHECK-NOV-NEXT:    sw a4, 8(s0)
+; CHECK-NOV-NEXT:    sw a0, 12(s0)
 ; CHECK-NOV-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -992,10 +992,10 @@ define <4 x i16> @stest_f32i16(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB12_8: # %entry
 ; CHECK-NOV-NEXT:    lui a1, 1048568
 ; CHECK-NOV-NEXT:  .LBB12_9: # %entry
-; CHECK-NOV-NEXT:    sh a1, 6(a0)
-; CHECK-NOV-NEXT:    sh a2, 4(a0)
-; CHECK-NOV-NEXT:    sh a3, 2(a0)
 ; CHECK-NOV-NEXT:    sh a4, 0(a0)
+; CHECK-NOV-NEXT:    sh a3, 2(a0)
+; CHECK-NOV-NEXT:    sh a2, 4(a0)
+; CHECK-NOV-NEXT:    sh a1, 6(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB12_10: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a5
@@ -1059,10 +1059,10 @@ define <4 x i16> @utest_f32i16(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB13_4: # %entry
 ; CHECK-NOV-NEXT:    mv a5, a3
 ; CHECK-NOV-NEXT:  .LBB13_5: # %entry
-; CHECK-NOV-NEXT:    sh a5, 6(a0)
-; CHECK-NOV-NEXT:    sh a4, 4(a0)
-; CHECK-NOV-NEXT:    sh a2, 2(a0)
 ; CHECK-NOV-NEXT:    sh a1, 0(a0)
+; CHECK-NOV-NEXT:    sh a2, 2(a0)
+; CHECK-NOV-NEXT:    sh a4, 4(a0)
+; CHECK-NOV-NEXT:    sh a5, 6(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB13_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a3
@@ -1123,10 +1123,10 @@ define <4 x i16> @ustest_f32i16(<4 x float> %x) {
 ; CHECK-NOV-NEXT:    and a2, a6, a2
 ; CHECK-NOV-NEXT:    negw a4, a4
 ; CHECK-NOV-NEXT:    and a1, a4, a1
-; CHECK-NOV-NEXT:    sh a1, 6(a0)
-; CHECK-NOV-NEXT:    sh a2, 4(a0)
-; CHECK-NOV-NEXT:    sh a3, 2(a0)
 ; CHECK-NOV-NEXT:    sh a5, 0(a0)
+; CHECK-NOV-NEXT:    sh a3, 2(a0)
+; CHECK-NOV-NEXT:    sh a2, 4(a0)
+; CHECK-NOV-NEXT:    sh a1, 6(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB14_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a4
@@ -1237,14 +1237,14 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    fcvt.l.s a1, fs5, rtz
 ; CHECK-NOV-NEXT:    bge s1, a7, .LBB15_19
 ; CHECK-NOV-NEXT:  .LBB15_2: # %entry
-; CHECK-NOV-NEXT:    fcvt.l.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    fcvt.l.s a3, fs4, rtz
 ; CHECK-NOV-NEXT:    bge a1, a7, .LBB15_20
 ; CHECK-NOV-NEXT:  .LBB15_3: # %entry
-; CHECK-NOV-NEXT:    fcvt.l.s a3, fs3, rtz
-; CHECK-NOV-NEXT:    bge a2, a7, .LBB15_21
+; CHECK-NOV-NEXT:    fcvt.l.s a2, fs3, rtz
+; CHECK-NOV-NEXT:    bge a3, a7, .LBB15_21
 ; CHECK-NOV-NEXT:  .LBB15_4: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a4, fs2, rtz
-; CHECK-NOV-NEXT:    bge a3, a7, .LBB15_22
+; CHECK-NOV-NEXT:    bge a2, a7, .LBB15_22
 ; CHECK-NOV-NEXT:  .LBB15_5: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a5, fs1, rtz
 ; CHECK-NOV-NEXT:    bge a4, a7, .LBB15_23
@@ -1261,9 +1261,9 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB15_10: # %entry
 ; CHECK-NOV-NEXT:    bge a7, a4, .LBB15_28
 ; CHECK-NOV-NEXT:  .LBB15_11: # %entry
-; CHECK-NOV-NEXT:    bge a7, a3, .LBB15_29
+; CHECK-NOV-NEXT:    bge a7, a2, .LBB15_29
 ; CHECK-NOV-NEXT:  .LBB15_12: # %entry
-; CHECK-NOV-NEXT:    bge a7, a2, .LBB15_30
+; CHECK-NOV-NEXT:    bge a7, a3, .LBB15_30
 ; CHECK-NOV-NEXT:  .LBB15_13: # %entry
 ; CHECK-NOV-NEXT:    bge a7, a1, .LBB15_31
 ; CHECK-NOV-NEXT:  .LBB15_14: # %entry
@@ -1273,14 +1273,14 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB15_16: # %entry
 ; CHECK-NOV-NEXT:    lui a0, 1048568
 ; CHECK-NOV-NEXT:  .LBB15_17: # %entry
-; CHECK-NOV-NEXT:    sh a0, 14(s0)
-; CHECK-NOV-NEXT:    sh s1, 12(s0)
+; CHECK-NOV-NEXT:    sh a3, 8(s0)
 ; CHECK-NOV-NEXT:    sh a1, 10(s0)
-; CHECK-NOV-NEXT:    sh a2, 8(s0)
-; CHECK-NOV-NEXT:    sh a3, 6(s0)
-; CHECK-NOV-NEXT:    sh a4, 4(s0)
-; CHECK-NOV-NEXT:    sh a5, 2(s0)
+; CHECK-NOV-NEXT:    sh s1, 12(s0)
+; CHECK-NOV-NEXT:    sh a0, 14(s0)
 ; CHECK-NOV-NEXT:    sh a6, 0(s0)
+; CHECK-NOV-NEXT:    sh a5, 2(s0)
+; CHECK-NOV-NEXT:    sh a4, 4(s0)
+; CHECK-NOV-NEXT:    sh a2, 6(s0)
 ; CHECK-NOV-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
@@ -1305,18 +1305,18 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    blt s1, a7, .LBB15_2
 ; CHECK-NOV-NEXT:  .LBB15_19: # %entry
 ; CHECK-NOV-NEXT:    mv s1, a7
-; CHECK-NOV-NEXT:    fcvt.l.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    fcvt.l.s a3, fs4, rtz
 ; CHECK-NOV-NEXT:    blt a1, a7, .LBB15_3
 ; CHECK-NOV-NEXT:  .LBB15_20: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a7
-; CHECK-NOV-NEXT:    fcvt.l.s a3, fs3, rtz
-; CHECK-NOV-NEXT:    blt a2, a7, .LBB15_4
+; CHECK-NOV-NEXT:    fcvt.l.s a2, fs3, rtz
+; CHECK-NOV-NEXT:    blt a3, a7, .LBB15_4
 ; CHECK-NOV-NEXT:  .LBB15_21: # %entry
-; CHECK-NOV-NEXT:    mv a2, a7
+; CHECK-NOV-NEXT:    mv a3, a7
 ; CHECK-NOV-NEXT:    fcvt.l.s a4, fs2, rtz
-; CHECK-NOV-NEXT:    blt a3, a7, .LBB15_5
+; CHECK-NOV-NEXT:    blt a2, a7, .LBB15_5
 ; CHECK-NOV-NEXT:  .LBB15_22: # %entry
-; CHECK-NOV-NEXT:    mv a3, a7
+; CHECK-NOV-NEXT:    mv a2, a7
 ; CHECK-NOV-NEXT:    fcvt.l.s a5, fs1, rtz
 ; CHECK-NOV-NEXT:    blt a4, a7, .LBB15_6
 ; CHECK-NOV-NEXT:  .LBB15_23: # %entry
@@ -1338,12 +1338,12 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    blt a7, a4, .LBB15_11
 ; CHECK-NOV-NEXT:  .LBB15_28: # %entry
 ; CHECK-NOV-NEXT:    lui a4, 1048568
-; CHECK-NOV-NEXT:    blt a7, a3, .LBB15_12
+; CHECK-NOV-NEXT:    blt a7, a2, .LBB15_12
 ; CHECK-NOV-NEXT:  .LBB15_29: # %entry
-; CHECK-NOV-NEXT:    lui a3, 1048568
-; CHECK-NOV-NEXT:    blt a7, a2, .LBB15_13
-; CHECK-NOV-NEXT:  .LBB15_30: # %entry
 ; CHECK-NOV-NEXT:    lui a2, 1048568
+; CHECK-NOV-NEXT:    blt a7, a3, .LBB15_13
+; CHECK-NOV-NEXT:  .LBB15_30: # %entry
+; CHECK-NOV-NEXT:    lui a3, 1048568
 ; CHECK-NOV-NEXT:    blt a7, a1, .LBB15_14
 ; CHECK-NOV-NEXT:  .LBB15_31: # %entry
 ; CHECK-NOV-NEXT:    lui a1, 1048568
@@ -1585,40 +1585,40 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    fcvt.lu.s s1, fs6, rtz
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.lu.s a0, fa0, rtz
-; CHECK-NOV-NEXT:    lui a1, 16
-; CHECK-NOV-NEXT:    addiw a1, a1, -1
-; CHECK-NOV-NEXT:    bgeu a0, a1, .LBB16_10
+; CHECK-NOV-NEXT:    lui a3, 16
+; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    bgeu a0, a3, .LBB16_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs5, rtz
-; CHECK-NOV-NEXT:    bgeu s1, a1, .LBB16_11
+; CHECK-NOV-NEXT:    fcvt.lu.s a1, fs5, rtz
+; CHECK-NOV-NEXT:    bgeu s1, a3, .LBB16_11
 ; CHECK-NOV-NEXT:  .LBB16_2: # %entry
-; CHECK-NOV-NEXT:    fcvt.lu.s a3, fs4, rtz
-; CHECK-NOV-NEXT:    bgeu a2, a1, .LBB16_12
+; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    bgeu a1, a3, .LBB16_12
 ; CHECK-NOV-NEXT:  .LBB16_3: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a4, fs3, rtz
-; CHECK-NOV-NEXT:    bgeu a3, a1, .LBB16_13
+; CHECK-NOV-NEXT:    bgeu a2, a3, .LBB16_13
 ; CHECK-NOV-NEXT:  .LBB16_4: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a5, fs2, rtz
-; CHECK-NOV-NEXT:    bgeu a4, a1, .LBB16_14
+; CHECK-NOV-NEXT:    bgeu a4, a3, .LBB16_14
 ; CHECK-NOV-NEXT:  .LBB16_5: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a6, fs1, rtz
-; CHECK-NOV-NEXT:    bgeu a5, a1, .LBB16_15
+; CHECK-NOV-NEXT:    bgeu a5, a3, .LBB16_15
 ; CHECK-NOV-NEXT:  .LBB16_6: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a7, fs0, rtz
-; CHECK-NOV-NEXT:    bgeu a6, a1, .LBB16_16
+; CHECK-NOV-NEXT:    bgeu a6, a3, .LBB16_16
 ; CHECK-NOV-NEXT:  .LBB16_7: # %entry
-; CHECK-NOV-NEXT:    bltu a7, a1, .LBB16_9
+; CHECK-NOV-NEXT:    bltu a7, a3, .LBB16_9
 ; CHECK-NOV-NEXT:  .LBB16_8: # %entry
-; CHECK-NOV-NEXT:    mv a7, a1
+; CHECK-NOV-NEXT:    mv a7, a3
 ; CHECK-NOV-NEXT:  .LBB16_9: # %entry
-; CHECK-NOV-NEXT:    sh a7, 14(s0)
-; CHECK-NOV-NEXT:    sh a6, 12(s0)
-; CHECK-NOV-NEXT:    sh a5, 10(s0)
 ; CHECK-NOV-NEXT:    sh a4, 8(s0)
-; CHECK-NOV-NEXT:    sh a3, 6(s0)
-; CHECK-NOV-NEXT:    sh a2, 4(s0)
-; CHECK-NOV-NEXT:    sh s1, 2(s0)
+; CHECK-NOV-NEXT:    sh a5, 10(s0)
+; CHECK-NOV-NEXT:    sh a6, 12(s0)
+; CHECK-NOV-NEXT:    sh a7, 14(s0)
 ; CHECK-NOV-NEXT:    sh a0, 0(s0)
+; CHECK-NOV-NEXT:    sh s1, 2(s0)
+; CHECK-NOV-NEXT:    sh a1, 4(s0)
+; CHECK-NOV-NEXT:    sh a2, 6(s0)
 ; CHECK-NOV-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
@@ -1638,32 +1638,32 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    addi sp, sp, 128
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB16_10: # %entry
-; CHECK-NOV-NEXT:    mv a0, a1
-; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs5, rtz
-; CHECK-NOV-NEXT:    bltu s1, a1, .LBB16_2
+; CHECK-NOV-NEXT:    mv a0, a3
+; CHECK-NOV-NEXT:    fcvt.lu.s a1, fs5, rtz
+; CHECK-NOV-NEXT:    bltu s1, a3, .LBB16_2
 ; CHECK-NOV-NEXT:  .LBB16_11: # %entry
-; CHECK-NOV-NEXT:    mv s1, a1
-; CHECK-NOV-NEXT:    fcvt.lu.s a3, fs4, rtz
-; CHECK-NOV-NEXT:    bltu a2, a1, .LBB16_3
+; CHECK-NOV-NEXT:    mv s1, a3
+; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    bltu a1, a3, .LBB16_3
 ; CHECK-NOV-NEXT:  .LBB16_12: # %entry
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a1, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a4, fs3, rtz
-; CHECK-NOV-NEXT:    bltu a3, a1, .LBB16_4
+; CHECK-NOV-NEXT:    bltu a2, a3, .LBB16_4
 ; CHECK-NOV-NEXT:  .LBB16_13: # %entry
-; CHECK-NOV-NEXT:    mv a3, a1
+; CHECK-NOV-NEXT:    mv a2, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a5, fs2, rtz
-; CHECK-NOV-NEXT:    bltu a4, a1, .LBB16_5
+; CHECK-NOV-NEXT:    bltu a4, a3, .LBB16_5
 ; CHECK-NOV-NEXT:  .LBB16_14: # %entry
-; CHECK-NOV-NEXT:    mv a4, a1
+; CHECK-NOV-NEXT:    mv a4, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a6, fs1, rtz
-; CHECK-NOV-NEXT:    bltu a5, a1, .LBB16_6
+; CHECK-NOV-NEXT:    bltu a5, a3, .LBB16_6
 ; CHECK-NOV-NEXT:  .LBB16_15: # %entry
-; CHECK-NOV-NEXT:    mv a5, a1
+; CHECK-NOV-NEXT:    mv a5, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a7, fs0, rtz
-; CHECK-NOV-NEXT:    bltu a6, a1, .LBB16_7
+; CHECK-NOV-NEXT:    bltu a6, a3, .LBB16_7
 ; CHECK-NOV-NEXT:  .LBB16_16: # %entry
-; CHECK-NOV-NEXT:    mv a6, a1
-; CHECK-NOV-NEXT:    bgeu a7, a1, .LBB16_8
+; CHECK-NOV-NEXT:    mv a6, a3
+; CHECK-NOV-NEXT:    bgeu a7, a3, .LBB16_8
 ; CHECK-NOV-NEXT:    j .LBB16_9
 ;
 ; CHECK-V-LABEL: utesth_f16i16:
@@ -1946,14 +1946,14 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    and t0, t0, s1
 ; CHECK-NOV-NEXT:    negw a3, a3
 ; CHECK-NOV-NEXT:    and a0, a3, a0
-; CHECK-NOV-NEXT:    sh a0, 14(s0)
-; CHECK-NOV-NEXT:    sh t0, 12(s0)
-; CHECK-NOV-NEXT:    sh a1, 10(s0)
 ; CHECK-NOV-NEXT:    sh a2, 8(s0)
-; CHECK-NOV-NEXT:    sh a4, 6(s0)
-; CHECK-NOV-NEXT:    sh a5, 4(s0)
-; CHECK-NOV-NEXT:    sh a6, 2(s0)
+; CHECK-NOV-NEXT:    sh a1, 10(s0)
+; CHECK-NOV-NEXT:    sh t0, 12(s0)
+; CHECK-NOV-NEXT:    sh a0, 14(s0)
 ; CHECK-NOV-NEXT:    sh a7, 0(s0)
+; CHECK-NOV-NEXT:    sh a6, 2(s0)
+; CHECK-NOV-NEXT:    sh a5, 4(s0)
+; CHECK-NOV-NEXT:    sh a4, 6(s0)
 ; CHECK-NOV-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
@@ -3488,10 +3488,10 @@ define <4 x i32> @stest_f32i32_mm(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB30_8: # %entry
 ; CHECK-NOV-NEXT:    lui a1, 524288
 ; CHECK-NOV-NEXT:  .LBB30_9: # %entry
-; CHECK-NOV-NEXT:    sw a1, 12(a0)
-; CHECK-NOV-NEXT:    sw a2, 8(a0)
-; CHECK-NOV-NEXT:    sw a4, 4(a0)
 ; CHECK-NOV-NEXT:    sw a5, 0(a0)
+; CHECK-NOV-NEXT:    sw a4, 4(a0)
+; CHECK-NOV-NEXT:    sw a2, 8(a0)
+; CHECK-NOV-NEXT:    sw a1, 12(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB30_10: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a6
@@ -3551,10 +3551,10 @@ define <4 x i32> @utest_f32i32_mm(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB31_4: # %entry
 ; CHECK-NOV-NEXT:    mv a5, a3
 ; CHECK-NOV-NEXT:  .LBB31_5: # %entry
-; CHECK-NOV-NEXT:    sw a5, 12(a0)
-; CHECK-NOV-NEXT:    sw a4, 8(a0)
-; CHECK-NOV-NEXT:    sw a2, 4(a0)
 ; CHECK-NOV-NEXT:    sw a1, 0(a0)
+; CHECK-NOV-NEXT:    sw a2, 4(a0)
+; CHECK-NOV-NEXT:    sw a4, 8(a0)
+; CHECK-NOV-NEXT:    sw a5, 12(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB31_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a3
@@ -3613,10 +3613,10 @@ define <4 x i32> @ustest_f32i32_mm(<4 x float> %x) {
 ; CHECK-NOV-NEXT:    sgtz a5, a1
 ; CHECK-NOV-NEXT:    negw a5, a5
 ; CHECK-NOV-NEXT:    and a1, a5, a1
-; CHECK-NOV-NEXT:    sw a1, 12(a0)
-; CHECK-NOV-NEXT:    sw a2, 8(a0)
-; CHECK-NOV-NEXT:    sw a4, 4(a0)
 ; CHECK-NOV-NEXT:    sw a3, 0(a0)
+; CHECK-NOV-NEXT:    sw a4, 4(a0)
+; CHECK-NOV-NEXT:    sw a2, 8(a0)
+; CHECK-NOV-NEXT:    sw a1, 12(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB32_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a3
@@ -3709,10 +3709,10 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB33_8: # %entry
 ; CHECK-NOV-NEXT:    lui a0, 524288
 ; CHECK-NOV-NEXT:  .LBB33_9: # %entry
-; CHECK-NOV-NEXT:    sw a0, 12(s0)
-; CHECK-NOV-NEXT:    sw s1, 8(s0)
-; CHECK-NOV-NEXT:    sw a2, 4(s0)
 ; CHECK-NOV-NEXT:    sw a3, 0(s0)
+; CHECK-NOV-NEXT:    sw a2, 4(s0)
+; CHECK-NOV-NEXT:    sw s1, 8(s0)
+; CHECK-NOV-NEXT:    sw a0, 12(s0)
 ; CHECK-NOV-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -3882,10 +3882,10 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB34_4: # %entry
 ; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:  .LBB34_5: # %entry
-; CHECK-NOV-NEXT:    sw a3, 12(s0)
-; CHECK-NOV-NEXT:    sw a2, 8(s0)
-; CHECK-NOV-NEXT:    sw s1, 4(s0)
 ; CHECK-NOV-NEXT:    sw a0, 0(s0)
+; CHECK-NOV-NEXT:    sw s1, 4(s0)
+; CHECK-NOV-NEXT:    sw a2, 8(s0)
+; CHECK-NOV-NEXT:    sw a3, 12(s0)
 ; CHECK-NOV-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -4054,10 +4054,10 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
 ; CHECK-NOV-NEXT:    sgtz a4, a0
 ; CHECK-NOV-NEXT:    negw a4, a4
 ; CHECK-NOV-NEXT:    and a0, a4, a0
-; CHECK-NOV-NEXT:    sw a0, 12(s0)
-; CHECK-NOV-NEXT:    sw a3, 8(s0)
-; CHECK-NOV-NEXT:    sw a1, 4(s0)
 ; CHECK-NOV-NEXT:    sw a2, 0(s0)
+; CHECK-NOV-NEXT:    sw a1, 4(s0)
+; CHECK-NOV-NEXT:    sw a3, 8(s0)
+; CHECK-NOV-NEXT:    sw a0, 12(s0)
 ; CHECK-NOV-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
@@ -4311,10 +4311,10 @@ define <4 x i16> @stest_f32i16_mm(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB39_8: # %entry
 ; CHECK-NOV-NEXT:    lui a1, 1048568
 ; CHECK-NOV-NEXT:  .LBB39_9: # %entry
-; CHECK-NOV-NEXT:    sh a1, 6(a0)
-; CHECK-NOV-NEXT:    sh a2, 4(a0)
-; CHECK-NOV-NEXT:    sh a3, 2(a0)
 ; CHECK-NOV-NEXT:    sh a4, 0(a0)
+; CHECK-NOV-NEXT:    sh a3, 2(a0)
+; CHECK-NOV-NEXT:    sh a2, 4(a0)
+; CHECK-NOV-NEXT:    sh a1, 6(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB39_10: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a5
@@ -4376,10 +4376,10 @@ define <4 x i16> @utest_f32i16_mm(<4 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB40_4: # %entry
 ; CHECK-NOV-NEXT:    mv a5, a3
 ; CHECK-NOV-NEXT:  .LBB40_5: # %entry
-; CHECK-NOV-NEXT:    sh a5, 6(a0)
-; CHECK-NOV-NEXT:    sh a4, 4(a0)
-; CHECK-NOV-NEXT:    sh a2, 2(a0)
 ; CHECK-NOV-NEXT:    sh a1, 0(a0)
+; CHECK-NOV-NEXT:    sh a2, 2(a0)
+; CHECK-NOV-NEXT:    sh a4, 4(a0)
+; CHECK-NOV-NEXT:    sh a5, 6(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB40_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a3
@@ -4439,10 +4439,10 @@ define <4 x i16> @ustest_f32i16_mm(<4 x float> %x) {
 ; CHECK-NOV-NEXT:    sgtz a5, a1
 ; CHECK-NOV-NEXT:    negw a5, a5
 ; CHECK-NOV-NEXT:    and a1, a5, a1
-; CHECK-NOV-NEXT:    sh a1, 6(a0)
-; CHECK-NOV-NEXT:    sh a2, 4(a0)
-; CHECK-NOV-NEXT:    sh a4, 2(a0)
 ; CHECK-NOV-NEXT:    sh a3, 0(a0)
+; CHECK-NOV-NEXT:    sh a4, 2(a0)
+; CHECK-NOV-NEXT:    sh a2, 4(a0)
+; CHECK-NOV-NEXT:    sh a1, 6(a0)
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB41_6: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a3
@@ -4551,14 +4551,14 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    fcvt.l.s a1, fs5, rtz
 ; CHECK-NOV-NEXT:    bge s1, a7, .LBB42_19
 ; CHECK-NOV-NEXT:  .LBB42_2: # %entry
-; CHECK-NOV-NEXT:    fcvt.l.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    fcvt.l.s a3, fs4, rtz
 ; CHECK-NOV-NEXT:    bge a1, a7, .LBB42_20
 ; CHECK-NOV-NEXT:  .LBB42_3: # %entry
-; CHECK-NOV-NEXT:    fcvt.l.s a3, fs3, rtz
-; CHECK-NOV-NEXT:    bge a2, a7, .LBB42_21
+; CHECK-NOV-NEXT:    fcvt.l.s a2, fs3, rtz
+; CHECK-NOV-NEXT:    bge a3, a7, .LBB42_21
 ; CHECK-NOV-NEXT:  .LBB42_4: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a4, fs2, rtz
-; CHECK-NOV-NEXT:    bge a3, a7, .LBB42_22
+; CHECK-NOV-NEXT:    bge a2, a7, .LBB42_22
 ; CHECK-NOV-NEXT:  .LBB42_5: # %entry
 ; CHECK-NOV-NEXT:    fcvt.l.s a5, fs1, rtz
 ; CHECK-NOV-NEXT:    bge a4, a7, .LBB42_23
@@ -4575,9 +4575,9 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB42_10: # %entry
 ; CHECK-NOV-NEXT:    bge a7, a4, .LBB42_28
 ; CHECK-NOV-NEXT:  .LBB42_11: # %entry
-; CHECK-NOV-NEXT:    bge a7, a3, .LBB42_29
+; CHECK-NOV-NEXT:    bge a7, a2, .LBB42_29
 ; CHECK-NOV-NEXT:  .LBB42_12: # %entry
-; CHECK-NOV-NEXT:    bge a7, a2, .LBB42_30
+; CHECK-NOV-NEXT:    bge a7, a3, .LBB42_30
 ; CHECK-NOV-NEXT:  .LBB42_13: # %entry
 ; CHECK-NOV-NEXT:    bge a7, a1, .LBB42_31
 ; CHECK-NOV-NEXT:  .LBB42_14: # %entry
@@ -4587,14 +4587,14 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB42_16: # %entry
 ; CHECK-NOV-NEXT:    lui a0, 1048568
 ; CHECK-NOV-NEXT:  .LBB42_17: # %entry
-; CHECK-NOV-NEXT:    sh a0, 14(s0)
-; CHECK-NOV-NEXT:    sh s1, 12(s0)
+; CHECK-NOV-NEXT:    sh a3, 8(s0)
 ; CHECK-NOV-NEXT:    sh a1, 10(s0)
-; CHECK-NOV-NEXT:    sh a2, 8(s0)
-; CHECK-NOV-NEXT:    sh a3, 6(s0)
-; CHECK-NOV-NEXT:    sh a4, 4(s0)
-; CHECK-NOV-NEXT:    sh a5, 2(s0)
+; CHECK-NOV-NEXT:    sh s1, 12(s0)
+; CHECK-NOV-NEXT:    sh a0, 14(s0)
 ; CHECK-NOV-NEXT:    sh a6, 0(s0)
+; CHECK-NOV-NEXT:    sh a5, 2(s0)
+; CHECK-NOV-NEXT:    sh a4, 4(s0)
+; CHECK-NOV-NEXT:    sh a2, 6(s0)
 ; CHECK-NOV-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
@@ -4619,18 +4619,18 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    blt s1, a7, .LBB42_2
 ; CHECK-NOV-NEXT:  .LBB42_19: # %entry
 ; CHECK-NOV-NEXT:    mv s1, a7
-; CHECK-NOV-NEXT:    fcvt.l.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    fcvt.l.s a3, fs4, rtz
 ; CHECK-NOV-NEXT:    blt a1, a7, .LBB42_3
 ; CHECK-NOV-NEXT:  .LBB42_20: # %entry
 ; CHECK-NOV-NEXT:    mv a1, a7
-; CHECK-NOV-NEXT:    fcvt.l.s a3, fs3, rtz
-; CHECK-NOV-NEXT:    blt a2, a7, .LBB42_4
+; CHECK-NOV-NEXT:    fcvt.l.s a2, fs3, rtz
+; CHECK-NOV-NEXT:    blt a3, a7, .LBB42_4
 ; CHECK-NOV-NEXT:  .LBB42_21: # %entry
-; CHECK-NOV-NEXT:    mv a2, a7
+; CHECK-NOV-NEXT:    mv a3, a7
 ; CHECK-NOV-NEXT:    fcvt.l.s a4, fs2, rtz
-; CHECK-NOV-NEXT:    blt a3, a7, .LBB42_5
+; CHECK-NOV-NEXT:    blt a2, a7, .LBB42_5
 ; CHECK-NOV-NEXT:  .LBB42_22: # %entry
-; CHECK-NOV-NEXT:    mv a3, a7
+; CHECK-NOV-NEXT:    mv a2, a7
 ; CHECK-NOV-NEXT:    fcvt.l.s a5, fs1, rtz
 ; CHECK-NOV-NEXT:    blt a4, a7, .LBB42_6
 ; CHECK-NOV-NEXT:  .LBB42_23: # %entry
@@ -4652,12 +4652,12 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    blt a7, a4, .LBB42_11
 ; CHECK-NOV-NEXT:  .LBB42_28: # %entry
 ; CHECK-NOV-NEXT:    lui a4, 1048568
-; CHECK-NOV-NEXT:    blt a7, a3, .LBB42_12
+; CHECK-NOV-NEXT:    blt a7, a2, .LBB42_12
 ; CHECK-NOV-NEXT:  .LBB42_29: # %entry
-; CHECK-NOV-NEXT:    lui a3, 1048568
-; CHECK-NOV-NEXT:    blt a7, a2, .LBB42_13
-; CHECK-NOV-NEXT:  .LBB42_30: # %entry
 ; CHECK-NOV-NEXT:    lui a2, 1048568
+; CHECK-NOV-NEXT:    blt a7, a3, .LBB42_13
+; CHECK-NOV-NEXT:  .LBB42_30: # %entry
+; CHECK-NOV-NEXT:    lui a3, 1048568
 ; CHECK-NOV-NEXT:    blt a7, a1, .LBB42_14
 ; CHECK-NOV-NEXT:  .LBB42_31: # %entry
 ; CHECK-NOV-NEXT:    lui a1, 1048568
@@ -4897,40 +4897,40 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    fcvt.lu.s s1, fs6, rtz
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    fcvt.lu.s a0, fa0, rtz
-; CHECK-NOV-NEXT:    lui a1, 16
-; CHECK-NOV-NEXT:    addiw a1, a1, -1
-; CHECK-NOV-NEXT:    bgeu a0, a1, .LBB43_10
+; CHECK-NOV-NEXT:    lui a3, 16
+; CHECK-NOV-NEXT:    addiw a3, a3, -1
+; CHECK-NOV-NEXT:    bgeu a0, a3, .LBB43_10
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs5, rtz
-; CHECK-NOV-NEXT:    bgeu s1, a1, .LBB43_11
+; CHECK-NOV-NEXT:    fcvt.lu.s a1, fs5, rtz
+; CHECK-NOV-NEXT:    bgeu s1, a3, .LBB43_11
 ; CHECK-NOV-NEXT:  .LBB43_2: # %entry
-; CHECK-NOV-NEXT:    fcvt.lu.s a3, fs4, rtz
-; CHECK-NOV-NEXT:    bgeu a2, a1, .LBB43_12
+; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    bgeu a1, a3, .LBB43_12
 ; CHECK-NOV-NEXT:  .LBB43_3: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a4, fs3, rtz
-; CHECK-NOV-NEXT:    bgeu a3, a1, .LBB43_13
+; CHECK-NOV-NEXT:    bgeu a2, a3, .LBB43_13
 ; CHECK-NOV-NEXT:  .LBB43_4: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a5, fs2, rtz
-; CHECK-NOV-NEXT:    bgeu a4, a1, .LBB43_14
+; CHECK-NOV-NEXT:    bgeu a4, a3, .LBB43_14
 ; CHECK-NOV-NEXT:  .LBB43_5: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a6, fs1, rtz
-; CHECK-NOV-NEXT:    bgeu a5, a1, .LBB43_15
+; CHECK-NOV-NEXT:    bgeu a5, a3, .LBB43_15
 ; CHECK-NOV-NEXT:  .LBB43_6: # %entry
 ; CHECK-NOV-NEXT:    fcvt.lu.s a7, fs0, rtz
-; CHECK-NOV-NEXT:    bgeu a6, a1, .LBB43_16
+; CHECK-NOV-NEXT:    bgeu a6, a3, .LBB43_16
 ; CHECK-NOV-NEXT:  .LBB43_7: # %entry
-; CHECK-NOV-NEXT:    bltu a7, a1, .LBB43_9
+; CHECK-NOV-NEXT:    bltu a7, a3, .LBB43_9
 ; CHECK-NOV-NEXT:  .LBB43_8: # %entry
-; CHECK-NOV-NEXT:    mv a7, a1
+; CHECK-NOV-NEXT:    mv a7, a3
 ; CHECK-NOV-NEXT:  .LBB43_9: # %entry
-; CHECK-NOV-NEXT:    sh a7, 14(s0)
-; CHECK-NOV-NEXT:    sh a6, 12(s0)
-; CHECK-NOV-NEXT:    sh a5, 10(s0)
 ; CHECK-NOV-NEXT:    sh a4, 8(s0)
-; CHECK-NOV-NEXT:    sh a3, 6(s0)
-; CHECK-NOV-NEXT:    sh a2, 4(s0)
-; CHECK-NOV-NEXT:    sh s1, 2(s0)
+; CHECK-NOV-NEXT:    sh a5, 10(s0)
+; CHECK-NOV-NEXT:    sh a6, 12(s0)
+; CHECK-NOV-NEXT:    sh a7, 14(s0)
 ; CHECK-NOV-NEXT:    sh a0, 0(s0)
+; CHECK-NOV-NEXT:    sh s1, 2(s0)
+; CHECK-NOV-NEXT:    sh a1, 4(s0)
+; CHECK-NOV-NEXT:    sh a2, 6(s0)
 ; CHECK-NOV-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
@@ -4950,32 +4950,32 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    addi sp, sp, 128
 ; CHECK-NOV-NEXT:    ret
 ; CHECK-NOV-NEXT:  .LBB43_10: # %entry
-; CHECK-NOV-NEXT:    mv a0, a1
-; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs5, rtz
-; CHECK-NOV-NEXT:    bltu s1, a1, .LBB43_2
+; CHECK-NOV-NEXT:    mv a0, a3
+; CHECK-NOV-NEXT:    fcvt.lu.s a1, fs5, rtz
+; CHECK-NOV-NEXT:    bltu s1, a3, .LBB43_2
 ; CHECK-NOV-NEXT:  .LBB43_11: # %entry
-; CHECK-NOV-NEXT:    mv s1, a1
-; CHECK-NOV-NEXT:    fcvt.lu.s a3, fs4, rtz
-; CHECK-NOV-NEXT:    bltu a2, a1, .LBB43_3
+; CHECK-NOV-NEXT:    mv s1, a3
+; CHECK-NOV-NEXT:    fcvt.lu.s a2, fs4, rtz
+; CHECK-NOV-NEXT:    bltu a1, a3, .LBB43_3
 ; CHECK-NOV-NEXT:  .LBB43_12: # %entry
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a1, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a4, fs3, rtz
-; CHECK-NOV-NEXT:    bltu a3, a1, .LBB43_4
+; CHECK-NOV-NEXT:    bltu a2, a3, .LBB43_4
 ; CHECK-NOV-NEXT:  .LBB43_13: # %entry
-; CHECK-NOV-NEXT:    mv a3, a1
+; CHECK-NOV-NEXT:    mv a2, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a5, fs2, rtz
-; CHECK-NOV-NEXT:    bltu a4, a1, .LBB43_5
+; CHECK-NOV-NEXT:    bltu a4, a3, .LBB43_5
 ; CHECK-NOV-NEXT:  .LBB43_14: # %entry
-; CHECK-NOV-NEXT:    mv a4, a1
+; CHECK-NOV-NEXT:    mv a4, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a6, fs1, rtz
-; CHECK-NOV-NEXT:    bltu a5, a1, .LBB43_6
+; CHECK-NOV-NEXT:    bltu a5, a3, .LBB43_6
 ; CHECK-NOV-NEXT:  .LBB43_15: # %entry
-; CHECK-NOV-NEXT:    mv a5, a1
+; CHECK-NOV-NEXT:    mv a5, a3
 ; CHECK-NOV-NEXT:    fcvt.lu.s a7, fs0, rtz
-; CHECK-NOV-NEXT:    bltu a6, a1, .LBB43_7
+; CHECK-NOV-NEXT:    bltu a6, a3, .LBB43_7
 ; CHECK-NOV-NEXT:  .LBB43_16: # %entry
-; CHECK-NOV-NEXT:    mv a6, a1
-; CHECK-NOV-NEXT:    bgeu a7, a1, .LBB43_8
+; CHECK-NOV-NEXT:    mv a6, a3
+; CHECK-NOV-NEXT:    bgeu a7, a3, .LBB43_8
 ; CHECK-NOV-NEXT:    j .LBB43_9
 ;
 ; CHECK-V-LABEL: utesth_f16i16_mm:
@@ -5257,14 +5257,14 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) {
 ; CHECK-NOV-NEXT:    sgtz t0, a0
 ; CHECK-NOV-NEXT:    negw t0, t0
 ; CHECK-NOV-NEXT:    and a0, t0, a0
-; CHECK-NOV-NEXT:    sh a0, 14(s0)
-; CHECK-NOV-NEXT:    sh a7, 12(s0)
-; CHECK-NOV-NEXT:    sh a1, 10(s0)
 ; CHECK-NOV-NEXT:    sh a2, 8(s0)
-; CHECK-NOV-NEXT:    sh a4, 6(s0)
-; CHECK-NOV-NEXT:    sh a5, 4(s0)
-; CHECK-NOV-NEXT:    sh a6, 2(s0)
+; CHECK-NOV-NEXT:    sh a1, 10(s0)
+; CHECK-NOV-NEXT:    sh a7, 12(s0)
+; CHECK-NOV-NEXT:    sh a0, 14(s0)
 ; CHECK-NOV-NEXT:    sh a3, 0(s0)
+; CHECK-NOV-NEXT:    sh a6, 2(s0)
+; CHECK-NOV-NEXT:    sh a5, 4(s0)
+; CHECK-NOV-NEXT:    sh a4, 6(s0)
 ; CHECK-NOV-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s1, 104(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
index 3aca3130cc5459d21e0478f59cbf602d3339b612..5c14ed1e813c0ed01ea018c05dd00a52b549fa06 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
@@ -1372,8 +1372,8 @@ define <vscale x 1 x i1> @intrinsic_vmseq_mask_vx_nxv1i64_i64(<vscale x 1 x i64>
 ; RV32-LABEL: intrinsic_vmseq_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1408,8 +1408,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_mask_vx_nxv1i64_i64(<vscale x 1 x i64>
 ; RV32-LABEL: intrinsic_vmsge_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll b/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll
index accc18519d6260b2bed9ef70aa8d9e670cc70c5b..5349878b5d111740bf0d4b0cfb058e771b1ca4e7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/memset-inline.ll
@@ -31,14 +31,14 @@ define void @memset_1(ptr %a, i8 %value) nounwind {
 define void @memset_2(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_2:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 1(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_2:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 1(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_2:
@@ -63,18 +63,18 @@ define void @memset_2(ptr %a, i8 %value) nounwind {
 define void @memset_4(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_4:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    sb a1, 1(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_4:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    sb a1, 1(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_4:
@@ -102,26 +102,26 @@ define void @memset_4(ptr %a, i8 %value) nounwind {
 define void @memset_8(ptr %a, i8 %value) nounwind {
 ; RV32-LABEL: memset_8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb a1, 7(a0)
-; RV32-NEXT:    sb a1, 6(a0)
-; RV32-NEXT:    sb a1, 5(a0)
 ; RV32-NEXT:    sb a1, 4(a0)
-; RV32-NEXT:    sb a1, 3(a0)
-; RV32-NEXT:    sb a1, 2(a0)
-; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 5(a0)
+; RV32-NEXT:    sb a1, 6(a0)
+; RV32-NEXT:    sb a1, 7(a0)
 ; RV32-NEXT:    sb a1, 0(a0)
+; RV32-NEXT:    sb a1, 1(a0)
+; RV32-NEXT:    sb a1, 2(a0)
+; RV32-NEXT:    sb a1, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: memset_8:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb a1, 7(a0)
-; RV64-NEXT:    sb a1, 6(a0)
-; RV64-NEXT:    sb a1, 5(a0)
 ; RV64-NEXT:    sb a1, 4(a0)
-; RV64-NEXT:    sb a1, 3(a0)
-; RV64-NEXT:    sb a1, 2(a0)
-; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 5(a0)
+; RV64-NEXT:    sb a1, 6(a0)
+; RV64-NEXT:    sb a1, 7(a0)
 ; RV64-NEXT:    sb a1, 0(a0)
+; RV64-NEXT:    sb a1, 1(a0)
+; RV64-NEXT:    sb a1, 2(a0)
+; RV64-NEXT:    sb a1, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: memset_8:
@@ -130,8 +130,8 @@ define void @memset_8(ptr %a, i8 %value) nounwind {
 ; RV32-FAST-NEXT:    lui a2, 4112
 ; RV32-FAST-NEXT:    addi a2, a2, 257
 ; RV32-FAST-NEXT:    mul a1, a1, a2
-; RV32-FAST-NEXT:    sw a1, 4(a0)
 ; RV32-FAST-NEXT:    sw a1, 0(a0)
+; RV32-FAST-NEXT:    sw a1, 4(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: memset_8:
@@ -270,8 +270,8 @@ define void @aligned_memset_8(ptr align 8 %a, i8 %value) nounwind {
 ; RV32-BOTH-NEXT:    lui a2, 4112
 ; RV32-BOTH-NEXT:    addi a2, a2, 257
 ; RV32-BOTH-NEXT:    mul a1, a1, a2
-; RV32-BOTH-NEXT:    sw a1, 4(a0)
 ; RV32-BOTH-NEXT:    sw a1, 0(a0)
+; RV32-BOTH-NEXT:    sw a1, 4(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_memset_8:
@@ -377,14 +377,14 @@ define void @bzero_1(ptr %a) nounwind {
 define void @bzero_2(ptr %a) nounwind {
 ; RV32-LABEL: bzero_2:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 1(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_2:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 1(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_2:
@@ -403,18 +403,18 @@ define void @bzero_2(ptr %a) nounwind {
 define void @bzero_4(ptr %a) nounwind {
 ; RV32-LABEL: bzero_4:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 3(a0)
-; RV32-NEXT:    sb zero, 2(a0)
-; RV32-NEXT:    sb zero, 1(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 2(a0)
+; RV32-NEXT:    sb zero, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_4:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 3(a0)
-; RV64-NEXT:    sb zero, 2(a0)
-; RV64-NEXT:    sb zero, 1(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 2(a0)
+; RV64-NEXT:    sb zero, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_4:
@@ -433,32 +433,32 @@ define void @bzero_4(ptr %a) nounwind {
 define void @bzero_8(ptr %a) nounwind {
 ; RV32-LABEL: bzero_8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sb zero, 7(a0)
-; RV32-NEXT:    sb zero, 6(a0)
-; RV32-NEXT:    sb zero, 5(a0)
 ; RV32-NEXT:    sb zero, 4(a0)
-; RV32-NEXT:    sb zero, 3(a0)
-; RV32-NEXT:    sb zero, 2(a0)
-; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 5(a0)
+; RV32-NEXT:    sb zero, 6(a0)
+; RV32-NEXT:    sb zero, 7(a0)
 ; RV32-NEXT:    sb zero, 0(a0)
+; RV32-NEXT:    sb zero, 1(a0)
+; RV32-NEXT:    sb zero, 2(a0)
+; RV32-NEXT:    sb zero, 3(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: bzero_8:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sb zero, 7(a0)
-; RV64-NEXT:    sb zero, 6(a0)
-; RV64-NEXT:    sb zero, 5(a0)
 ; RV64-NEXT:    sb zero, 4(a0)
-; RV64-NEXT:    sb zero, 3(a0)
-; RV64-NEXT:    sb zero, 2(a0)
-; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 5(a0)
+; RV64-NEXT:    sb zero, 6(a0)
+; RV64-NEXT:    sb zero, 7(a0)
 ; RV64-NEXT:    sb zero, 0(a0)
+; RV64-NEXT:    sb zero, 1(a0)
+; RV64-NEXT:    sb zero, 2(a0)
+; RV64-NEXT:    sb zero, 3(a0)
 ; RV64-NEXT:    ret
 ;
 ; RV32-FAST-LABEL: bzero_8:
 ; RV32-FAST:       # %bb.0:
-; RV32-FAST-NEXT:    sw zero, 4(a0)
 ; RV32-FAST-NEXT:    sw zero, 0(a0)
+; RV32-FAST-NEXT:    sw zero, 4(a0)
 ; RV32-FAST-NEXT:    ret
 ;
 ; RV64-FAST-LABEL: bzero_8:
@@ -608,8 +608,8 @@ define void @aligned_bzero_4(ptr %a) nounwind {
 define void @aligned_bzero_8(ptr %a) nounwind {
 ; RV32-BOTH-LABEL: aligned_bzero_8:
 ; RV32-BOTH:       # %bb.0:
-; RV32-BOTH-NEXT:    sw zero, 4(a0)
 ; RV32-BOTH-NEXT:    sw zero, 0(a0)
+; RV32-BOTH-NEXT:    sw zero, 4(a0)
 ; RV32-BOTH-NEXT:    ret
 ;
 ; RV64-BOTH-LABEL: aligned_bzero_8:
diff --git a/llvm/test/CodeGen/RISCV/rvv/no-reserved-frame.ll b/llvm/test/CodeGen/RISCV/rvv/no-reserved-frame.ll
index 9cdced88c7bdc2348184d30babfe67eb60dc6925..59ba857dca8a5f340efd5e6e8982571fea646079 100644
--- a/llvm/test/CodeGen/RISCV/rvv/no-reserved-frame.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/no-reserved-frame.ll
@@ -27,15 +27,15 @@ define signext i32 @foo(i32 signext %aa) #0 {
 ; CHECK-NEXT:    lw a6, 24(s1)
 ; CHECK-NEXT:    lw a7, 20(s1)
 ; CHECK-NEXT:    lw t1, 16(s1)
-; CHECK-NEXT:    lw a1, 12(s1)
-; CHECK-NEXT:    lw t2, 8(s1)
+; CHECK-NEXT:    lw t2, 12(s1)
+; CHECK-NEXT:    lw t3, 8(s1)
 ; CHECK-NEXT:    sw a0, 52(s1)
 ; CHECK-NEXT:    sw a0, 48(s1)
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd t2, 16(sp)
-; CHECK-NEXT:    sd a1, 8(sp)
 ; CHECK-NEXT:    addi a1, s1, 48
 ; CHECK-NEXT:    sd t1, 0(sp)
+; CHECK-NEXT:    sd t2, 8(sp)
+; CHECK-NEXT:    sd t3, 16(sp)
 ; CHECK-NEXT:    mv a0, t0
 ; CHECK-NEXT:    call gfunc
 ; CHECK-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr104480.ll b/llvm/test/CodeGen/RISCV/rvv/pr104480.ll
index 93cf4d3766089a12ed866b1b8a7797ccc5ca583e..1e34d9aa6d0568fd1f9b08da3f01625a299866b2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/pr104480.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/pr104480.ll
@@ -45,9 +45,9 @@ define <vscale x 4 x i64> @test_mulhs_expand(<vscale x 4 x i64> %broadcast.splat
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    addi a1, a0, 1365
-; CHECK-NEXT:    sw a1, 12(sp)
 ; CHECK-NEXT:    addi a0, a0, 1366
 ; CHECK-NEXT:    sw a0, 8(sp)
+; CHECK-NEXT:    sw a1, 12(sp)
 ; CHECK-NEXT:    addi a0, sp, 8
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vlse64.v v12, (a0), zero
@@ -73,9 +73,9 @@ define <vscale x 4 x i64> @test_mulhu_expand(<vscale x 4 x i64> %broadcast.splat
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    lui a0, 699051
 ; CHECK-NEXT:    addi a1, a0, -1366
-; CHECK-NEXT:    sw a1, 12(sp)
 ; CHECK-NEXT:    addi a0, a0, -1365
 ; CHECK-NEXT:    sw a0, 8(sp)
+; CHECK-NEXT:    sw a1, 12(sp)
 ; CHECK-NEXT:    addi a0, sp, 8
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vlse64.v v12, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll
index c164b62a679be05ca8ebbaa5aed024dcad07293c..7a2e40e86f00274527ed89bc09bdb797d49acef0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-args-by-mem.ll
@@ -45,8 +45,8 @@ define <vscale x 16 x i32> @foo(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5,
 ; CHECK-NEXT:    add t1, s1, t1
 ; CHECK-NEXT:    addi t1, t1, 64
 ; CHECK-NEXT:    vs8r.v v8, (t1)
-; CHECK-NEXT:    sd t0, 8(sp)
 ; CHECK-NEXT:    sd t1, 0(sp)
+; CHECK-NEXT:    sd t0, 8(sp)
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    call bar
 ; CHECK-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
index eb8c58d2d377907fd456f402965a6284e2c8c17f..7315fd6cfbbecb406bd18b5e7b92cfde9ce0cb0f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
@@ -2375,8 +2375,8 @@ define <vscale x 1 x i1> @icmp_eq_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2401,8 +2401,8 @@ define <vscale x 1 x i1> @icmp_eq_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %b
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2457,8 +2457,8 @@ define <vscale x 1 x i1> @icmp_ne_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2483,8 +2483,8 @@ define <vscale x 1 x i1> @icmp_ne_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %b
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2539,8 +2539,8 @@ define <vscale x 1 x i1> @icmp_ugt_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2565,8 +2565,8 @@ define <vscale x 1 x i1> @icmp_ugt_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2621,8 +2621,8 @@ define <vscale x 1 x i1> @icmp_uge_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2649,8 +2649,8 @@ define <vscale x 1 x i1> @icmp_uge_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2705,8 +2705,8 @@ define <vscale x 1 x i1> @icmp_ult_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2731,8 +2731,8 @@ define <vscale x 1 x i1> @icmp_ult_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2787,8 +2787,8 @@ define <vscale x 1 x i1> @icmp_sgt_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2813,8 +2813,8 @@ define <vscale x 1 x i1> @icmp_sgt_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2869,8 +2869,8 @@ define <vscale x 1 x i1> @icmp_sge_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2897,8 +2897,8 @@ define <vscale x 1 x i1> @icmp_sge_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2953,8 +2953,8 @@ define <vscale x 1 x i1> @icmp_slt_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2979,8 +2979,8 @@ define <vscale x 1 x i1> @icmp_slt_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -3035,8 +3035,8 @@ define <vscale x 1 x i1> @icmp_sle_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -3061,8 +3061,8 @@ define <vscale x 1 x i1> @icmp_sle_vx_swap_nxv1i64(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -3122,8 +3122,8 @@ define <vscale x 8 x i1> @icmp_eq_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3150,8 +3150,8 @@ define <vscale x 8 x i1> @icmp_eq_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %b
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3211,8 +3211,8 @@ define <vscale x 8 x i1> @icmp_ne_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3239,8 +3239,8 @@ define <vscale x 8 x i1> @icmp_ne_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %b
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3300,8 +3300,8 @@ define <vscale x 8 x i1> @icmp_ugt_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3328,8 +3328,8 @@ define <vscale x 8 x i1> @icmp_ugt_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3389,8 +3389,8 @@ define <vscale x 8 x i1> @icmp_uge_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3419,8 +3419,8 @@ define <vscale x 8 x i1> @icmp_uge_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3480,8 +3480,8 @@ define <vscale x 8 x i1> @icmp_ult_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3508,8 +3508,8 @@ define <vscale x 8 x i1> @icmp_ult_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3569,8 +3569,8 @@ define <vscale x 8 x i1> @icmp_sgt_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3597,8 +3597,8 @@ define <vscale x 8 x i1> @icmp_sgt_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3658,8 +3658,8 @@ define <vscale x 8 x i1> @icmp_sge_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3688,8 +3688,8 @@ define <vscale x 8 x i1> @icmp_sge_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3749,8 +3749,8 @@ define <vscale x 8 x i1> @icmp_slt_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3777,8 +3777,8 @@ define <vscale x 8 x i1> @icmp_slt_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3838,8 +3838,8 @@ define <vscale x 8 x i1> @icmp_sle_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -3866,8 +3866,8 @@ define <vscale x 8 x i1> @icmp_sle_vx_swap_nxv8i64(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
index 90ffeff9689e074041cadad8d851a2eea7953871..cd3f02f0400f0c80e75ef465eeb498abdf6b7194 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
@@ -2055,8 +2055,8 @@ define <vscale x 8 x i1> @icmp_eq_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2080,8 +2080,8 @@ define <vscale x 8 x i1> @icmp_eq_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2145,8 +2145,8 @@ define <vscale x 8 x i1> @icmp_ne_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2170,8 +2170,8 @@ define <vscale x 8 x i1> @icmp_ne_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2215,8 +2215,8 @@ define <vscale x 8 x i1> @icmp_ugt_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2240,8 +2240,8 @@ define <vscale x 8 x i1> @icmp_ugt_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2285,8 +2285,8 @@ define <vscale x 8 x i1> @icmp_uge_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2311,8 +2311,8 @@ define <vscale x 8 x i1> @icmp_uge_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2417,8 +2417,8 @@ define <vscale x 8 x i1> @icmp_ult_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2442,8 +2442,8 @@ define <vscale x 8 x i1> @icmp_ult_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2538,8 +2538,8 @@ define <vscale x 8 x i1> @icmp_ule_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2563,8 +2563,8 @@ define <vscale x 8 x i1> @icmp_ule_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2609,8 +2609,8 @@ define <vscale x 8 x i1> @icmp_sgt_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2634,8 +2634,8 @@ define <vscale x 8 x i1> @icmp_sgt_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2679,8 +2679,8 @@ define <vscale x 8 x i1> @icmp_sge_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2705,8 +2705,8 @@ define <vscale x 8 x i1> @icmp_sge_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2791,8 +2791,8 @@ define <vscale x 8 x i1> @icmp_slt_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2816,8 +2816,8 @@ define <vscale x 8 x i1> @icmp_slt_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2902,8 +2902,8 @@ define <vscale x 8 x i1> @icmp_sle_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2927,8 +2927,8 @@ define <vscale x 8 x i1> @icmp_sle_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/splat-vector-split-i64-vl-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/splat-vector-split-i64-vl-sdnode.ll
index 28583efccdbca45ec55c632b86c1960e5b5c51ac..3912bc48c795194600a31715d44eb028d39e440d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/splat-vector-split-i64-vl-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/splat-vector-split-i64-vl-sdnode.ll
@@ -14,9 +14,9 @@ define i32 @splat_vector_split_i64() {
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, tu, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 3
-; CHECK-NEXT:    sw zero, 12(sp)
 ; CHECK-NEXT:    lui a0, 1044480
 ; CHECK-NEXT:    sw a0, 8(sp)
+; CHECK-NEXT:    sw zero, 12(sp)
 ; CHECK-NEXT:    li a0, 56
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vsrl.vx v10, v8, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/stepvector.ll b/llvm/test/CodeGen/RISCV/rvv/stepvector.ll
index 721f03120bd4994162428ca08402e67e9fb5f59e..27a4d37a83da837f3764d1396b057d871441411d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/stepvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/stepvector.ll
@@ -483,10 +483,10 @@ define <vscale x 8 x i64> @mul_bigimm_stepvector_nxv8i64() {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    li a0, 7
+; RV32-NEXT:    lui a1, 797989
+; RV32-NEXT:    addi a1, a1, -683
+; RV32-NEXT:    sw a1, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
-; RV32-NEXT:    lui a0, 797989
-; RV32-NEXT:    addi a0, a0, -683
-; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -531,9 +531,9 @@ define <vscale x 16 x i64> @stepvector_nxv16i64() {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -558,10 +558,10 @@ define <vscale x 16 x i64> @add_stepvector_nxv16i64() {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 1
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -592,11 +592,11 @@ define <vscale x 16 x i64> @mul_stepvector_nxv16i64() {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a1, a0, 1
 ; RV32-NEXT:    add a0, a1, a0
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -630,23 +630,23 @@ define <vscale x 16 x i64> @mul_bigimm_stepvector_nxv16i64() {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    li a0, 7
+; RV32-NEXT:    lui a1, 797989
+; RV32-NEXT:    addi a1, a1, -683
+; RV32-NEXT:    sw a1, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
-; RV32-NEXT:    lui a0, 797989
-; RV32-NEXT:    addi a0, a0, -683
-; RV32-NEXT:    sw a0, 8(sp)
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    lui a1, 11557
 ; RV32-NEXT:    addi a1, a1, -683
 ; RV32-NEXT:    mul a1, a0, a1
-; RV32-NEXT:    sw a1, 0(sp)
 ; RV32-NEXT:    srli a0, a0, 3
-; RV32-NEXT:    lui a1, 92455
-; RV32-NEXT:    addi a1, a1, -1368
-; RV32-NEXT:    mulhu a1, a0, a1
-; RV32-NEXT:    slli a2, a0, 1
+; RV32-NEXT:    lui a2, 92455
+; RV32-NEXT:    addi a2, a2, -1368
+; RV32-NEXT:    mulhu a2, a0, a2
+; RV32-NEXT:    slli a3, a0, 1
 ; RV32-NEXT:    slli a0, a0, 6
-; RV32-NEXT:    sub a0, a0, a2
-; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sub a0, a0, a3
+; RV32-NEXT:    add a0, a2, a0
+; RV32-NEXT:    sw a1, 0(sp)
 ; RV32-NEXT:    sw a0, 4(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
@@ -683,10 +683,10 @@ define <vscale x 16 x i64> @shl_stepvector_nxv16i64() {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 2
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll b/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll
index ae8c36a7cb5e3d6465cdace00c4bfed1f56186c1..bcc7bb9f072f6d28bb539792853671c28d8a18d7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unmasked-ta.ll
@@ -279,8 +279,8 @@ define <vscale x 1 x i64>  @intrinsic_vmacc_vx_nxv1i64_i64_nxv1i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vmacc_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -314,8 +314,8 @@ define <vscale x 1 x i64>  @intrinsic_vmadd_vx_nxv1i64_i64_nxv1i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vmadd_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -349,8 +349,8 @@ define <vscale x 1 x i64>  @intrinsic_vnmsac_vx_nxv1i64_i64_nxv1i64(<vscale x 1
 ; RV32-LABEL: intrinsic_vnmsac_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -384,8 +384,8 @@ define <vscale x 1 x i64>  @intrinsic_vnmsub_vx_nxv1i64_i64_nxv1i64(<vscale x 1
 ; RV32-LABEL: intrinsic_vnmsub_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -905,8 +905,8 @@ define <vscale x 1 x i64> @intrinsic_vmv.s.x_x_nxv1i64(i64 %0, iXLen %1) nounwin
 ; RV32-LABEL: intrinsic_vmv.s.x_x_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
index 1f027aef3103d4ec1f8390885afb41dde6ba00bc..aeb3f6c174859c4ec874279ecfb248afb8bf43ed 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
@@ -1157,8 +1157,8 @@ define <vscale x 1 x i64> @intrinsic_vrsub_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vrsub_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1192,8 +1192,8 @@ define <vscale x 1 x i64> @intrinsic_vsadd_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vsadd_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1296,8 +1296,8 @@ define <vscale x 1 x i64> @intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1468,8 +1468,8 @@ define <vscale x 1 x i64> @intrinsic_vssub_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vssub_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1503,8 +1503,8 @@ define <vscale x 1 x i64> @intrinsic_vssubu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vssubu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2492,8 +2492,8 @@ define <vscale x 8 x i64> @intrinsic_vmerge_vxm_nxv8i64_nxv8i64_i64(<vscale x 8
 ; RV32-LABEL: intrinsic_vmerge_vxm_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -2523,9 +2523,9 @@ define <vscale x 8 x i64> @intrinsic_vmerge_vim_nxv8i64_nxv8i64_i64(<vscale x 8
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    li a1, 15
+; RV32-NEXT:    li a2, -1
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    li a1, -1
-; RV32-NEXT:    sw a1, 8(sp)
 ; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a1), zero
@@ -2673,8 +2673,8 @@ define <vscale x 1 x i64> @intrinsic_vmv.v.x_x_nxv1i64(<vscale x 1 x i64> %0, i6
 ; RV32-LABEL: intrinsic_vmv.v.x_x_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, tu, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vaadd.ll b/llvm/test/CodeGen/RISCV/rvv/vaadd.ll
index 096e60b6285ffdf38786a392771fdaab73c209d7..28b8b180b76772ac8d9f81f2acfb34c22a69ab4c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vaadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vaadd.ll
@@ -1938,8 +1938,8 @@ define <vscale x 1 x i64> @intrinsic_vaadd_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vaadd_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1975,8 +1975,8 @@ define <vscale x 1 x i64> @intrinsic_vaadd_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vaadd_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2012,8 +2012,8 @@ define <vscale x 2 x i64> @intrinsic_vaadd_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vaadd_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2049,8 +2049,8 @@ define <vscale x 2 x i64> @intrinsic_vaadd_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vaadd_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2086,8 +2086,8 @@ define <vscale x 4 x i64> @intrinsic_vaadd_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vaadd_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2123,8 +2123,8 @@ define <vscale x 4 x i64> @intrinsic_vaadd_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vaadd_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2160,8 +2160,8 @@ define <vscale x 8 x i64> @intrinsic_vaadd_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vaadd_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2197,8 +2197,8 @@ define <vscale x 8 x i64> @intrinsic_vaadd_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vaadd_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll
index cd9edca1d4c44fff998670658ce0bcc14d2322d7..b03948c15d0780175329771d8cac7bffcb596673 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll
@@ -163,8 +163,8 @@ define <vscale x 8 x i64> @vaaddu_vx_nxv8i64_floor(<vscale x 8 x i64> %x, i64 %y
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -382,8 +382,8 @@ define <vscale x 8 x i64> @vaaddu_vx_nxv8i64_ceil(<vscale x 8 x i64> %x, i64 %y)
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll b/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll
index a15a1932360a5cf219a30f4806ae2dc1fe08c7d2..7dd556dd16cf134860bbc48873628337127e9ea5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll
@@ -1938,8 +1938,8 @@ define <vscale x 1 x i64> @intrinsic_vaaddu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vaaddu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1975,8 +1975,8 @@ define <vscale x 1 x i64> @intrinsic_vaaddu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vaaddu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2012,8 +2012,8 @@ define <vscale x 2 x i64> @intrinsic_vaaddu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vaaddu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2049,8 +2049,8 @@ define <vscale x 2 x i64> @intrinsic_vaaddu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vaaddu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2086,8 +2086,8 @@ define <vscale x 4 x i64> @intrinsic_vaaddu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vaaddu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2123,8 +2123,8 @@ define <vscale x 4 x i64> @intrinsic_vaaddu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vaaddu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2160,8 +2160,8 @@ define <vscale x 8 x i64> @intrinsic_vaaddu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vaaddu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2197,8 +2197,8 @@ define <vscale x 8 x i64> @intrinsic_vaaddu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vaaddu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vadc.ll b/llvm/test/CodeGen/RISCV/rvv/vadc.ll
index c33fb3a0c9a89133e04afe3d707cba26554b224b..d2bd58bf3dc67e5fbfce047cd8436c58b1c6f575 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vadc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vadc.ll
@@ -975,8 +975,8 @@ define <vscale x 1 x i64> @intrinsic_vadc_vxm_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vadc_vxm_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1011,8 +1011,8 @@ define <vscale x 2 x i64> @intrinsic_vadc_vxm_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vadc_vxm_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1047,8 +1047,8 @@ define <vscale x 4 x i64> @intrinsic_vadc_vxm_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vadc_vxm_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1083,8 +1083,8 @@ define <vscale x 8 x i64> @intrinsic_vadc_vxm_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vadc_vxm_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vadd-sdnode.ll
index 2b141097366cfbd07258991d158ddd17be39180d..e6dd81f42d838e0a92b022ff9864ac4dec98c4ee 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vadd-sdnode.ll
@@ -594,8 +594,8 @@ define <vscale x 1 x i64> @vadd_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -639,8 +639,8 @@ define <vscale x 2 x i64> @vadd_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -684,8 +684,8 @@ define <vscale x 4 x i64> @vadd_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -729,8 +729,8 @@ define <vscale x 8 x i64> @vadd_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -774,10 +774,10 @@ define <vscale x 8 x i64> @vadd_xx_nxv8i64(i64 %a, i64 %b) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    add a2, a0, a2
-; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    add a1, a1, a3
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sw a2, 8(sp)
 ; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
index feeef73e538ae0053a41d60643045cd65e905fec..58dee75dd55a15191dc4c70828695320929d3bdd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
@@ -1484,8 +1484,8 @@ define <vscale x 1 x i64> @vadd_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1510,8 +1510,8 @@ define <vscale x 1 x i64> @vadd_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1578,8 +1578,8 @@ define <vscale x 2 x i64> @vadd_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1604,8 +1604,8 @@ define <vscale x 2 x i64> @vadd_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1672,8 +1672,8 @@ define <vscale x 4 x i64> @vadd_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1698,8 +1698,8 @@ define <vscale x 4 x i64> @vadd_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1766,8 +1766,8 @@ define <vscale x 8 x i64> @vadd_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1792,8 +1792,8 @@ define <vscale x 8 x i64> @vadd_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vadd.ll b/llvm/test/CodeGen/RISCV/rvv/vadd.ll
index 2654e7daafb0c24ee9fd5882c3f1b2a22c2ef409..6c8cab84840258379454ee4d7297469f2e19ead7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vadd.ll
@@ -1858,8 +1858,8 @@ define <vscale x 1 x i64> @intrinsic_vadd_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vadd_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1936,8 +1936,8 @@ define <vscale x 1 x i64> @intrinsic_vadd_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vadd_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1971,8 +1971,8 @@ define <vscale x 2 x i64> @intrinsic_vadd_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vadd_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2006,8 +2006,8 @@ define <vscale x 2 x i64> @intrinsic_vadd_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vadd_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2041,8 +2041,8 @@ define <vscale x 4 x i64> @intrinsic_vadd_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vadd_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vadd_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vadd_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vadd_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vadd_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2146,8 +2146,8 @@ define <vscale x 8 x i64> @intrinsic_vadd_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vadd_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vand-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vand-sdnode.ll
index a84e2c984f669c25b523f61e5448ec0af4d1fa95..de1e220e9f5666faa70241ea23b555a74616ffc1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vand-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vand-sdnode.ll
@@ -971,8 +971,8 @@ define <vscale x 1 x i64> @vand_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1037,8 +1037,8 @@ define <vscale x 2 x i64> @vand_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1103,8 +1103,8 @@ define <vscale x 4 x i64> @vand_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1169,8 +1169,8 @@ define <vscale x 8 x i64> @vand_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1225,9 +1225,9 @@ define <vscale x 8 x i64> @vand_xx_nxv8i64(i64 %a, i64 %b) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    and a1, a1, a3
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    and a0, a0, a2
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vand-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vand-vp.ll
index 7b4a68d5867f992d2b8422296ab57bd5ed8cd3aa..03ea4646fcf8ac5fe64a98e6eabc2fd0cc2cf880 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vand-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vand-vp.ll
@@ -1311,8 +1311,8 @@ define <vscale x 1 x i64> @vand_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1337,8 +1337,8 @@ define <vscale x 1 x i64> @vand_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1405,8 +1405,8 @@ define <vscale x 2 x i64> @vand_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1431,8 +1431,8 @@ define <vscale x 2 x i64> @vand_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1499,8 +1499,8 @@ define <vscale x 4 x i64> @vand_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1525,8 +1525,8 @@ define <vscale x 4 x i64> @vand_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1593,8 +1593,8 @@ define <vscale x 8 x i64> @vand_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1619,8 +1619,8 @@ define <vscale x 8 x i64> @vand_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vand.ll b/llvm/test/CodeGen/RISCV/rvv/vand.ll
index 2cf5eab4da6a617f7a99b3918549a5448fd05126..94b4b5748b220d43be2f2ce2235a1852bcf8e427 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vand.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vand.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vand_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vand_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vand_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vand_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vand_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vand_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vand_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vand_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vand_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vand_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vand_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vand_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vand_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vand_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vand_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vand_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vandn-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vandn-sdnode.ll
index f25a3f937f1b4fff6aebea2863302f2447982b15..fb912c8a28432ebd96837887037439b1501808c7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vandn-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vandn-sdnode.ll
@@ -1415,8 +1415,8 @@ define <vscale x 1 x i64> @vandn_vx_nxv1i64(i64 %x, <vscale x 1 x i64> %y) {
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1437,8 +1437,8 @@ define <vscale x 1 x i64> @vandn_vx_nxv1i64(i64 %x, <vscale x 1 x i64> %y) {
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v9, (a0), zero
@@ -1465,8 +1465,8 @@ define <vscale x 1 x i64> @vandn_vx_swapped_nxv1i64(i64 %x, <vscale x 1 x i64> %
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1487,8 +1487,8 @@ define <vscale x 1 x i64> @vandn_vx_swapped_nxv1i64(i64 %x, <vscale x 1 x i64> %
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v9, (a0), zero
@@ -1551,8 +1551,8 @@ define <vscale x 2 x i64> @vandn_vx_nxv2i64(i64 %x, <vscale x 2 x i64> %y) {
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1573,8 +1573,8 @@ define <vscale x 2 x i64> @vandn_vx_nxv2i64(i64 %x, <vscale x 2 x i64> %y) {
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v10, (a0), zero
@@ -1601,8 +1601,8 @@ define <vscale x 2 x i64> @vandn_vx_swapped_nxv2i64(i64 %x, <vscale x 2 x i64> %
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1623,8 +1623,8 @@ define <vscale x 2 x i64> @vandn_vx_swapped_nxv2i64(i64 %x, <vscale x 2 x i64> %
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v10, (a0), zero
@@ -1687,8 +1687,8 @@ define <vscale x 4 x i64> @vandn_vx_nxv4i64(i64 %x, <vscale x 4 x i64> %y) {
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1709,8 +1709,8 @@ define <vscale x 4 x i64> @vandn_vx_nxv4i64(i64 %x, <vscale x 4 x i64> %y) {
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v12, (a0), zero
@@ -1737,8 +1737,8 @@ define <vscale x 4 x i64> @vandn_vx_swapped_nxv4i64(i64 %x, <vscale x 4 x i64> %
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1759,8 +1759,8 @@ define <vscale x 4 x i64> @vandn_vx_swapped_nxv4i64(i64 %x, <vscale x 4 x i64> %
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v12, (a0), zero
@@ -1823,8 +1823,8 @@ define <vscale x 8 x i64> @vandn_vx_nxv8i64(i64 %x, <vscale x 8 x i64> %y) {
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1845,8 +1845,8 @@ define <vscale x 8 x i64> @vandn_vx_nxv8i64(i64 %x, <vscale x 8 x i64> %y) {
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v16, (a0), zero
@@ -1873,8 +1873,8 @@ define <vscale x 8 x i64> @vandn_vx_swapped_nxv8i64(i64 %x, <vscale x 8 x i64> %
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1895,8 +1895,8 @@ define <vscale x 8 x i64> @vandn_vx_swapped_nxv8i64(i64 %x, <vscale x 8 x i64> %
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vandn-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vandn-vp.ll
index 95866543828fc7fc751c0d2c9fe68bf83c307115..5986033638853b549352f6e20c00db53cf4fdc7c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vandn-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vandn-vp.ll
@@ -1112,8 +1112,8 @@ define <vscale x 1 x i64> @vandn_vx_vp_nxv1i64(i64 %a, <vscale x 1 x i64> %b, <v
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1135,8 +1135,8 @@ define <vscale x 1 x i64> @vandn_vx_vp_nxv1i64(i64 %a, <vscale x 1 x i64> %b, <v
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v9, (a0), zero
@@ -1203,8 +1203,8 @@ define <vscale x 2 x i64> @vandn_vx_vp_nxv2i64(i64 %a, <vscale x 2 x i64> %b, <v
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1226,8 +1226,8 @@ define <vscale x 2 x i64> @vandn_vx_vp_nxv2i64(i64 %a, <vscale x 2 x i64> %b, <v
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v10, (a0), zero
@@ -1294,8 +1294,8 @@ define <vscale x 4 x i64> @vandn_vx_vp_nxv4i64(i64 %a, <vscale x 4 x i64> %b, <v
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1317,8 +1317,8 @@ define <vscale x 4 x i64> @vandn_vx_vp_nxv4i64(i64 %a, <vscale x 4 x i64> %b, <v
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v12, (a0), zero
@@ -1385,8 +1385,8 @@ define <vscale x 8 x i64> @vandn_vx_vp_nxv8i64(i64 %a, <vscale x 8 x i64> %b, <v
 ; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-RV32-NEXT:    not a0, a0
 ; CHECK-RV32-NEXT:    not a1, a1
-; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    sw a0, 8(sp)
+; CHECK-RV32-NEXT:    sw a1, 12(sp)
 ; CHECK-RV32-NEXT:    addi a0, sp, 8
 ; CHECK-RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1408,8 +1408,8 @@ define <vscale x 8 x i64> @vandn_vx_vp_nxv8i64(i64 %a, <vscale x 8 x i64> %b, <v
 ; CHECK-ZVKB32-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-ZVKB32-NEXT:    not a0, a0
 ; CHECK-ZVKB32-NEXT:    not a1, a1
-; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    sw a0, 8(sp)
+; CHECK-ZVKB32-NEXT:    sw a1, 12(sp)
 ; CHECK-ZVKB32-NEXT:    addi a0, sp, 8
 ; CHECK-ZVKB32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-ZVKB32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vandn.ll b/llvm/test/CodeGen/RISCV/rvv/vandn.ll
index a5e29d488da522a6c0f469659da37c883462547c..de4ad5088d374b614e2df9a9247056996be0de63 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vandn.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vandn.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vandn_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vandn_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vandn_mask_vx_nxv1i64_i64(<vscale x 1 x i64
 ; RV32-LABEL: intrinsic_vandn_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vandn_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vandn_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vandn_mask_vx_nxv2i64_i64(<vscale x 2 x i64
 ; RV32-LABEL: intrinsic_vandn_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vandn_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vandn_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vandn_mask_vx_nxv4i64_i64(<vscale x 4 x i64
 ; RV32-LABEL: intrinsic_vandn_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vandn_vx_nxv8i64_i64(<vscale x 8 x i64> %0,
 ; RV32-LABEL: intrinsic_vandn_vx_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vandn_mask_vx_nxv8i64_i64(<vscale x 8 x i64
 ; RV32-LABEL: intrinsic_vandn_mask_vx_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vasub.ll b/llvm/test/CodeGen/RISCV/rvv/vasub.ll
index c3cb66f7f230a607187f4700e4a90f003f94f597..43903ba5833bef89c2f34c10ea8925322ab1b348 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vasub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vasub.ll
@@ -1938,8 +1938,8 @@ define <vscale x 1 x i64> @intrinsic_vasub_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vasub_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1975,8 +1975,8 @@ define <vscale x 1 x i64> @intrinsic_vasub_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vasub_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2012,8 +2012,8 @@ define <vscale x 2 x i64> @intrinsic_vasub_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vasub_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2049,8 +2049,8 @@ define <vscale x 2 x i64> @intrinsic_vasub_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vasub_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2086,8 +2086,8 @@ define <vscale x 4 x i64> @intrinsic_vasub_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vasub_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2123,8 +2123,8 @@ define <vscale x 4 x i64> @intrinsic_vasub_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vasub_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2160,8 +2160,8 @@ define <vscale x 8 x i64> @intrinsic_vasub_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vasub_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2197,8 +2197,8 @@ define <vscale x 8 x i64> @intrinsic_vasub_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vasub_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vasubu.ll b/llvm/test/CodeGen/RISCV/rvv/vasubu.ll
index 6d790a9ce0277de0fd80ea3ff91bc6cc5a99c827..cfa0f31857e150d87ff5844cb70e24025aaae1eb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vasubu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vasubu.ll
@@ -1938,8 +1938,8 @@ define <vscale x 1 x i64> @intrinsic_vasubu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vasubu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1975,8 +1975,8 @@ define <vscale x 1 x i64> @intrinsic_vasubu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vasubu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2012,8 +2012,8 @@ define <vscale x 2 x i64> @intrinsic_vasubu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vasubu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2049,8 +2049,8 @@ define <vscale x 2 x i64> @intrinsic_vasubu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vasubu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2086,8 +2086,8 @@ define <vscale x 4 x i64> @intrinsic_vasubu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vasubu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2123,8 +2123,8 @@ define <vscale x 4 x i64> @intrinsic_vasubu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vasubu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2160,8 +2160,8 @@ define <vscale x 8 x i64> @intrinsic_vasubu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vasubu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2197,8 +2197,8 @@ define <vscale x 8 x i64> @intrinsic_vasubu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vasubu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vclmul.ll b/llvm/test/CodeGen/RISCV/rvv/vclmul.ll
index 1184d4b96cc5e332140759430f683c5ec4aed1b7..c132145f71af074fadd6561e04f8f8df5a5c88bf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vclmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vclmul.ll
@@ -203,8 +203,8 @@ define <vscale x 1 x i64> @intrinsic_vclmul_vx_nxv1i64_i64(<vscale x 1 x i64> %0
 ; RV32-LABEL: intrinsic_vclmul_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -239,8 +239,8 @@ define <vscale x 1 x i64> @intrinsic_vclmul_mask_vx_nxv1i64_i64(<vscale x 1 x i6
 ; RV32-LABEL: intrinsic_vclmul_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -274,8 +274,8 @@ define <vscale x 2 x i64> @intrinsic_vclmul_vx_nxv2i64_i64(<vscale x 2 x i64> %0
 ; RV32-LABEL: intrinsic_vclmul_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -310,8 +310,8 @@ define <vscale x 2 x i64> @intrinsic_vclmul_mask_vx_nxv2i64_i64(<vscale x 2 x i6
 ; RV32-LABEL: intrinsic_vclmul_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -345,8 +345,8 @@ define <vscale x 4 x i64> @intrinsic_vclmul_vx_nxv4i64_i64(<vscale x 4 x i64> %0
 ; RV32-LABEL: intrinsic_vclmul_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -381,8 +381,8 @@ define <vscale x 4 x i64> @intrinsic_vclmul_mask_vx_nxv4i64_i64(<vscale x 4 x i6
 ; RV32-LABEL: intrinsic_vclmul_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -416,8 +416,8 @@ define <vscale x 8 x i64> @intrinsic_vclmul_vx_nxv8i64_i64(<vscale x 8 x i64> %0
 ; RV32-LABEL: intrinsic_vclmul_vx_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -452,8 +452,8 @@ define <vscale x 8 x i64> @intrinsic_vclmul_mask_vx_nxv8i64_i64(<vscale x 8 x i6
 ; RV32-LABEL: intrinsic_vclmul_mask_vx_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vclmulh.ll b/llvm/test/CodeGen/RISCV/rvv/vclmulh.ll
index 98befd81ace0c68754abf42bfb4ea41517006380..287c8e95d1f401b0f6684b997b87cc3fd740ca57 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vclmulh.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vclmulh.ll
@@ -203,8 +203,8 @@ define <vscale x 1 x i64> @intrinsic_vclmulh_vx_nxv1i64_i64(<vscale x 1 x i64> %
 ; RV32-LABEL: intrinsic_vclmulh_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -239,8 +239,8 @@ define <vscale x 1 x i64> @intrinsic_vclmulh_mask_vx_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vclmulh_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -274,8 +274,8 @@ define <vscale x 2 x i64> @intrinsic_vclmulh_vx_nxv2i64_i64(<vscale x 2 x i64> %
 ; RV32-LABEL: intrinsic_vclmulh_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -310,8 +310,8 @@ define <vscale x 2 x i64> @intrinsic_vclmulh_mask_vx_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vclmulh_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -345,8 +345,8 @@ define <vscale x 4 x i64> @intrinsic_vclmulh_vx_nxv4i64_i64(<vscale x 4 x i64> %
 ; RV32-LABEL: intrinsic_vclmulh_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -381,8 +381,8 @@ define <vscale x 4 x i64> @intrinsic_vclmulh_mask_vx_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vclmulh_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -416,8 +416,8 @@ define <vscale x 8 x i64> @intrinsic_vclmulh_vx_nxv8i64_i64(<vscale x 8 x i64> %
 ; RV32-LABEL: intrinsic_vclmulh_vx_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -452,8 +452,8 @@ define <vscale x 8 x i64> @intrinsic_vclmulh_mask_vx_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vclmulh_mask_vx_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode.ll
index ef9b2104b2d2d29f04d7dc8e6115f605cb6795a6..24ce211f0a984f6ab4111a15dfe69d6378a81683 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode.ll
@@ -770,8 +770,8 @@ define <vscale x 1 x i64> @vdiv_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -797,10 +797,10 @@ define <vscale x 1 x i64> @vdiv_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-V-NEXT:    vlse64.v v9, (a0), zero
@@ -849,8 +849,8 @@ define <vscale x 2 x i64> @vdiv_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -876,10 +876,10 @@ define <vscale x 2 x i64> @vdiv_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-V-NEXT:    vlse64.v v10, (a0), zero
@@ -928,8 +928,8 @@ define <vscale x 4 x i64> @vdiv_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -955,10 +955,10 @@ define <vscale x 4 x i64> @vdiv_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-V-NEXT:    vlse64.v v12, (a0), zero
@@ -1007,8 +1007,8 @@ define <vscale x 8 x i64> @vdiv_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1034,10 +1034,10 @@ define <vscale x 8 x i64> @vdiv_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-V-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdiv-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vdiv-vp.ll
index a4b7ca7f39768fb09b3e2c99f91f1e91bca7c010..061c2d1066a141811e924b7874ebbedc6c8b2fa7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdiv-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdiv-vp.ll
@@ -892,8 +892,8 @@ define <vscale x 1 x i64> @vdiv_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -918,8 +918,8 @@ define <vscale x 1 x i64> @vdiv_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -966,8 +966,8 @@ define <vscale x 2 x i64> @vdiv_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -992,8 +992,8 @@ define <vscale x 2 x i64> @vdiv_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1040,8 +1040,8 @@ define <vscale x 4 x i64> @vdiv_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1066,8 +1066,8 @@ define <vscale x 4 x i64> @vdiv_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1114,8 +1114,8 @@ define <vscale x 8 x i64> @vdiv_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1140,8 +1140,8 @@ define <vscale x 8 x i64> @vdiv_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdiv.ll b/llvm/test/CodeGen/RISCV/rvv/vdiv.ll
index fd32afcc7ce6e2937275e0351b2c5e68610f6af2..a2d3a39d57d6b3f407fd30d9df824145761c070a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdiv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdiv.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vdiv_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vdiv_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vdiv_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdiv_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vdiv_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vdiv_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vdiv_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdiv_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vdiv_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vdiv_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vdiv_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdiv_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vdiv_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vdiv_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vdiv_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdiv_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode.ll
index 4f2fb937ca73f40cd961d1c19c62c539c460c604..c8ec465e3cd3e8cd7c4afa44efb82f000de2b35a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode.ll
@@ -662,8 +662,8 @@ define <vscale x 1 x i64> @vdivu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -688,9 +688,9 @@ define <vscale x 1 x i64> @vdivu_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-V-NEXT:    vlse64.v v9, (a0), zero
@@ -759,8 +759,8 @@ define <vscale x 2 x i64> @vdivu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -785,9 +785,9 @@ define <vscale x 2 x i64> @vdivu_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-V-NEXT:    vlse64.v v10, (a0), zero
@@ -856,8 +856,8 @@ define <vscale x 4 x i64> @vdivu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -882,9 +882,9 @@ define <vscale x 4 x i64> @vdivu_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-V-NEXT:    vlse64.v v12, (a0), zero
@@ -953,8 +953,8 @@ define <vscale x 8 x i64> @vdivu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -979,9 +979,9 @@ define <vscale x 8 x i64> @vdivu_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-V-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdivu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vdivu-vp.ll
index 67c3f9dbf2869a974acbed50b686726400e6e625..25f6a2300f6739add10eb592d953cbbba091f9e4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdivu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdivu-vp.ll
@@ -891,8 +891,8 @@ define <vscale x 1 x i64> @vdivu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -917,8 +917,8 @@ define <vscale x 1 x i64> @vdivu_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -965,8 +965,8 @@ define <vscale x 2 x i64> @vdivu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -991,8 +991,8 @@ define <vscale x 2 x i64> @vdivu_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1039,8 +1039,8 @@ define <vscale x 4 x i64> @vdivu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1065,8 +1065,8 @@ define <vscale x 4 x i64> @vdivu_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1113,8 +1113,8 @@ define <vscale x 8 x i64> @vdivu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1139,8 +1139,8 @@ define <vscale x 8 x i64> @vdivu_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdivu.ll b/llvm/test/CodeGen/RISCV/rvv/vdivu.ll
index 99d63c218d8e60bab5c107a1ccd1f9e86774be13..85beeb1d8a37ba41e26a35ce2da3d4a36f252085 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdivu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdivu.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vdivu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vdivu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vdivu_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdivu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vdivu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vdivu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vdivu_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdivu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vdivu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vdivu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vdivu_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdivu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vdivu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vdivu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vdivu_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vdivu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll b/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll
index 122ac13cb257314bf1ffa08b4f633b63d6aee132..b19995a5aba9a48becf7fb3b7e638ce4af0a4f25 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll
@@ -32,8 +32,8 @@ define void @vec3_setcc_crash(ptr %in, ptr %out) {
 ; RV32-NEXT:    or a0, a0, a3
 ; RV32-NEXT:    neg a3, a6
 ; RV32-NEXT:    and a2, a3, a2
-; RV32-NEXT:    sb a2, 2(a1)
 ; RV32-NEXT:    sh a0, 0(a1)
+; RV32-NEXT:    sb a2, 2(a1)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vec3_setcc_crash:
@@ -59,8 +59,8 @@ define void @vec3_setcc_crash(ptr %in, ptr %out) {
 ; RV64-NEXT:    or a0, a0, a3
 ; RV64-NEXT:    negw a3, a6
 ; RV64-NEXT:    and a2, a3, a2
-; RV64-NEXT:    sb a2, 2(a1)
 ; RV64-NEXT:    sh a0, 0(a1)
+; RV64-NEXT:    sb a2, 2(a1)
 ; RV64-NEXT:    ret
   %a = load <3 x i8>, ptr %in
   %cmp = icmp sgt <3 x i8> %a, zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a360ae1998f77a2680f3cf3916fe51b59d39bf20
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
@@ -0,0 +1,1291 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb -riscv-enable-vl-optimizer=false -verify-machineinstrs | FileCheck %s --check-prefixes=NOVLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb -riscv-enable-vl-optimizer=false -verify-machineinstrs | FileCheck %s --check-prefixes=NOVLOPT
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb -riscv-enable-vl-optimizer -verify-machineinstrs | FileCheck %s --check-prefixes=VLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb -riscv-enable-vl-optimizer -verify-machineinstrs | FileCheck %s --check-prefixes=VLOPT
+
+; The purpose of this file is to check the behavior of specific instructions as it relates to the VL optimizer
+
+define <vscale x 4 x i32> @vadd_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vadd_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vadd_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vadd_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vadd_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vadd_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vadd_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vadd_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vadd_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsub_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsub_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsub.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsub.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsub_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsub.vv v8, v8, v10
+; VLOPT-NEXT:    vsub.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsub_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsub.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsub.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsub_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vsub.vx v10, v8, a0
+; VLOPT-NEXT:    vsub.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vrsub_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vrsub_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vrsub.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vrsub_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vrsub.vi v10, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vrsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vrsub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vrsub_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vrsub.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsub.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vrsub_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vrsub.vx v10, v8, a0
+; VLOPT-NEXT:    vsub.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vrsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vsub.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i64> @vwaddu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwaddu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwaddu.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwaddu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwaddu.vv v12, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwaddu.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwaddu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwaddu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwaddu.vx v12, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwaddu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwaddu.vx v12, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwaddu.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsubu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsubu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsubu.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsubu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsubu.vv v12, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsubu.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsubu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsubu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsubu.vx v12, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsubu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsubu.vx v12, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsubu.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwadd_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwadd_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwadd.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwadd_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwadd.vv v12, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwadd.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwadd_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwadd_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwadd.vx v12, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwadd_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwadd.vx v12, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwadd.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsub_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsub_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsub.vv v12, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsub_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsub.vv v12, v8, v10
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsub.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsub_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsub_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsub.vx v12, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsub_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsub.vx v12, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsub.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwaddu_wv(<vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwaddu_wv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwaddu.wv v8, v8, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwaddu_wv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwaddu.wv v8, v8, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwaddu.w.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwaddu_wx(<vscale x 4 x i64> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwaddu_wx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwaddu.wx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwaddu_wx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwaddu.wx v8, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwaddu.w.xv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsubu_wv(<vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsubu_wv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsubu.wv v8, v8, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsubu_wv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsubu.wv v8, v8, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsubu.w.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsubu_wx(<vscale x 4 x i64> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsubu_wx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsubu.wx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsubu_wx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsubu.wx v8, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsubu.w.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwadd_wv(<vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwadd_wv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwadd.wv v8, v8, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwadd_wv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwadd.wv v8, v8, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwadd.w.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwadd_wx(<vscale x 4 x i64> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwadd_wx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwadd.wx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwadd_wx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwadd.wx v8, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwadd.w.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsub_wv(<vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsub_wv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsub.wv v8, v8, v12
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsub_wv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsub.wv v8, v8, v12
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsub.w.nxv4i64.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @vwsub_wx(<vscale x 4 x i64> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsub_wx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vwsub.wx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsub_wx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vwsub.wx v8, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vwsub.w.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i64.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %1, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i32> @vsext_vf2(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsext_vf2:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsext.vf2 v12, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsext_vf2:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsext.vf2 v12, v8
+; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsext.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vsext_vf4(<vscale x 4 x i8> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsext_vf4:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsext.vf4 v12, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsext_vf4:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsext.vf4 v12, v8
+; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsext.nxv4i32.nxv4i8(<vscale x 4 x i32> poison, <vscale x 4 x i8> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i64> @vsext_vf8(<vscale x 4 x i8> %a, <vscale x 4 x i64> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vsext_vf8:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vsext.vf8 v16, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v16, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsext_vf8:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; VLOPT-NEXT:    vsext.vf8 v16, v8
+; VLOPT-NEXT:    vadd.vv v8, v16, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vsext.nxv4i32.nxv4i8(<vscale x 4 x i64> poison, <vscale x 4 x i8> %a, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %b, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i32> @vzext_vf2(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vzext_vf2:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vzext.vf2 v12, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vzext_vf2:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vzext.vf2 v12, v8
+; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vzext.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vzext_vf4(<vscale x 4 x i8> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vzext_vf4:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vzext.vf4 v12, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vzext_vf4:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vzext.vf4 v12, v8
+; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vzext.nxv4i32.nxv4i8(<vscale x 4 x i32> poison, <vscale x 4 x i8> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i64> @vzext_vf8(<vscale x 4 x i8> %a, <vscale x 4 x i64> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vzext_vf8:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vzext.vf8 v16, v8
+; NOVLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v16, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vzext_vf8:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; VLOPT-NEXT:    vzext.vf8 v16, v8
+; VLOPT-NEXT:    vadd.vv v8, v16, v12
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i64> @llvm.riscv.vzext.nxv4i32.nxv4i8(<vscale x 4 x i64> poison, <vscale x 4 x i8> %a, iXLen -1)
+  %2 = call <vscale x 4 x i64> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i64> poison, <vscale x 4 x i64> %1, <vscale x 4 x i64> %b, iXLen %vl)
+  ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i32> @vsll_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vsll_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsll.vi v10, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vsll.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vsll_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vsll.vi v10, v8, 5
+; VLOPT-NEXT:    vsll.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i16> @vnsrl_wi(<vscale x 4 x i32> %a, <vscale x 4 x i16> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vnsrl_wi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vnsrl.wi v11, v8, 5
+; NOVLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v11, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vnsrl_wi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; VLOPT-NEXT:    vnsrl.wi v11, v8, 5
+; VLOPT-NEXT:    vadd.vv v8, v11, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32(<vscale x 4 x i16> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen -1)
+  %2 = call <vscale x 4 x i16> @llvm.riscv.vadd.nxv4i16.nxv4i16(<vscale x 4 x i16> poison, <vscale x 4 x i16> %1, <vscale x 4 x i16> %b, iXLen %vl)
+  ret <vscale x 4 x i16> %2
+}
+
+define <vscale x 4 x i32> @vminu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vminu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vminu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vminu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vminu.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vminu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vminu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vminu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vminu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vminu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vminu.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vminu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmin_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmin_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmin.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmin_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmin.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmin.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmin_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmin_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmin.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmin_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmin.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmin.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmaxu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmaxu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmaxu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmaxu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmaxu.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmaxu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmaxu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmaxu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmaxu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmaxu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmaxu.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmaxu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmax_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmax_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmax.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmax_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmax.vv v8, v8, v10
+; VLOPT-NEXT:    vadd.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmax.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmax_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmax_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmax.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmax_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmax.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmax.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmul_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmul_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmul_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmul_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmul_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmul_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmul.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmulh_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmulh_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmulh.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmulh_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmulh.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmulh.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmulh_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmulh_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmulh.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmulh_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmulh.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmulh.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmulhu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmulhu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmulhu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmulhu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmulhu.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmulhu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmulhu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmulhu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmulhu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmulhu.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmulhsu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmulhsu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmulhsu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmulhsu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vmulhsu.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhsu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmulhsu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vmulhsu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmulhsu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmulhsu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmulhsu.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmulhsu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vdivu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vdivu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vdivu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vdivu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vdivu.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vdivu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vdivu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vdivu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vdivu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vdivu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vdivu.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vdivu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vdiv_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vdiv_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vdiv.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vdiv_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vdiv.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vdiv.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vdiv_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vdiv_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vdiv.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vdiv_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vdiv.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vdiv.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vremu_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vremu_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vremu.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vremu_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vremu.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vremu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vremu_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vremu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vremu.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vremu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vremu.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vremu.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vrem_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vrem_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vrem.vv v8, v8, v10
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v8, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vrem_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vrem.vv v8, v8, v10
+; VLOPT-NEXT:    vmul.vv v8, v8, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vrem.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vrem_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vrem_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vrem.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmul.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vrem_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vrem.vx v10, v8, a0
+; VLOPT-NEXT:    vmul.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vrem.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vmul.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwmacc_vx(<vscale x 4 x i16> %a, i16 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwmacc_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwmacc.vx v10, a0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwmacc_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; VLOPT-NEXT:    vwmacc.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmacc.nxv4i32.i16(<vscale x 4 x i32> poison, i16 %b, <vscale x 4 x i16> %a, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwmaccu_vx(<vscale x 4 x i16> %a, i16 %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwmaccu_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwmaccu.vx v10, a0, v8
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwmaccu_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; VLOPT-NEXT:    vwmaccu.vx v10, a0, v8
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwmaccu.nxv4i32.i16(<vscale x 4 x i32> poison, i16 %b, <vscale x 4 x i16> %a, iXLen -1, iXLen 0)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmv_v_i(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
+; NOVLOPT-LABEL: vmv_v_i:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmv.v.i v10, 5
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmv_v_i:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmv.v.i v10, 5
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmv.v.x.nxv4i32(<vscale x 4 x i32> poison, i32 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vmv_v_x(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
+; NOVLOPT-LABEL: vmv_v_x:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmv.v.x v10, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmv_v_x:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vmv.v.x v10, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vmv.v.x.nxv4i32(<vscale x 4 x i32> poison, i32 %x, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsll_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwsll.vi v12, v8, 1
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsll_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; VLOPT-NEXT:    vwsll.vi v12, v8, 1
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a,iXLen 1, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+; Test getOperandInfo
+
+define <vscale x 1 x i8> @vmerge_vim(<vscale x 1 x i8> %a, i8 %b, <vscale x 1 x i1> %m, iXLen %vl) {
+; NOVLOPT-LABEL: vmerge_vim:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e8, mf8, tu, ma
+; NOVLOPT-NEXT:    vmv.v.x v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmerge.vim v8, v8, 2, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmerge_vim:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
+; VLOPT-NEXT:    vmv.v.x v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmerge.vim v8, v8, 2, v0
+; VLOPT-NEXT:    ret
+  %2 = call <vscale x 1 x i8> @llvm.riscv.vmv.v.x.nxv1i8(<vscale x 1 x i8> %a, i8 %b, iXLen -1)
+  %3 = call <vscale x 1 x i8> @llvm.riscv.vmerge.nxv1i8.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> %2, i8 2, <vscale x 1 x i1> %m, iXLen %vl)
+  ret <vscale x 1 x i8> %3
+}
+
+define <vscale x 1 x i8> @vmerge_vxm(<vscale x 1 x i8> %a, i8 %b, <vscale x 1 x i1> %m, iXLen %vl) {
+; NOVLOPT-LABEL: vmerge_vxm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e8, mf8, tu, ma
+; NOVLOPT-NEXT:    vmv.v.x v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmerge.vxm v8, v8, a0, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmerge_vxm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
+; VLOPT-NEXT:    vmv.v.x v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmerge.vxm v8, v8, a0, v0
+; VLOPT-NEXT:    ret
+  %2 = call <vscale x 1 x i8> @llvm.riscv.vmv.v.x.nxv1i8(<vscale x 1 x i8> %a, i8 %b, iXLen -1)
+  %3 = call <vscale x 1 x i8> @llvm.riscv.vmerge.nxv1i8.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> %2, i8 %b, <vscale x 1 x i1> %m, iXLen %vl)
+  ret <vscale x 1 x i8> %3
+}
+
+define <vscale x 1 x i8> @vmerge_vvm(<vscale x 1 x i8> %a, i8 %b, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl) {
+; NOVLOPT-LABEL: vmerge_vvm:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e8, mf8, tu, ma
+; NOVLOPT-NEXT:    vmv.v.x v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vmerge.vvm v8, v8, v9, v0
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vmerge_vvm:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
+; VLOPT-NEXT:    vmv.v.x v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; VLOPT-NEXT:    vmerge.vvm v8, v8, v9, v0
+; VLOPT-NEXT:    ret
+  %2 = call <vscale x 1 x i8> @llvm.riscv.vmv.v.x.nxv1i8(<vscale x 1 x i8> %a, i8 %b, iXLen -1)
+  %3 = call <vscale x 1 x i8> @llvm.riscv.vmerge.nxv1i8.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i8> %2, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl)
+  ret <vscale x 1 x i8> %3
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-no-prop.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-no-prop.ll
index e1f641afd2cfe0bdcb1f168a8cc18974d526c860..85072542b6aba9538c6d5aad317da7e72ab7af52 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-no-prop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-no-prop.ll
@@ -1,6 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs \
+; RUN:   | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs \
+; RUN:   | FileCheck %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -riscv-enable-vl-optimizer \
+; RUN:   -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -riscv-enable-vl-optimizer \
+; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, iXLen)
 declare <vscale x 4 x i32> @llvm.riscv.vrgather.vv.nxv4i32.iXLen(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.ll
index 2b3c5417b15b554bc178c708700f370bd3bd2187..1a01a9bf77cff52a9061e32d17d1ff563be5e721 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.ll
@@ -1,6 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvl512b -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvl512b -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvl512b -verify-machineinstrs \
+; RUN:   | FileCheck %s -check-prefixes=CHECK,NOVLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvl512b -verify-machineinstrs \
+; RUN:   | FileCheck %s -check-prefixes=CHECK,NOVLOPT
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvl512b -riscv-enable-vl-optimizer \
+; RUN:   -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,VLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvl512b -riscv-enable-vl-optimizer \
+; RUN:   -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,VLOPT
 
 define <2 x i32> @vdot_lane_s32(<2 x i32> noundef %var_1, <8 x i8> noundef %var_3, <8 x i8> noundef %var_5, <8 x i16> %x) {
 ; CHECK-LABEL: vdot_lane_s32:
@@ -34,13 +40,20 @@ declare <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.nxv2i16(
   iXLen);
 
 define <vscale x 2 x i16> @intrinsic_vnsrl_wv_nxv2i16_nxv2i32_nxv2i16(<vscale x 2 x i16> %a, <vscale x 2 x i16> %b, iXLen %2, <vscale x 2 x i32> %3, <vscale x 2 x i32> %4, <vscale x 2 x i16> %z) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wv_nxv2i16_nxv2i32_nxv2i16:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vwadd.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wv v8, v10, v12
-; CHECK-NEXT:    ret
+; NOVLOPT-LABEL: intrinsic_vnsrl_wv_nxv2i16_nxv2i32_nxv2i16:
+; NOVLOPT:       # %bb.0: # %entry
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; NOVLOPT-NEXT:    vwadd.vv v10, v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; NOVLOPT-NEXT:    vnsrl.wv v8, v10, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: intrinsic_vnsrl_wv_nxv2i16_nxv2i32_nxv2i16:
+; VLOPT:       # %bb.0: # %entry
+; VLOPT-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; VLOPT-NEXT:    vwadd.vv v10, v8, v9
+; VLOPT-NEXT:    vnsrl.wv v8, v10, v12
+; VLOPT-NEXT:    ret
 entry:
   %c = sext <vscale x 2 x i16> %a to <vscale x 2 x i32>
   %d = sext <vscale x 2 x i16> %b to <vscale x 2 x i32>
@@ -61,14 +74,22 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.nxv2i16(
   iXLen, iXLen);
 
 define <vscale x 2 x i16> @vnclip(<vscale x 2 x i16> %a, <vscale x 2 x i16> %b, iXLen %2, <vscale x 2 x i32> %3, <vscale x 2 x i32> %4, <vscale x 2 x i16> %z) nounwind {
-; CHECK-LABEL: vnclip:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vwadd.vv v10, v8, v9
-; CHECK-NEXT:    csrwi vxrm, 0
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vnclip.wv v8, v10, v12
-; CHECK-NEXT:    ret
+; NOVLOPT-LABEL: vnclip:
+; NOVLOPT:       # %bb.0: # %entry
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; NOVLOPT-NEXT:    vwadd.vv v10, v8, v9
+; NOVLOPT-NEXT:    csrwi vxrm, 0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; NOVLOPT-NEXT:    vnclip.wv v8, v10, v12
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vnclip:
+; VLOPT:       # %bb.0: # %entry
+; VLOPT-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; VLOPT-NEXT:    vwadd.vv v10, v8, v9
+; VLOPT-NEXT:    csrwi vxrm, 0
+; VLOPT-NEXT:    vnclip.wv v8, v10, v12
+; VLOPT-NEXT:    ret
 entry:
   %c = sext <vscale x 2 x i16> %a to <vscale x 2 x i32>
   %d = sext <vscale x 2 x i16> %b to <vscale x 2 x i32>
@@ -81,3 +102,4 @@ entry:
 
   ret <vscale x 2 x i16> %x
 }
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
index b03ba0760595032396ac426a736e84a24928cd02..0b3e67ec8955664a5e5fa244ca730e5157b45747 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
@@ -1,6 +1,12 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,NOVLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,NOVLOPT
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -riscv-enable-vl-optimizer \
+; RUN:   -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,VLOPT
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -riscv-enable-vl-optimizer \
+; RUN:   -verify-machineinstrs | FileCheck %s -check-prefixes=CHECK,VLOPT
 
 declare <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, iXLen)
 
@@ -104,3 +110,7 @@ define <vscale x 4 x i32> @different_imm_vl_with_tu(<vscale x 4 x i32> %passthru
   %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a,iXLen 4)
   ret <vscale x 4 x i32> %w
 }
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NOVLOPT: {{.*}}
+; VLOPT: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir
new file mode 100644
index 0000000000000000000000000000000000000000..59a472c73a4624bd997b4fe0d314e14bfe03ca74
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir
@@ -0,0 +1,18 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc %s -o - -mtriple=riscv64 -mattr=+v -run-pass=riscv-vl-optimizer -verify-machineinstrs | FileCheck %s
+
+---
+name: vnsrl_wv_user
+body: |
+  bb.0:
+    liveins: $x1
+    ; CHECK-LABEL: name: vnsrl_wv_user
+    ; CHECK: liveins: $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %vl:gprnox0 = COPY $x1
+    ; CHECK-NEXT: %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    ; CHECK-NEXT: %y:vr = PseudoVNSRL_WV_MF4 $noreg, %x, $noreg, %vl, 4 /* e16 */, 0 /* tu, mu */
+    %vl:gprnox0 = COPY $x1
+    %x:vr = PseudoVADD_VV_MF4 $noreg, $noreg, $noreg, -1, 4 /* e16 */, 0 /* tu, mu */
+    %y:vr = PseudoVNSRL_WV_MF4 $noreg, %x, $noreg, %vl, 4 /* e16 */, 0 /* tu, mu */
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmacc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmacc-vp.ll
index e232ac255c56f053b039419d17e59abc4513de7f..9fe9af478f0b2ca9934eb0b180126a8787a91da8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmacc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmacc-vp.ll
@@ -1651,8 +1651,8 @@ define <vscale x 1 x i64> @vmacc_vx_nxv1i64(<vscale x 1 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1681,8 +1681,8 @@ define <vscale x 1 x i64> @vmacc_vx_nxv1i64_unmasked(<vscale x 1 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1724,8 +1724,8 @@ define <vscale x 1 x i64> @vmacc_vx_nxv1i64_ta(<vscale x 1 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1785,8 +1785,8 @@ define <vscale x 2 x i64> @vmacc_vx_nxv2i64(<vscale x 2 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1815,8 +1815,8 @@ define <vscale x 2 x i64> @vmacc_vx_nxv2i64_unmasked(<vscale x 2 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1858,8 +1858,8 @@ define <vscale x 2 x i64> @vmacc_vx_nxv2i64_ta(<vscale x 2 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1919,8 +1919,8 @@ define <vscale x 4 x i64> @vmacc_vx_nxv4i64(<vscale x 4 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1949,8 +1949,8 @@ define <vscale x 4 x i64> @vmacc_vx_nxv4i64_unmasked(<vscale x 4 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1992,8 +1992,8 @@ define <vscale x 4 x i64> @vmacc_vx_nxv4i64_ta(<vscale x 4 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2055,8 +2055,8 @@ define <vscale x 8 x i64> @vmacc_vx_nxv8i64(<vscale x 8 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -2085,8 +2085,8 @@ define <vscale x 8 x i64> @vmacc_vx_nxv8i64_unmasked(<vscale x 8 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -2129,8 +2129,8 @@ define <vscale x 8 x i64> @vmacc_vx_nxv8i64_ta(<vscale x 8 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vmacc.ll
index 622fcb5e9f71567849b192e30d8233d063520911..b8b4baf53b677dc8a075ecb380a765d53588afe6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmacc.ll
@@ -1566,8 +1566,8 @@ define <vscale x 1 x i64>  @intrinsic_vmacc_vx_nxv1i64_i64_nxv1i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vmacc_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1602,8 +1602,8 @@ define <vscale x 1 x i64> @intrinsic_vmacc_mask_vx_nxv1i64_i64_nxv1i64(<vscale x
 ; RV32-LABEL: intrinsic_vmacc_mask_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1639,8 +1639,8 @@ define <vscale x 2 x i64>  @intrinsic_vmacc_vx_nxv2i64_i64_nxv2i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vmacc_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1675,8 +1675,8 @@ define <vscale x 2 x i64> @intrinsic_vmacc_mask_vx_nxv2i64_i64_nxv2i64(<vscale x
 ; RV32-LABEL: intrinsic_vmacc_mask_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1712,8 +1712,8 @@ define <vscale x 4 x i64>  @intrinsic_vmacc_vx_nxv4i64_i64_nxv4i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vmacc_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1748,8 +1748,8 @@ define <vscale x 4 x i64> @intrinsic_vmacc_mask_vx_nxv4i64_i64_nxv4i64(<vscale x
 ; RV32-LABEL: intrinsic_vmacc_mask_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmadc.carry.in.ll b/llvm/test/CodeGen/RISCV/rvv/vmadc.carry.in.ll
index eb09de906fac532b9d3ba8c0c1046f5c16b4f85d..31c12db79a946f7c51a60eccc96034e0136f141d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmadc.carry.in.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmadc.carry.in.ll
@@ -934,8 +934,8 @@ define <vscale x 1 x i1> @intrinsic_vmadc.carry.in_vxm_nxv1i1_nxv1i64_i64(<vscal
 ; RV32-LABEL: intrinsic_vmadc.carry.in_vxm_nxv1i1_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -970,8 +970,8 @@ define <vscale x 2 x i1> @intrinsic_vmadc.carry.in_vxm_nxv2i1_nxv2i64_i64(<vscal
 ; RV32-LABEL: intrinsic_vmadc.carry.in_vxm_nxv2i1_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1006,8 +1006,8 @@ define <vscale x 4 x i1> @intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i64_i64(<vscal
 ; RV32-LABEL: intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1042,8 +1042,8 @@ define <vscale x 8 x i1> @intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i64_i64(<vscal
 ; RV32-LABEL: intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmadc.ll b/llvm/test/CodeGen/RISCV/rvv/vmadc.ll
index 5ec84f89f67c7c0e65305000489b991ca2be9eec..4777903558e4cb56c99cfa0d3feee9b476e221f7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmadc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmadc.ll
@@ -813,8 +813,8 @@ define <vscale x 1 x i1> @intrinsic_vmadc_vx_nxv1i1_nxv1i64_i64(<vscale x 1 x i6
 ; RV32-LABEL: intrinsic_vmadc_vx_nxv1i1_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -845,8 +845,8 @@ define <vscale x 2 x i1> @intrinsic_vmadc_vx_nxv2i1_nxv2i64_i64(<vscale x 2 x i6
 ; RV32-LABEL: intrinsic_vmadc_vx_nxv2i1_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -877,8 +877,8 @@ define <vscale x 4 x i1> @intrinsic_vmadc_vx_nxv4i1_nxv4i64_i64(<vscale x 4 x i6
 ; RV32-LABEL: intrinsic_vmadc_vx_nxv4i1_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -909,8 +909,8 @@ define <vscale x 8 x i1> @intrinsic_vmadc_vx_nxv8i1_nxv8i64_i64(<vscale x 8 x i6
 ; RV32-LABEL: intrinsic_vmadc_vx_nxv8i1_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vmadd-sdnode.ll
index e1988c058fac34e0fc8faafe79f4d6105e8dcb96..87877efeb771136d481d458b3d9a6f7f42854d56 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmadd-sdnode.ll
@@ -458,8 +458,8 @@ define <vscale x 1 x i64> @vmadd_vx_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -495,8 +495,8 @@ define <vscale x 2 x i64> @vmadd_vx_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -532,8 +532,8 @@ define <vscale x 4 x i64> @vmadd_vx_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -570,8 +570,8 @@ define <vscale x 8 x i64> @vmadd_vx_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll
index 5401bf7db49e38f2e6db9ab12d0eb2f690d65c53..ef74b26ba2646df82e58a3b8de021c0854886cb2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmadd-vp.ll
@@ -1635,8 +1635,8 @@ define <vscale x 1 x i64> @vmadd_vx_nxv1i64(<vscale x 1 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1665,8 +1665,8 @@ define <vscale x 1 x i64> @vmadd_vx_nxv1i64_unmasked(<vscale x 1 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1708,8 +1708,8 @@ define <vscale x 1 x i64> @vmadd_vx_nxv1i64_ta(<vscale x 1 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1770,8 +1770,8 @@ define <vscale x 2 x i64> @vmadd_vx_nxv2i64(<vscale x 2 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1800,8 +1800,8 @@ define <vscale x 2 x i64> @vmadd_vx_nxv2i64_unmasked(<vscale x 2 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1843,8 +1843,8 @@ define <vscale x 2 x i64> @vmadd_vx_nxv2i64_ta(<vscale x 2 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1905,8 +1905,8 @@ define <vscale x 4 x i64> @vmadd_vx_nxv4i64(<vscale x 4 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1935,8 +1935,8 @@ define <vscale x 4 x i64> @vmadd_vx_nxv4i64_unmasked(<vscale x 4 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1978,8 +1978,8 @@ define <vscale x 4 x i64> @vmadd_vx_nxv4i64_ta(<vscale x 4 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2042,8 +2042,8 @@ define <vscale x 8 x i64> @vmadd_vx_nxv8i64(<vscale x 8 x i64> %a, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -2072,8 +2072,8 @@ define <vscale x 8 x i64> @vmadd_vx_nxv8i64_unmasked(<vscale x 8 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -2116,8 +2116,8 @@ define <vscale x 8 x i64> @vmadd_vx_nxv8i64_ta(<vscale x 8 x i64> %a, i64 %b, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmadd.ll b/llvm/test/CodeGen/RISCV/rvv/vmadd.ll
index 184ce741aba9a6ade3ec00f2072b5b7a9cc008ac..829d082ab7a4fd43ffb40af2bbe462fbb0aadd47 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmadd.ll
@@ -1566,8 +1566,8 @@ define <vscale x 1 x i64>  @intrinsic_vmadd_vx_nxv1i64_i64_nxv1i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vmadd_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1602,8 +1602,8 @@ define <vscale x 1 x i64> @intrinsic_vmadd_mask_vx_nxv1i64_i64_nxv1i64(<vscale x
 ; RV32-LABEL: intrinsic_vmadd_mask_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1639,8 +1639,8 @@ define <vscale x 2 x i64>  @intrinsic_vmadd_vx_nxv2i64_i64_nxv2i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vmadd_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1675,8 +1675,8 @@ define <vscale x 2 x i64> @intrinsic_vmadd_mask_vx_nxv2i64_i64_nxv2i64(<vscale x
 ; RV32-LABEL: intrinsic_vmadd_mask_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1712,8 +1712,8 @@ define <vscale x 4 x i64>  @intrinsic_vmadd_vx_nxv4i64_i64_nxv4i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vmadd_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1748,8 +1748,8 @@ define <vscale x 4 x i64> @intrinsic_vmadd_mask_vx_nxv4i64_i64_nxv4i64(<vscale x
 ; RV32-LABEL: intrinsic_vmadd_mask_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmax-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vmax-sdnode.ll
index 52720755dd5b5b4f89cf46dda56b52f8281beb13..edf4211101e88213040cc7edf4d87beda7ce0d75 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmax-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmax-sdnode.ll
@@ -666,8 +666,8 @@ define <vscale x 1 x i64> @vmax_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -715,8 +715,8 @@ define <vscale x 2 x i64> @vmax_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -764,8 +764,8 @@ define <vscale x 4 x i64> @vmax_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -813,8 +813,8 @@ define <vscale x 8 x i64> @vmax_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
index f65e708f5303ccece09a8f1d7085bb443c28e02e..85cd97019fdfb1d1979819c8525ca79bfa4fe2ef 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
@@ -1123,8 +1123,8 @@ define <vscale x 1 x i64> @vmax_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1149,8 +1149,8 @@ define <vscale x 1 x i64> @vmax_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1197,8 +1197,8 @@ define <vscale x 2 x i64> @vmax_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1223,8 +1223,8 @@ define <vscale x 2 x i64> @vmax_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1271,8 +1271,8 @@ define <vscale x 4 x i64> @vmax_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1297,8 +1297,8 @@ define <vscale x 4 x i64> @vmax_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1345,8 +1345,8 @@ define <vscale x 8 x i64> @vmax_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1371,8 +1371,8 @@ define <vscale x 8 x i64> @vmax_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmax.ll b/llvm/test/CodeGen/RISCV/rvv/vmax.ll
index 86f17dc20f23e79f1b3ec2db5271dd50f9ec4da3..5acca2b80919e7dec17ebba75ba8a1056d588a3f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmax.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmax.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vmax_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vmax_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vmax_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmax_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vmax_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vmax_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vmax_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmax_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vmax_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vmax_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vmax_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmax_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vmax_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vmax_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vmax_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmax_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmaxu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vmaxu-sdnode.ll
index 8eb70fbc91fa54522938700d62b2a7491e9621cd..e7fd3699f0fed960d87484fcbec1564d40a8092a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmaxu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmaxu-sdnode.ll
@@ -666,8 +666,8 @@ define <vscale x 1 x i64> @vmax_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -715,8 +715,8 @@ define <vscale x 2 x i64> @vmax_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -764,8 +764,8 @@ define <vscale x 4 x i64> @vmax_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -813,8 +813,8 @@ define <vscale x 8 x i64> @vmax_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
index df1ad58e5ecbde76672f8e10ec41df559e5138d5..40e11e6211093551d5dbc30e8232cfca8590d3b1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
@@ -1122,8 +1122,8 @@ define <vscale x 1 x i64> @vmaxu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1148,8 +1148,8 @@ define <vscale x 1 x i64> @vmaxu_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1196,8 +1196,8 @@ define <vscale x 2 x i64> @vmaxu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1222,8 +1222,8 @@ define <vscale x 2 x i64> @vmaxu_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1270,8 +1270,8 @@ define <vscale x 4 x i64> @vmaxu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1296,8 +1296,8 @@ define <vscale x 4 x i64> @vmaxu_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1344,8 +1344,8 @@ define <vscale x 8 x i64> @vmaxu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1370,8 +1370,8 @@ define <vscale x 8 x i64> @vmaxu_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmaxu.ll b/llvm/test/CodeGen/RISCV/rvv/vmaxu.ll
index e2a5b95b2b4ad1420707b8cd70088c9896f2420c..f1a7cedbb4417d6fccaf0800d7f199ffbeab7334 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmaxu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmaxu.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vmaxu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vmaxu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vmaxu_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmaxu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vmaxu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vmaxu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vmaxu_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmaxu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vmaxu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vmaxu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vmaxu_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmaxu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vmaxu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vmaxu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vmaxu_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmaxu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmin-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vmin-sdnode.ll
index 7f526a21deac1c99e552b1f0b9bfbe941b436c84..fc7ed2979a06dc8699bf3448b51c160491f580f8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmin-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmin-sdnode.ll
@@ -666,8 +666,8 @@ define <vscale x 1 x i64> @vmin_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -715,8 +715,8 @@ define <vscale x 2 x i64> @vmin_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -764,8 +764,8 @@ define <vscale x 4 x i64> @vmin_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -813,8 +813,8 @@ define <vscale x 8 x i64> @vmin_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
index 0bf0638633aa4572277699c56e490d1b892e4252..2e4ace7212777c3a9111fb7158910c572368f027 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
@@ -1123,8 +1123,8 @@ define <vscale x 1 x i64> @vmin_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1149,8 +1149,8 @@ define <vscale x 1 x i64> @vmin_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1197,8 +1197,8 @@ define <vscale x 2 x i64> @vmin_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1223,8 +1223,8 @@ define <vscale x 2 x i64> @vmin_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1271,8 +1271,8 @@ define <vscale x 4 x i64> @vmin_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1297,8 +1297,8 @@ define <vscale x 4 x i64> @vmin_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1345,8 +1345,8 @@ define <vscale x 8 x i64> @vmin_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1371,8 +1371,8 @@ define <vscale x 8 x i64> @vmin_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmin.ll b/llvm/test/CodeGen/RISCV/rvv/vmin.ll
index 311c9f9f1526bd921e179c42f6d6e6a71d94a0e1..002c0081c8e32a7e0ad1e61921d879c478c0afa3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmin.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmin.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vmin_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vmin_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vmin_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmin_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vmin_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vmin_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vmin_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmin_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vmin_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vmin_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vmin_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmin_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vmin_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vmin_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vmin_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmin_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vminu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vminu-sdnode.ll
index d22a7dcccf0adc650833cc4050823b9917aaeca1..032faed878a03de13c81100ba257cac6248b36bb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vminu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vminu-sdnode.ll
@@ -666,8 +666,8 @@ define <vscale x 1 x i64> @vmin_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -715,8 +715,8 @@ define <vscale x 2 x i64> @vmin_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -764,8 +764,8 @@ define <vscale x 4 x i64> @vmin_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -813,8 +813,8 @@ define <vscale x 8 x i64> @vmin_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
index 2acebdf2e646d4515dfb8db43660b76c562a58e5..17a10868ff05b51359725d0a51ef2c085d4258db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
@@ -1122,8 +1122,8 @@ define <vscale x 1 x i64> @vminu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1148,8 +1148,8 @@ define <vscale x 1 x i64> @vminu_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1196,8 +1196,8 @@ define <vscale x 2 x i64> @vminu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1222,8 +1222,8 @@ define <vscale x 2 x i64> @vminu_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1270,8 +1270,8 @@ define <vscale x 4 x i64> @vminu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1296,8 +1296,8 @@ define <vscale x 4 x i64> @vminu_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1344,8 +1344,8 @@ define <vscale x 8 x i64> @vminu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1370,8 +1370,8 @@ define <vscale x 8 x i64> @vminu_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vminu.ll b/llvm/test/CodeGen/RISCV/rvv/vminu.ll
index b513331bc0b8fc17455d20c5fc07fd1176f7d008..3d2f6d00e68132841196dee5eb6c53d0387dbafb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vminu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vminu.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vminu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vminu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vminu_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vminu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vminu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vminu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vminu_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vminu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vminu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vminu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vminu_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vminu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vminu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vminu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vminu_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vminu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsbc.borrow.in.ll b/llvm/test/CodeGen/RISCV/rvv/vmsbc.borrow.in.ll
index b3af0d3a6f30540e34a0418a8ceba79586917bee..9ce7d68ba40125ae06c2be886b7bffd47e205882 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsbc.borrow.in.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsbc.borrow.in.ll
@@ -934,8 +934,8 @@ define <vscale x 1 x i1> @intrinsic_vmsbc.borrow.in_vxm_nxv1i1_nxv1i64_i64(<vsca
 ; RV32-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv1i1_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -970,8 +970,8 @@ define <vscale x 2 x i1> @intrinsic_vmsbc.borrow.in_vxm_nxv2i1_nxv2i64_i64(<vsca
 ; RV32-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv2i1_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1006,8 +1006,8 @@ define <vscale x 4 x i1> @intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i64_i64(<vsca
 ; RV32-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1042,8 +1042,8 @@ define <vscale x 8 x i1> @intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i64_i64(<vsca
 ; RV32-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsbc.ll b/llvm/test/CodeGen/RISCV/rvv/vmsbc.ll
index a4785a05de78217e4b843915c828d4d38e96615d..8c870a9332646e56d84fa9e27cccdea79a315f88 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsbc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsbc.ll
@@ -813,8 +813,8 @@ define <vscale x 1 x i1> @intrinsic_vmsbc_vx_nxv1i1_nxv1i64_i64(<vscale x 1 x i6
 ; RV32-LABEL: intrinsic_vmsbc_vx_nxv1i1_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -845,8 +845,8 @@ define <vscale x 2 x i1> @intrinsic_vmsbc_vx_nxv2i1_nxv2i64_i64(<vscale x 2 x i6
 ; RV32-LABEL: intrinsic_vmsbc_vx_nxv2i1_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -877,8 +877,8 @@ define <vscale x 4 x i1> @intrinsic_vmsbc_vx_nxv4i1_nxv4i64_i64(<vscale x 4 x i6
 ; RV32-LABEL: intrinsic_vmsbc_vx_nxv4i1_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -909,8 +909,8 @@ define <vscale x 8 x i1> @intrinsic_vmsbc_vx_nxv8i1_nxv8i64_i64(<vscale x 8 x i6
 ; RV32-LABEL: intrinsic_vmsbc_vx_nxv8i1_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmseq.ll b/llvm/test/CodeGen/RISCV/rvv/vmseq.ll
index 9f181f7a30ebed798a44cc4b3ab8361a37e84edb..c5769e0d1e51929932fa4858b22e47947349db47 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmseq.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmseq.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmseq_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmseq_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmseq_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmseq_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmseq_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmseq_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmseq_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmseq_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmseq_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmseq_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmseq_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmseq_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsge.ll b/llvm/test/CodeGen/RISCV/rvv/vmsge.ll
index 75fc407abbc2f330729f369b5f73d0a7a4a42b36..1ec304609699a6936dd32bc0b1d501f2e2ff72c3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsge.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsge.ll
@@ -1669,8 +1669,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsge_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1704,8 +1704,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsge_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1744,8 +1744,8 @@ define <vscale x 2 x i1> @intrinsic_vmsge_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsge_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1779,8 +1779,8 @@ define <vscale x 2 x i1> @intrinsic_vmsge_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsge_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1819,8 +1819,8 @@ define <vscale x 4 x i1> @intrinsic_vmsge_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsge_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1854,8 +1854,8 @@ define <vscale x 4 x i1> @intrinsic_vmsge_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsge_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2790,8 +2790,8 @@ define <vscale x 1 x i1> @intrinsic_vmsge_maskedoff_mask_vx_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmsge_maskedoff_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2820,8 +2820,8 @@ define <vscale x 2 x i1> @intrinsic_vmsge_maskedoff_mask_vx_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmsge_maskedoff_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2852,8 +2852,8 @@ define <vscale x 4 x i1> @intrinsic_vmsge_maskedoff_mask_vx_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmsge_maskedoff_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll
index 5568c1e9b1cfb93ceb55aa87b42de7c10a028598..05cc7a9d8f7b42bd0c18c946217674face843fb5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsgeu.ll
@@ -1669,8 +1669,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgeu_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1704,8 +1704,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsgeu_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1744,8 +1744,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgeu_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1779,8 +1779,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsgeu_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1819,8 +1819,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgeu_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1854,8 +1854,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsgeu_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2802,8 +2802,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgeu_maskedoff_mask_vx_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmsgeu_maskedoff_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -2832,8 +2832,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgeu_maskedoff_mask_vx_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmsgeu_maskedoff_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2864,8 +2864,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgeu_maskedoff_mask_vx_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmsgeu_maskedoff_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll b/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll
index f1fa6484d976b412d95893f62836acf6df18f4c2..98a03a2c562806d0805218b28d6960e2f1fc0339 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsgt.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgt_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgt_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgt_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsgt_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgt_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgt_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgt_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsgt_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgt_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgt_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgt_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsgt_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll
index de7a0ad87be27c6fb24f5ac01e0ffe12cd186bd4..69b22573c289e5095804c6731d03c820dbfd7fa0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsgtu.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgtu_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgtu_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmsgtu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsgtu_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgtu_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgtu_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmsgtu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsgtu_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgtu_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsgtu_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmsgtu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsle.ll b/llvm/test/CodeGen/RISCV/rvv/vmsle.ll
index f54aef3ed4052c90aafd366506709e32485dc8be..c8794e1b63900f662d90f6e482b36d70669c7750 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsle.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsle.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmsle_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsle_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmsle_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsle_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmsle_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsle_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmsle_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsle_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmsle_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsle_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmsle_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsle_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll
index 540577247484e3a9436929229ed48aecc982055d..86dc48d51cc2bc190c98573607d3f39b24c8c1cd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsleu.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmsleu_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsleu_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmsleu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsleu_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmsleu_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsleu_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmsleu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsleu_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmsleu_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsleu_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmsleu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsleu_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmslt.ll b/llvm/test/CodeGen/RISCV/rvv/vmslt.ll
index 554d25172d4fde4a3b473daee1f563f4f71fe56f..8d57f2adc53868680a9faa6e12ec15e1709456a1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmslt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmslt.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmslt_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmslt_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmslt_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmslt_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmslt_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmslt_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmslt_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmslt_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmslt_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmslt_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmslt_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmslt_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll b/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll
index 7a8efa6c80fb6b931617b860b32fbdb9e9970008..627b594308713722a521fbed54b7d14b3a024454 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsltu.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmsltu_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsltu_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsltu_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmsltu_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsltu_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsltu_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmsltu_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsltu_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsltu_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsne.ll b/llvm/test/CodeGen/RISCV/rvv/vmsne.ll
index bd6bd8a804bcc26a43740083de1a5bb1a0cf8f30..47d1048f46cab477d2076dc26031dd96e8733e06 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsne.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsne.ll
@@ -1654,8 +1654,8 @@ define <vscale x 1 x i1> @intrinsic_vmsne_vx_nxv1i64_i64(<vscale x 1 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsne_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1688,8 +1688,8 @@ define <vscale x 1 x i1> @intrinsic_vmsne_mask_vx_nxv1i64_i64(<vscale x 1 x i1>
 ; RV32-LABEL: intrinsic_vmsne_mask_vx_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v11, (a0), zero
@@ -1728,8 +1728,8 @@ define <vscale x 2 x i1> @intrinsic_vmsne_vx_nxv2i64_i64(<vscale x 2 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsne_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1762,8 +1762,8 @@ define <vscale x 2 x i1> @intrinsic_vmsne_mask_vx_nxv2i64_i64(<vscale x 2 x i1>
 ; RV32-LABEL: intrinsic_vmsne_mask_vx_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1802,8 +1802,8 @@ define <vscale x 4 x i1> @intrinsic_vmsne_vx_nxv4i64_i64(<vscale x 4 x i64> %0,
 ; RV32-LABEL: intrinsic_vmsne_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1836,8 +1836,8 @@ define <vscale x 4 x i1> @intrinsic_vmsne_mask_vx_nxv4i64_i64(<vscale x 4 x i1>
 ; RV32-LABEL: intrinsic_vmsne_mask_vx_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vmul-sdnode.ll
index 0b8620c90c62e077564a5f915cda1b4f6c2f28f4..c05ab8fe7db2fc3dfcc33ebb428bf63f9e75a866 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmul-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmul-sdnode.ll
@@ -614,8 +614,8 @@ define <vscale x 1 x i64> @vmul_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -680,8 +680,8 @@ define <vscale x 2 x i64> @vmul_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -746,8 +746,8 @@ define <vscale x 4 x i64> @vmul_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -812,8 +812,8 @@ define <vscale x 8 x i64> @vmul_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -867,13 +867,13 @@ define <vscale x 8 x i64> @vmul_xx_nxv8i64(i64 %a, i64 %b) nounwind {
 ; RV32NOM-LABEL: vmul_xx_nxv8i64:
 ; RV32NOM:       # %bb.0:
 ; RV32NOM-NEXT:    addi sp, sp, -16
-; RV32NOM-NEXT:    sw a1, 12(sp)
 ; RV32NOM-NEXT:    sw a0, 8(sp)
+; RV32NOM-NEXT:    sw a1, 12(sp)
 ; RV32NOM-NEXT:    addi a0, sp, 8
 ; RV32NOM-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32NOM-NEXT:    vlse64.v v8, (a0), zero
-; RV32NOM-NEXT:    sw a3, 4(sp)
 ; RV32NOM-NEXT:    sw a2, 0(sp)
+; RV32NOM-NEXT:    sw a3, 4(sp)
 ; RV32NOM-NEXT:    mv a0, sp
 ; RV32NOM-NEXT:    vlse64.v v16, (a0), zero
 ; RV32NOM-NEXT:    vmul.vv v8, v8, v16
@@ -891,12 +891,12 @@ define <vscale x 8 x i64> @vmul_xx_nxv8i64(i64 %a, i64 %b) nounwind {
 ; RV32M:       # %bb.0:
 ; RV32M-NEXT:    addi sp, sp, -16
 ; RV32M-NEXT:    mul a4, a0, a2
-; RV32M-NEXT:    sw a4, 8(sp)
 ; RV32M-NEXT:    mul a3, a0, a3
 ; RV32M-NEXT:    mulhu a0, a0, a2
 ; RV32M-NEXT:    add a0, a0, a3
 ; RV32M-NEXT:    mul a1, a1, a2
 ; RV32M-NEXT:    add a0, a0, a1
+; RV32M-NEXT:    sw a4, 8(sp)
 ; RV32M-NEXT:    sw a0, 12(sp)
 ; RV32M-NEXT:    addi a0, sp, 8
 ; RV32M-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmul-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmul-vp.ll
index a207d5be6ae041c9c5cdbe2a9febb19947393405..31dd6f9cfe2eb9367681203564994f0ae6225601 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmul-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmul-vp.ll
@@ -931,8 +931,8 @@ define <vscale x 1 x i64> @vmul_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -957,8 +957,8 @@ define <vscale x 1 x i64> @vmul_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1005,8 +1005,8 @@ define <vscale x 2 x i64> @vmul_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1031,8 +1031,8 @@ define <vscale x 2 x i64> @vmul_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1079,8 +1079,8 @@ define <vscale x 4 x i64> @vmul_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1105,8 +1105,8 @@ define <vscale x 4 x i64> @vmul_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1153,8 +1153,8 @@ define <vscale x 8 x i64> @vmul_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1179,8 +1179,8 @@ define <vscale x 8 x i64> @vmul_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmul.ll b/llvm/test/CodeGen/RISCV/rvv/vmul.ll
index 91d255ffeeecfdc96db90abcc7bd8f3eb329424e..81b9b60dc70c2b0881f12fc9b67bb283f8a370ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmul.ll
@@ -1862,8 +1862,8 @@ define <vscale x 1 x i64> @intrinsic_vmul_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vmul_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1897,8 +1897,8 @@ define <vscale x 1 x i64> @intrinsic_vmul_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmul_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1932,8 +1932,8 @@ define <vscale x 2 x i64> @intrinsic_vmul_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vmul_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1967,8 +1967,8 @@ define <vscale x 2 x i64> @intrinsic_vmul_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmul_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2002,8 +2002,8 @@ define <vscale x 4 x i64> @intrinsic_vmul_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vmul_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2037,8 +2037,8 @@ define <vscale x 4 x i64> @intrinsic_vmul_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmul_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2072,8 +2072,8 @@ define <vscale x 8 x i64> @intrinsic_vmul_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vmul_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2107,8 +2107,8 @@ define <vscale x 8 x i64> @intrinsic_vmul_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmul_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmulh.ll b/llvm/test/CodeGen/RISCV/rvv/vmulh.ll
index e6e60853d490b68b48433f0ee95af7aa39cad423..d0d86af7ea622a6ae281d9c936a80d42f810a0f6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmulh.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmulh.ll
@@ -1864,8 +1864,8 @@ define <vscale x 1 x i64> @intrinsic_vmulh_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vmulh_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1899,8 +1899,8 @@ define <vscale x 1 x i64> @intrinsic_vmulh_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmulh_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 2 x i64> @intrinsic_vmulh_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vmulh_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vmulh_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmulh_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2004,8 +2004,8 @@ define <vscale x 4 x i64> @intrinsic_vmulh_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vmulh_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2039,8 +2039,8 @@ define <vscale x 4 x i64> @intrinsic_vmulh_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmulh_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2074,8 +2074,8 @@ define <vscale x 8 x i64> @intrinsic_vmulh_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vmulh_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2109,8 +2109,8 @@ define <vscale x 8 x i64> @intrinsic_vmulh_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vmulh_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmulhsu.ll b/llvm/test/CodeGen/RISCV/rvv/vmulhsu.ll
index 42752473a4b350d48828939340336bb2dcfb21d2..03c4f3fa1de8ef3a8fa5f6c83cafeda180d2c4fd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmulhsu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmulhsu.ll
@@ -1864,8 +1864,8 @@ define <vscale x 1 x i64> @intrinsic_vmulhsu_vx_nxv1i64_nxv1i64_i64(<vscale x 1
 ; RV32-LABEL: intrinsic_vmulhsu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1899,8 +1899,8 @@ define <vscale x 1 x i64> @intrinsic_vmulhsu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhsu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 2 x i64> @intrinsic_vmulhsu_vx_nxv2i64_nxv2i64_i64(<vscale x 2
 ; RV32-LABEL: intrinsic_vmulhsu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vmulhsu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhsu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2004,8 +2004,8 @@ define <vscale x 4 x i64> @intrinsic_vmulhsu_vx_nxv4i64_nxv4i64_i64(<vscale x 4
 ; RV32-LABEL: intrinsic_vmulhsu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2039,8 +2039,8 @@ define <vscale x 4 x i64> @intrinsic_vmulhsu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhsu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2074,8 +2074,8 @@ define <vscale x 8 x i64> @intrinsic_vmulhsu_vx_nxv8i64_nxv8i64_i64(<vscale x 8
 ; RV32-LABEL: intrinsic_vmulhsu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2109,8 +2109,8 @@ define <vscale x 8 x i64> @intrinsic_vmulhsu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhsu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmulhu.ll b/llvm/test/CodeGen/RISCV/rvv/vmulhu.ll
index 6d50959cb0e4efe44aa0ad6f622e1a72d8dd7f26..8f5ef1664ad94c059920f14020d3e50dbc600372 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmulhu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmulhu.ll
@@ -1864,8 +1864,8 @@ define <vscale x 1 x i64> @intrinsic_vmulhu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vmulhu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1899,8 +1899,8 @@ define <vscale x 1 x i64> @intrinsic_vmulhu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 2 x i64> @intrinsic_vmulhu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vmulhu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vmulhu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2004,8 +2004,8 @@ define <vscale x 4 x i64> @intrinsic_vmulhu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vmulhu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2039,8 +2039,8 @@ define <vscale x 4 x i64> @intrinsic_vmulhu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2074,8 +2074,8 @@ define <vscale x 8 x i64> @intrinsic_vmulhu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vmulhu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2109,8 +2109,8 @@ define <vscale x 8 x i64> @intrinsic_vmulhu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vmulhu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv.s.x.ll b/llvm/test/CodeGen/RISCV/rvv/vmv.s.x.ll
index afb9cba328e7e633e469db65cc5ffeb231b7352b..07661c57640457b31782eb5b8b377a808d249f37 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmv.s.x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmv.s.x.ll
@@ -244,8 +244,8 @@ define <vscale x 1 x i64> @intrinsic_vmv.s.x_x_nxv1i64(<vscale x 1 x i64> %0, i6
 ; RV32-LABEL: intrinsic_vmv.s.x_x_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vid.v v9
 ; RV32-NEXT:    vmseq.vi v0, v9, 0
@@ -270,8 +270,8 @@ define <vscale x 2 x i64> @intrinsic_vmv.s.x_x_nxv2i64(<vscale x 2 x i64> %0, i6
 ; RV32-LABEL: intrinsic_vmv.s.x_x_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vid.v v10
 ; RV32-NEXT:    vmseq.vi v0, v10, 0
@@ -296,8 +296,8 @@ define <vscale x 4 x i64> @intrinsic_vmv.s.x_x_nxv4i64(<vscale x 4 x i64> %0, i6
 ; RV32-LABEL: intrinsic_vmv.s.x_x_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vid.v v12
 ; RV32-NEXT:    vmseq.vi v0, v12, 0
@@ -322,8 +322,8 @@ define <vscale x 8 x i64> @intrinsic_vmv.s.x_x_nxv8i64(<vscale x 8 x i64> %0, i6
 ; RV32-LABEL: intrinsic_vmv.s.x_x_nxv8i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vid.v v16
 ; RV32-NEXT:    vmseq.vi v0, v16, 0
@@ -347,10 +347,10 @@ define <vscale x 1 x i64> @intrinsic_vmv.s.x_x_nxv1i64_bug(<vscale x 1 x i64> %0
 ; RV32-LABEL: intrinsic_vmv.s.x_x_nxv1i64_bug:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    lw a1, 4(a0)
-; RV32-NEXT:    lw a0, 0(a0)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    lw a1, 0(a0)
+; RV32-NEXT:    lw a0, 4(a0)
+; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw a0, 12(sp)
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
 ; RV32-NEXT:    vid.v v9
 ; RV32-NEXT:    vmseq.vi v0, v9, 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv.v.x.ll b/llvm/test/CodeGen/RISCV/rvv/vmv.v.x.ll
index 4fa95fb2d945db91b8782cbf2ddc1190a7538470..79bdd30291c08c2e9573c59f82f9911c563f1430 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmv.v.x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmv.v.x.ll
@@ -373,8 +373,8 @@ define <vscale x 1 x i64> @intrinsic_vmv.v.x_x_nxv1i64(i64 %0, iXLen %1) nounwin
 ; RV32-LABEL: intrinsic_vmv.v.x_x_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -404,8 +404,8 @@ define <vscale x 2 x i64> @intrinsic_vmv.v.x_x_nxv2i64(i64 %0, iXLen %1) nounwin
 ; RV32-LABEL: intrinsic_vmv.v.x_x_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -435,8 +435,8 @@ define <vscale x 4 x i64> @intrinsic_vmv.v.x_x_nxv4i64(i64 %0, iXLen %1) nounwin
 ; RV32-LABEL: intrinsic_vmv.v.x_x_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -466,8 +466,8 @@ define <vscale x 8 x i64> @intrinsic_vmv.v.x_x_nxv8i64(i64 %0, iXLen %1) nounwin
 ; RV32-LABEL: intrinsic_vmv.v.x_x_nxv8i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnmsac-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vnmsac-vp.ll
index 30edcaf9b15b53c4a418645446610253e2b802b9..4eadd11d42104865363f7af5785fcfbf7ef8934b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnmsac-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnmsac-vp.ll
@@ -1651,8 +1651,8 @@ define <vscale x 1 x i64> @vnmsac_vx_nxv1i64(<vscale x 1 x i64> %a, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1681,8 +1681,8 @@ define <vscale x 1 x i64> @vnmsac_vx_nxv1i64_unmasked(<vscale x 1 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1724,8 +1724,8 @@ define <vscale x 1 x i64> @vnmsac_vx_nxv1i64_ta(<vscale x 1 x i64> %a, i64 %b, <
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1785,8 +1785,8 @@ define <vscale x 2 x i64> @vnmsac_vx_nxv2i64(<vscale x 2 x i64> %a, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1815,8 +1815,8 @@ define <vscale x 2 x i64> @vnmsac_vx_nxv2i64_unmasked(<vscale x 2 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1858,8 +1858,8 @@ define <vscale x 2 x i64> @vnmsac_vx_nxv2i64_ta(<vscale x 2 x i64> %a, i64 %b, <
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1919,8 +1919,8 @@ define <vscale x 4 x i64> @vnmsac_vx_nxv4i64(<vscale x 4 x i64> %a, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1949,8 +1949,8 @@ define <vscale x 4 x i64> @vnmsac_vx_nxv4i64_unmasked(<vscale x 4 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1992,8 +1992,8 @@ define <vscale x 4 x i64> @vnmsac_vx_nxv4i64_ta(<vscale x 4 x i64> %a, i64 %b, <
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2055,8 +2055,8 @@ define <vscale x 8 x i64> @vnmsac_vx_nxv8i64(<vscale x 8 x i64> %a, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -2085,8 +2085,8 @@ define <vscale x 8 x i64> @vnmsac_vx_nxv8i64_unmasked(<vscale x 8 x i64> %a, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
@@ -2129,8 +2129,8 @@ define <vscale x 8 x i64> @vnmsac_vx_nxv8i64_ta(<vscale x 8 x i64> %a, i64 %b, <
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vnmsac.ll
index 760f4d47ce2346dcc472d99e40917c31f1a7cda7..505443d93720bb55e324bb99c5d1388f4daf2f41 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnmsac.ll
@@ -1566,8 +1566,8 @@ define <vscale x 1 x i64>  @intrinsic_vnmsac_vx_nxv1i64_i64_nxv1i64(<vscale x 1
 ; RV32-LABEL: intrinsic_vnmsac_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1602,8 +1602,8 @@ define <vscale x 1 x i64> @intrinsic_vnmsac_mask_vx_nxv1i64_i64_nxv1i64(<vscale
 ; RV32-LABEL: intrinsic_vnmsac_mask_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1639,8 +1639,8 @@ define <vscale x 2 x i64>  @intrinsic_vnmsac_vx_nxv2i64_i64_nxv2i64(<vscale x 2
 ; RV32-LABEL: intrinsic_vnmsac_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1675,8 +1675,8 @@ define <vscale x 2 x i64> @intrinsic_vnmsac_mask_vx_nxv2i64_i64_nxv2i64(<vscale
 ; RV32-LABEL: intrinsic_vnmsac_mask_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1712,8 +1712,8 @@ define <vscale x 4 x i64>  @intrinsic_vnmsac_vx_nxv4i64_i64_nxv4i64(<vscale x 4
 ; RV32-LABEL: intrinsic_vnmsac_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1748,8 +1748,8 @@ define <vscale x 4 x i64> @intrinsic_vnmsac_mask_vx_nxv4i64_i64_nxv4i64(<vscale
 ; RV32-LABEL: intrinsic_vnmsac_mask_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnmsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vnmsub-sdnode.ll
index 186ffb64e5902503daa068eee189ed3a4a4810f5..c465785797844c2dca0b03fce3039166c8610d2b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnmsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnmsub-sdnode.ll
@@ -458,8 +458,8 @@ define <vscale x 1 x i64> @vnmsub_vx_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -495,8 +495,8 @@ define <vscale x 2 x i64> @vnmsub_vx_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -532,8 +532,8 @@ define <vscale x 4 x i64> @vnmsub_vx_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -570,8 +570,8 @@ define <vscale x 8 x i64> @vnmsub_vx_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnmsub.ll b/llvm/test/CodeGen/RISCV/rvv/vnmsub.ll
index 52036875c7d033430cb8df35f41394919f0054de..d9c7560830fecb1d416dd0b2f2ec0bc2aff28895 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnmsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnmsub.ll
@@ -1566,8 +1566,8 @@ define <vscale x 1 x i64>  @intrinsic_vnmsub_vx_nxv1i64_i64_nxv1i64(<vscale x 1
 ; RV32-LABEL: intrinsic_vnmsub_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1602,8 +1602,8 @@ define <vscale x 1 x i64> @intrinsic_vnmsub_mask_vx_nxv1i64_i64_nxv1i64(<vscale
 ; RV32-LABEL: intrinsic_vnmsub_mask_vx_nxv1i64_i64_nxv1i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1639,8 +1639,8 @@ define <vscale x 2 x i64>  @intrinsic_vnmsub_vx_nxv2i64_i64_nxv2i64(<vscale x 2
 ; RV32-LABEL: intrinsic_vnmsub_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1675,8 +1675,8 @@ define <vscale x 2 x i64> @intrinsic_vnmsub_mask_vx_nxv2i64_i64_nxv2i64(<vscale
 ; RV32-LABEL: intrinsic_vnmsub_mask_vx_nxv2i64_i64_nxv2i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1712,8 +1712,8 @@ define <vscale x 4 x i64>  @intrinsic_vnmsub_vx_nxv4i64_i64_nxv4i64(<vscale x 4
 ; RV32-LABEL: intrinsic_vnmsub_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1748,8 +1748,8 @@ define <vscale x 4 x i64> @intrinsic_vnmsub_mask_vx_nxv4i64_i64_nxv4i64(<vscale
 ; RV32-LABEL: intrinsic_vnmsub_mask_vx_nxv4i64_i64_nxv4i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vor-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vor-sdnode.ll
index dcfe07c1fba6583f954e74b66ce7a7ebac6bbaa1..33a7476ac852a11719334e2fc821845542c2bf72 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vor-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vor-sdnode.ll
@@ -781,8 +781,8 @@ define <vscale x 1 x i64> @vor_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -837,8 +837,8 @@ define <vscale x 2 x i64> @vor_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -893,8 +893,8 @@ define <vscale x 4 x i64> @vor_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -949,8 +949,8 @@ define <vscale x 8 x i64> @vor_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1016,9 +1016,9 @@ define <vscale x 8 x i64> @vor_xx_nxv8i64(i64 %a, i64 %b) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    or a1, a1, a3
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    or a0, a0, a2
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vor-vp.ll
index b9388e58797048d6cd56e31db0e984ce8f68c4b5..a23044967db44a9992770a1007932c82bc1b183c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vor-vp.ll
@@ -1323,8 +1323,8 @@ define <vscale x 1 x i64> @vor_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vscal
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1349,8 +1349,8 @@ define <vscale x 1 x i64> @vor_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1417,8 +1417,8 @@ define <vscale x 2 x i64> @vor_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vscal
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1443,8 +1443,8 @@ define <vscale x 2 x i64> @vor_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1511,8 +1511,8 @@ define <vscale x 4 x i64> @vor_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vscal
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1537,8 +1537,8 @@ define <vscale x 4 x i64> @vor_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1605,8 +1605,8 @@ define <vscale x 8 x i64> @vor_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vscal
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1631,8 +1631,8 @@ define <vscale x 8 x i64> @vor_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64 %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vor.ll b/llvm/test/CodeGen/RISCV/rvv/vor.ll
index 3badc121f0322b6661884c232697bef550a81dfe..8280f4c64684eeba7e52d0c2bfc7c8b64980d75e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vor.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vor.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vor_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i6
 ; RV32-LABEL: intrinsic_vor_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vor_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1
 ; RV32-LABEL: intrinsic_vor_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vor_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i6
 ; RV32-LABEL: intrinsic_vor_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vor_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2
 ; RV32-LABEL: intrinsic_vor_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vor_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i6
 ; RV32-LABEL: intrinsic_vor_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vor_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4
 ; RV32-LABEL: intrinsic_vor_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vor_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i6
 ; RV32-LABEL: intrinsic_vor_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vor_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8
 ; RV32-LABEL: intrinsic_vor_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splat.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splat.ll
index 0da05c1bd4364d85e7408945c43a43aad9f8f546..7fc9388ff10a4044379c41c26dba6158cec3c4ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splat.ll
@@ -191,8 +191,8 @@ define <vscale x 1 x i64> @vp_splat_nxv1i64(i64 %val, <vscale x 1 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -213,8 +213,8 @@ define <vscale x 2 x i64> @vp_splat_nxv2i64(i64 %val, <vscale x 2 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -235,8 +235,8 @@ define <vscale x 4 x i64> @vp_splat_nxv4i64(i64 %val, <vscale x 4 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
@@ -257,8 +257,8 @@ define <vscale x 8 x i64> @vp_splat_nxv8i64(i64 %val, <vscale x 8 x i1> %m, i32
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
index 86dfc74fdee37b0db9e28ad04736eb19407d1ee7..ea7bf65fc5644dfacbfb0eddf15a3f2ed2985c3e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
@@ -853,8 +853,8 @@ define <vscale x 1 x i64> @vpmerge_vx_nxv1i64(i64 %a, <vscale x 1 x i64> %vb, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, tu, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
@@ -900,8 +900,8 @@ define <vscale x 2 x i64> @vpmerge_vx_nxv2i64(i64 %a, <vscale x 2 x i64> %vb, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, tu, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
@@ -947,8 +947,8 @@ define <vscale x 4 x i64> @vpmerge_vx_nxv4i64(i64 %a, <vscale x 4 x i64> %vb, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, tu, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
@@ -994,8 +994,8 @@ define <vscale x 8 x i64> @vpmerge_vx_nxv8i64(i64 %a, <vscale x 8 x i64> %vb, <v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, tu, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-int-vp.ll
index c41ddaee75a86c0eb8506b7d8c1fe58ced908d52..85ad876004526a8b3670845ad733dceec7eecdca 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-int-vp.ll
@@ -1221,8 +1221,8 @@ define signext i64 @vpreduce_add_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1253,8 +1253,8 @@ define signext i64 @vpwreduce_add_nxv1i32(i64 signext %s, <vscale x 1 x i32> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1287,8 +1287,8 @@ define signext i64 @vpwreduce_uadd_nxv1i32(i64 signext %s, <vscale x 1 x i32> %v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1323,8 +1323,8 @@ define signext i64 @vpreduce_umax_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1357,8 +1357,8 @@ define signext i64 @vpreduce_smax_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1391,8 +1391,8 @@ define signext i64 @vpreduce_umin_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1425,8 +1425,8 @@ define signext i64 @vpreduce_smin_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1459,8 +1459,8 @@ define signext i64 @vpreduce_and_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1493,8 +1493,8 @@ define signext i64 @vpreduce_or_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v, <
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1527,8 +1527,8 @@ define signext i64 @vpreduce_xor_nxv1i64(i64 signext %s, <vscale x 1 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1561,8 +1561,8 @@ define signext i64 @vpreduce_add_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1593,8 +1593,8 @@ define signext i64 @vwpreduce_add_nxv2i32(i64 signext %s, <vscale x 2 x i32> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1627,8 +1627,8 @@ define signext i64 @vwpreduce_uadd_nxv2i32(i64 signext %s, <vscale x 2 x i32> %v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1663,8 +1663,8 @@ define signext i64 @vpreduce_umax_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1697,8 +1697,8 @@ define signext i64 @vpreduce_smax_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1731,8 +1731,8 @@ define signext i64 @vpreduce_umin_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1765,8 +1765,8 @@ define signext i64 @vpreduce_smin_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1799,8 +1799,8 @@ define signext i64 @vpreduce_and_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1833,8 +1833,8 @@ define signext i64 @vpreduce_or_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v, <
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1867,8 +1867,8 @@ define signext i64 @vpreduce_xor_nxv2i64(i64 signext %s, <vscale x 2 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1901,8 +1901,8 @@ define signext i64 @vpreduce_add_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1933,8 +1933,8 @@ define signext i64 @vpwreduce_add_nxv4i32(i64 signext %s, <vscale x 4 x i32> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1967,8 +1967,8 @@ define signext i64 @vpwreduce_uadd_nxv4i32(i64 signext %s, <vscale x 4 x i32> %v
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2003,8 +2003,8 @@ define signext i64 @vpreduce_umax_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2037,8 +2037,8 @@ define signext i64 @vpreduce_smax_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2071,8 +2071,8 @@ define signext i64 @vpreduce_umin_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2105,8 +2105,8 @@ define signext i64 @vpreduce_smin_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2139,8 +2139,8 @@ define signext i64 @vpreduce_and_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2173,8 +2173,8 @@ define signext i64 @vpreduce_or_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v, <
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2207,8 +2207,8 @@ define signext i64 @vpreduce_xor_nxv4i64(i64 signext %s, <vscale x 4 x i64> %v,
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode.ll
index 3a6ae5fdb21073e8fc3b6b3dbec0acabc822e754..8a3a47f23488674efebc38631fe4c9d201213227 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode.ll
@@ -939,8 +939,8 @@ define <vscale x 1 x i64> @vrem_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -966,10 +966,10 @@ define <vscale x 1 x i64> @vrem_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-V-NEXT:    vlse64.v v9, (a0), zero
@@ -1022,8 +1022,8 @@ define <vscale x 2 x i64> @vrem_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1049,10 +1049,10 @@ define <vscale x 2 x i64> @vrem_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-V-NEXT:    vlse64.v v10, (a0), zero
@@ -1105,8 +1105,8 @@ define <vscale x 4 x i64> @vrem_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1132,10 +1132,10 @@ define <vscale x 4 x i64> @vrem_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-V-NEXT:    vlse64.v v12, (a0), zero
@@ -1188,8 +1188,8 @@ define <vscale x 8 x i64> @vrem_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1215,10 +1215,10 @@ define <vscale x 8 x i64> @vrem_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 748983
 ; RV32-V-NEXT:    addi a0, a0, -586
+; RV32-V-NEXT:    lui a1, 898779
+; RV32-V-NEXT:    addi a1, a1, 1755
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    lui a0, 898779
-; RV32-V-NEXT:    addi a0, a0, 1755
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-V-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrem-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vrem-vp.ll
index 2ef96f4b3896fc9361bd1b170fb987349723aa8d..48b89907c57a3bd37b1f8b736a00bb58cd768759 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrem-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrem-vp.ll
@@ -892,8 +892,8 @@ define <vscale x 1 x i64> @vrem_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -918,8 +918,8 @@ define <vscale x 1 x i64> @vrem_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -966,8 +966,8 @@ define <vscale x 2 x i64> @vrem_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -992,8 +992,8 @@ define <vscale x 2 x i64> @vrem_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1040,8 +1040,8 @@ define <vscale x 4 x i64> @vrem_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1066,8 +1066,8 @@ define <vscale x 4 x i64> @vrem_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1114,8 +1114,8 @@ define <vscale x 8 x i64> @vrem_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1140,8 +1140,8 @@ define <vscale x 8 x i64> @vrem_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrem.ll b/llvm/test/CodeGen/RISCV/rvv/vrem.ll
index 15692419dd76ced1e15d0dff85d178062d28eff8..3c6488e84fd52e487538fa439f4a551dc0352655 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrem.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrem.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vrem_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vrem_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vrem_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrem_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vrem_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vrem_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vrem_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrem_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vrem_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vrem_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vrem_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrem_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vrem_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vrem_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vrem_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrem_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode.ll
index ed40f5af4fa4ca8ceb51397f4872340f7eda40b3..2013f1794e70007b99527788f1495d90a171a1ee 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode.ll
@@ -678,8 +678,8 @@ define <vscale x 1 x i64> @vremu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -704,9 +704,9 @@ define <vscale x 1 x i64> @vremu_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-V-NEXT:    vlse64.v v9, (a0), zero
@@ -783,8 +783,8 @@ define <vscale x 2 x i64> @vremu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -809,9 +809,9 @@ define <vscale x 2 x i64> @vremu_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-V-NEXT:    vlse64.v v10, (a0), zero
@@ -888,8 +888,8 @@ define <vscale x 4 x i64> @vremu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -914,9 +914,9 @@ define <vscale x 4 x i64> @vremu_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-V-NEXT:    vlse64.v v12, (a0), zero
@@ -993,8 +993,8 @@ define <vscale x 8 x i64> @vremu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1019,9 +1019,9 @@ define <vscale x 8 x i64> @vremu_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; RV32-V-NEXT:    addi sp, sp, -16
 ; RV32-V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-V-NEXT:    lui a0, 131072
+; RV32-V-NEXT:    li a1, 1
+; RV32-V-NEXT:    sw a1, 8(sp)
 ; RV32-V-NEXT:    sw a0, 12(sp)
-; RV32-V-NEXT:    li a0, 1
-; RV32-V-NEXT:    sw a0, 8(sp)
 ; RV32-V-NEXT:    addi a0, sp, 8
 ; RV32-V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-V-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vremu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vremu-vp.ll
index 1f1ed4a1269acbb988ff52720c804f12735899f3..20d33f6cadf22325f25542db27472ae59acf76c0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vremu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vremu-vp.ll
@@ -891,8 +891,8 @@ define <vscale x 1 x i64> @vremu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -917,8 +917,8 @@ define <vscale x 1 x i64> @vremu_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -965,8 +965,8 @@ define <vscale x 2 x i64> @vremu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -991,8 +991,8 @@ define <vscale x 2 x i64> @vremu_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1039,8 +1039,8 @@ define <vscale x 4 x i64> @vremu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1065,8 +1065,8 @@ define <vscale x 4 x i64> @vremu_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1113,8 +1113,8 @@ define <vscale x 8 x i64> @vremu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1139,8 +1139,8 @@ define <vscale x 8 x i64> @vremu_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vremu.ll b/llvm/test/CodeGen/RISCV/rvv/vremu.ll
index 3a090a51e62624eff07e131c95a98698d3d80a50..777407c144b51593ce99a35f6444de297b0e9a5a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vremu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vremu.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vremu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vremu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vremu_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vremu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vremu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vremu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vremu_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vremu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vremu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vremu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vremu_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vremu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vremu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vremu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vremu_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vremu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vrsub-sdnode.ll
index e97b1f41ad3d368852e03e3d3cf1e1c5d3d4497a..e7b9f716a70d5facc22e8c39f831f91f9830ccc0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrsub-sdnode.ll
@@ -403,8 +403,8 @@ define <vscale x 1 x i64> @vrsub_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -438,8 +438,8 @@ define <vscale x 2 x i64> @vrsub_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -473,8 +473,8 @@ define <vscale x 4 x i64> @vrsub_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -508,8 +508,8 @@ define <vscale x 8 x i64> @vrsub_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vrsub-vp.ll
index be372c9aa54d5d049e2f4e06dccd9fc31f2e2860..f211c900ee5d3ae99022100ec02055676c77dc7e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrsub-vp.ll
@@ -839,8 +839,8 @@ define <vscale x 1 x i64> @vrsub_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -865,8 +865,8 @@ define <vscale x 1 x i64> @vrsub_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -913,8 +913,8 @@ define <vscale x 2 x i64> @vrsub_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -939,8 +939,8 @@ define <vscale x 2 x i64> @vrsub_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -987,8 +987,8 @@ define <vscale x 4 x i64> @vrsub_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1013,8 +1013,8 @@ define <vscale x 4 x i64> @vrsub_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1061,8 +1061,8 @@ define <vscale x 8 x i64> @vrsub_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1087,8 +1087,8 @@ define <vscale x 8 x i64> @vrsub_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrsub.ll b/llvm/test/CodeGen/RISCV/rvv/vrsub.ll
index 1a8db00a1c95910c62a655e3822312c2a85fe2fb..19d6afbd0e57efe859b057639bbeb3812b95ae88 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrsub.ll
@@ -842,8 +842,8 @@ define <vscale x 1 x i64> @intrinsic_vrsub_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vrsub_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -877,8 +877,8 @@ define <vscale x 1 x i64> @intrinsic_vrsub_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrsub_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -912,8 +912,8 @@ define <vscale x 2 x i64> @intrinsic_vrsub_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vrsub_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -947,8 +947,8 @@ define <vscale x 2 x i64> @intrinsic_vrsub_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrsub_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -982,8 +982,8 @@ define <vscale x 4 x i64> @intrinsic_vrsub_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vrsub_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1017,8 +1017,8 @@ define <vscale x 4 x i64> @intrinsic_vrsub_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrsub_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1052,8 +1052,8 @@ define <vscale x 8 x i64> @intrinsic_vrsub_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vrsub_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1087,8 +1087,8 @@ define <vscale x 8 x i64> @intrinsic_vrsub_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vrsub_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vsadd-sdnode.ll
index 6a8b801254057bb3c2f82d16aef28dbcc5bfa1f9..2961bdece3cef238f8f12d855a1b601eceaadc6e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsadd-sdnode.ll
@@ -633,8 +633,8 @@ define <vscale x 1 x i64> @sadd_nxv1i64_vx(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -680,8 +680,8 @@ define <vscale x 2 x i64> @sadd_nxv2i64_vx(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -727,8 +727,8 @@ define <vscale x 4 x i64> @sadd_nxv4i64_vx(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -774,8 +774,8 @@ define <vscale x 8 x i64> @sadd_nxv8i64_vx(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll
index 71b91f56e89a5e1417773c7c65c4c599ff37a788..7c6167bd5bc7ba0c5d4360176debe019cde3d973 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsadd-vp.ll
@@ -1423,8 +1423,8 @@ define <vscale x 1 x i64> @vsadd_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1449,8 +1449,8 @@ define <vscale x 1 x i64> @vsadd_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1517,8 +1517,8 @@ define <vscale x 2 x i64> @vsadd_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1543,8 +1543,8 @@ define <vscale x 2 x i64> @vsadd_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1611,8 +1611,8 @@ define <vscale x 4 x i64> @vsadd_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1637,8 +1637,8 @@ define <vscale x 4 x i64> @vsadd_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1705,8 +1705,8 @@ define <vscale x 8 x i64> @vsadd_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1731,8 +1731,8 @@ define <vscale x 8 x i64> @vsadd_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsadd.ll b/llvm/test/CodeGen/RISCV/rvv/vsadd.ll
index a108d98c1731b395b0c7a84f3637c7ce9d346a6d..445942f8ef2353058eda72fb364c698a07471aca 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsadd.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vsadd_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vsadd_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vsadd_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsadd_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vsadd_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vsadd_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vsadd_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsadd_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vsadd_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vsadd_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vsadd_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsadd_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vsadd_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vsadd_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vsadd_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsadd_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsaddu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vsaddu-sdnode.ll
index 4fe765c34ba6ce023d2cafba378993ed78ed31d5..3a595db88ef85b2d715584943899c0a7c1e29025 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsaddu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsaddu-sdnode.ll
@@ -633,8 +633,8 @@ define <vscale x 1 x i64> @uadd_nxv1i64_vx(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -680,8 +680,8 @@ define <vscale x 2 x i64> @uadd_nxv2i64_vx(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -727,8 +727,8 @@ define <vscale x 4 x i64> @uadd_nxv4i64_vx(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -774,8 +774,8 @@ define <vscale x 8 x i64> @uadd_nxv8i64_vx(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll
index 454a4ebab04a285ecbee12396dfcbb08a96c32c3..916cbdc7e62207dd38a1cd2120c26e667e124858 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsaddu-vp.ll
@@ -1422,8 +1422,8 @@ define <vscale x 1 x i64> @vsaddu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1448,8 +1448,8 @@ define <vscale x 1 x i64> @vsaddu_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1516,8 +1516,8 @@ define <vscale x 2 x i64> @vsaddu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1542,8 +1542,8 @@ define <vscale x 2 x i64> @vsaddu_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1610,8 +1610,8 @@ define <vscale x 4 x i64> @vsaddu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1636,8 +1636,8 @@ define <vscale x 4 x i64> @vsaddu_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1704,8 +1704,8 @@ define <vscale x 8 x i64> @vsaddu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1730,8 +1730,8 @@ define <vscale x 8 x i64> @vsaddu_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsaddu.ll b/llvm/test/CodeGen/RISCV/rvv/vsaddu.ll
index 57a89d6fe7d234f0eb7510088d6d7ecca671902d..37f93fd90e5360f1f0b2eacc8bf919d3d748f021 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsaddu.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vsaddu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vsaddu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vsaddu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vsaddu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vsaddu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vsaddu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vsaddu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vsaddu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vsaddu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vsaddu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vsaddu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vsaddu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vsaddu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vsaddu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vsaddu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vsaddu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsbc.ll b/llvm/test/CodeGen/RISCV/rvv/vsbc.ll
index 178935a1df32ea002066183dba3259812346afaa..effc04b87dee213f8cb6b478ad4c597d80f438a7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsbc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsbc.ll
@@ -975,8 +975,8 @@ define <vscale x 1 x i64> @intrinsic_vsbc_vxm_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vsbc_vxm_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1011,8 +1011,8 @@ define <vscale x 2 x i64> @intrinsic_vsbc_vxm_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vsbc_vxm_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1047,8 +1047,8 @@ define <vscale x 4 x i64> @intrinsic_vsbc_vxm_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vsbc_vxm_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1083,8 +1083,8 @@ define <vscale x 8 x i64> @intrinsic_vsbc_vxm_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vsbc_vxm_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll
index 2715ec78bd794918f803629d619f6b9cc3fbbb9f..a92904cea5a0f5885530c1b2864d4cda91296905 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-int.ll
@@ -627,8 +627,8 @@ define <vscale x 1 x i64> @vmerge_xv_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
@@ -671,8 +671,8 @@ define <vscale x 2 x i64> @vmerge_xv_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
@@ -715,8 +715,8 @@ define <vscale x 4 x i64> @vmerge_xv_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
@@ -759,8 +759,8 @@ define <vscale x 8 x i64> @vmerge_xv_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v8, (a0), zero, v0.t
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsmul.ll b/llvm/test/CodeGen/RISCV/rvv/vsmul.ll
index bc53bce889ddb7cb8adebd33101166fbe946cab5..8592f01cdfbbb45c7ae9c6a0427a214d28ce27d5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsmul.ll
@@ -1942,8 +1942,8 @@ define <vscale x 1 x i64> @intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1979,8 +1979,8 @@ define <vscale x 1 x i64> @intrinsic_vsmul_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsmul_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2016,8 +2016,8 @@ define <vscale x 2 x i64> @intrinsic_vsmul_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vsmul_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2053,8 +2053,8 @@ define <vscale x 2 x i64> @intrinsic_vsmul_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsmul_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2090,8 +2090,8 @@ define <vscale x 4 x i64> @intrinsic_vsmul_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vsmul_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2127,8 +2127,8 @@ define <vscale x 4 x i64> @intrinsic_vsmul_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsmul_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2164,8 +2164,8 @@ define <vscale x 8 x i64> @intrinsic_vsmul_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vsmul_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2201,8 +2201,8 @@ define <vscale x 8 x i64> @intrinsic_vsmul_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsmul_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll b/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll
index 721d6ef26d61e129db63fdfca573ab8b2abd5eb7..5a180e2c8f76146ceeeb2a2ab26a5f1210c3b1f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsplats-i64.ll
@@ -37,10 +37,10 @@ define <vscale x 8 x i64> @vsplat_nxv8i64_4() {
 ; RV32V:       # %bb.0:
 ; RV32V-NEXT:    addi sp, sp, -16
 ; RV32V-NEXT:    .cfi_def_cfa_offset 16
-; RV32V-NEXT:    sw zero, 12(sp)
 ; RV32V-NEXT:    lui a0, 1028096
 ; RV32V-NEXT:    addi a0, a0, -1281
 ; RV32V-NEXT:    sw a0, 8(sp)
+; RV32V-NEXT:    sw zero, 12(sp)
 ; RV32V-NEXT:    addi a0, sp, 8
 ; RV32V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32V-NEXT:    vlse64.v v8, (a0), zero
@@ -63,8 +63,8 @@ define <vscale x 8 x i64> @vsplat_nxv8i64_5(i64 %a) {
 ; RV32V:       # %bb.0:
 ; RV32V-NEXT:    addi sp, sp, -16
 ; RV32V-NEXT:    .cfi_def_cfa_offset 16
-; RV32V-NEXT:    sw a1, 12(sp)
 ; RV32V-NEXT:    sw a0, 8(sp)
+; RV32V-NEXT:    sw a1, 12(sp)
 ; RV32V-NEXT:    addi a0, sp, 8
 ; RV32V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32V-NEXT:    vlse64.v v8, (a0), zero
@@ -137,10 +137,10 @@ define <vscale x 8 x i64> @vadd_vx_nxv8i64_10(<vscale x 8 x i64> %v) {
 ; RV32V:       # %bb.0:
 ; RV32V-NEXT:    addi sp, sp, -16
 ; RV32V-NEXT:    .cfi_def_cfa_offset 16
-; RV32V-NEXT:    sw zero, 12(sp)
 ; RV32V-NEXT:    lui a0, 1028096
 ; RV32V-NEXT:    addi a0, a0, -1281
 ; RV32V-NEXT:    sw a0, 8(sp)
+; RV32V-NEXT:    sw zero, 12(sp)
 ; RV32V-NEXT:    addi a0, sp, 8
 ; RV32V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32V-NEXT:    vlse64.v v16, (a0), zero
@@ -166,10 +166,10 @@ define <vscale x 8 x i64> @vadd_vx_nxv8i64_11(<vscale x 8 x i64> %v) {
 ; RV32V-NEXT:    addi sp, sp, -16
 ; RV32V-NEXT:    .cfi_def_cfa_offset 16
 ; RV32V-NEXT:    li a0, 1
+; RV32V-NEXT:    lui a1, 1028096
+; RV32V-NEXT:    addi a1, a1, -1281
+; RV32V-NEXT:    sw a1, 8(sp)
 ; RV32V-NEXT:    sw a0, 12(sp)
-; RV32V-NEXT:    lui a0, 1028096
-; RV32V-NEXT:    addi a0, a0, -1281
-; RV32V-NEXT:    sw a0, 8(sp)
 ; RV32V-NEXT:    addi a0, sp, 8
 ; RV32V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32V-NEXT:    vlse64.v v16, (a0), zero
@@ -194,8 +194,8 @@ define <vscale x 8 x i64> @vadd_vx_nxv8i64_12(<vscale x 8 x i64> %v, i64 %a) {
 ; RV32V:       # %bb.0:
 ; RV32V-NEXT:    addi sp, sp, -16
 ; RV32V-NEXT:    .cfi_def_cfa_offset 16
-; RV32V-NEXT:    sw a1, 12(sp)
 ; RV32V-NEXT:    sw a0, 8(sp)
+; RV32V-NEXT:    sw a1, 12(sp)
 ; RV32V-NEXT:    addi a0, sp, 8
 ; RV32V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32V-NEXT:    vlse64.v v16, (a0), zero
@@ -238,8 +238,8 @@ define <vscale x 8 x i64> @vsplat_nxv8i64_14(i32 %a) {
 ; RV32V:       # %bb.0:
 ; RV32V-NEXT:    addi sp, sp, -16
 ; RV32V-NEXT:    .cfi_def_cfa_offset 16
-; RV32V-NEXT:    sw zero, 12(sp)
 ; RV32V-NEXT:    sw a0, 8(sp)
+; RV32V-NEXT:    sw zero, 12(sp)
 ; RV32V-NEXT:    addi a0, sp, 8
 ; RV32V-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32V-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vssub-sdnode.ll
index c043858c029473a2fb7ee986ec779c1dfe31c47b..632e28607b9be18b37e9bbbbbc8828e2d3608382 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssub-sdnode.ll
@@ -651,8 +651,8 @@ define <vscale x 1 x i64> @ssub_nxv1i64_vx(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -699,8 +699,8 @@ define <vscale x 2 x i64> @ssub_nxv2i64_vx(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -747,8 +747,8 @@ define <vscale x 4 x i64> @ssub_nxv4i64_vx(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -795,8 +795,8 @@ define <vscale x 8 x i64> @ssub_nxv8i64_vx(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll
index 613b58b0f1b88a5e41064683eaf8d1b586bf2c35..2f9786e237084ded0c3c8a5c6d1eeb68ddc7a55e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssub-vp.ll
@@ -1467,8 +1467,8 @@ define <vscale x 1 x i64> @vssub_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1493,8 +1493,8 @@ define <vscale x 1 x i64> @vssub_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1563,8 +1563,8 @@ define <vscale x 2 x i64> @vssub_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1589,8 +1589,8 @@ define <vscale x 2 x i64> @vssub_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1659,8 +1659,8 @@ define <vscale x 4 x i64> @vssub_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1685,8 +1685,8 @@ define <vscale x 4 x i64> @vssub_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1755,8 +1755,8 @@ define <vscale x 8 x i64> @vssub_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsc
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1781,8 +1781,8 @@ define <vscale x 8 x i64> @vssub_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssub.ll b/llvm/test/CodeGen/RISCV/rvv/vssub.ll
index 50fca5e832af5068ed185e777f66c687159accf9..2ac94f3c55a8d119839b26a79f94f03a2c10b2d4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssub.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vssub_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vssub_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vssub_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vssub_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vssub_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vssub_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vssub_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vssub_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vssub_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vssub_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vssub_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vssub_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vssub_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vssub_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vssub_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vssub_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssubu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vssubu-sdnode.ll
index 5349548a213bc7322da4a249278f82e02373a78d..83ffb81382f1956fa1acecd13e2fa6fce9e27104 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssubu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssubu-sdnode.ll
@@ -651,8 +651,8 @@ define <vscale x 1 x i64> @usub_nxv1i64_vx(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -699,8 +699,8 @@ define <vscale x 2 x i64> @usub_nxv2i64_vx(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -747,8 +747,8 @@ define <vscale x 4 x i64> @usub_nxv4i64_vx(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -795,8 +795,8 @@ define <vscale x 8 x i64> @usub_nxv8i64_vx(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll
index 8c729d7d9bfb6eacb921ab7161591420f00eb932..dea0a82eca375d7e99a0cf7f5bbce5c929eaf0e0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssubu-vp.ll
@@ -1465,8 +1465,8 @@ define <vscale x 1 x i64> @vssubu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1491,8 +1491,8 @@ define <vscale x 1 x i64> @vssubu_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1561,8 +1561,8 @@ define <vscale x 2 x i64> @vssubu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1587,8 +1587,8 @@ define <vscale x 2 x i64> @vssubu_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1657,8 +1657,8 @@ define <vscale x 4 x i64> @vssubu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1683,8 +1683,8 @@ define <vscale x 4 x i64> @vssubu_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1753,8 +1753,8 @@ define <vscale x 8 x i64> @vssubu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vs
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1779,8 +1779,8 @@ define <vscale x 8 x i64> @vssubu_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i6
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssubu.ll b/llvm/test/CodeGen/RISCV/rvv/vssubu.ll
index db1b4ce34e9b381444c75017c1e72b6e8af4d596..9827b20b5ca0a96701e70b6a022daa0a68dbd2f1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssubu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssubu.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vssubu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-LABEL: intrinsic_vssubu_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vssubu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vssubu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vssubu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; RV32-LABEL: intrinsic_vssubu_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vssubu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vssubu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vssubu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; RV32-LABEL: intrinsic_vssubu_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vssubu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vssubu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vssubu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; RV32-LABEL: intrinsic_vssubu_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vssubu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ; RV32-LABEL: intrinsic_vssubu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
index c2173c9a291fcfadde12495f0cbe46a799fbf25c..e2b0a11e16883dbea73dd7515822fac587df27de 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
@@ -622,8 +622,8 @@ define <vscale x 1 x i64> @vsub_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -668,8 +668,8 @@ define <vscale x 2 x i64> @vsub_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -714,8 +714,8 @@ define <vscale x 4 x i64> @vsub_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -760,8 +760,8 @@ define <vscale x 8 x i64> @vsub_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -796,10 +796,10 @@ define <vscale x 8 x i64> @vsub_xx_nxv8i64(i64 %a, i64 %b) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    sub a4, a0, a2
-; RV32-NEXT:    sw a4, 8(sp)
 ; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    sub a1, a1, a3
 ; RV32-NEXT:    sub a1, a1, a0
+; RV32-NEXT:    sw a4, 8(sp)
 ; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsub-vp.ll
index a2b9285fedeaf78a30b3f2671946b26b1dabbe00..634f5cd0a9a0e1a7a74f883422d4bc1c8620c4e5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsub-vp.ll
@@ -919,8 +919,8 @@ define <vscale x 1 x i64> @vsub_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -945,8 +945,8 @@ define <vscale x 1 x i64> @vsub_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -993,8 +993,8 @@ define <vscale x 2 x i64> @vsub_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1019,8 +1019,8 @@ define <vscale x 2 x i64> @vsub_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1067,8 +1067,8 @@ define <vscale x 4 x i64> @vsub_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1093,8 +1093,8 @@ define <vscale x 4 x i64> @vsub_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1141,8 +1141,8 @@ define <vscale x 8 x i64> @vsub_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1167,8 +1167,8 @@ define <vscale x 8 x i64> @vsub_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsub.ll b/llvm/test/CodeGen/RISCV/rvv/vsub.ll
index c7e153d86fce3a828ca242c46589437b43698582..ecf23722917d3bd94a7000d323cb2a100dcb537a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsub.ll
@@ -1858,8 +1858,8 @@ define <vscale x 1 x i64> @intrinsic_vsub_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vsub_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1893,8 +1893,8 @@ define <vscale x 1 x i64> @intrinsic_vsub_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsub_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1928,8 +1928,8 @@ define <vscale x 2 x i64> @intrinsic_vsub_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vsub_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1963,8 +1963,8 @@ define <vscale x 2 x i64> @intrinsic_vsub_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsub_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1998,8 +1998,8 @@ define <vscale x 4 x i64> @intrinsic_vsub_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vsub_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2033,8 +2033,8 @@ define <vscale x 4 x i64> @intrinsic_vsub_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsub_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2068,8 +2068,8 @@ define <vscale x 8 x i64> @intrinsic_vsub_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vsub_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2103,8 +2103,8 @@ define <vscale x 8 x i64> @intrinsic_vsub_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vsub_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
index a8b836b7ae3930b463a06a9751fe63f45bdf4d8f..2f7dca0182a4fab216bd0bd5cd1bbc50086c26db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
@@ -1470,8 +1470,8 @@ define <vscale x 8 x i64> @vwadd_vx_splat_zext(<vscale x 8 x i32> %va, i32 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1526,8 +1526,8 @@ define <vscale x 8 x i64> @vwadd_wx_splat_zext(<vscale x 8 x i64> %va, i32 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxor-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vxor-sdnode.ll
index b03a105610dfdf5d8be28a7b147a60406e414afc..e98bd1a1ad78208c89f44f7eb647df5f2397a5e4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxor-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vxor-sdnode.ll
@@ -971,8 +971,8 @@ define <vscale x 1 x i64> @vxor_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1037,8 +1037,8 @@ define <vscale x 2 x i64> @vxor_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1103,8 +1103,8 @@ define <vscale x 4 x i64> @vxor_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1169,8 +1169,8 @@ define <vscale x 8 x i64> @vxor_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -1225,9 +1225,9 @@ define <vscale x 8 x i64> @vxor_xx_nxv8i64(i64 %a, i64 %b) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    xor a1, a1, a3
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    xor a0, a0, a2
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v8, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vxor-vp.ll
index f2235b4fdc94ba14664ce00852a57eda7b544a42..3ad0c22894561aa227218f7657ec817966f09456 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vxor-vp.ll
@@ -1691,8 +1691,8 @@ define <vscale x 1 x i64> @vxor_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1717,8 +1717,8 @@ define <vscale x 1 x i64> @vxor_vx_nxv1i64_unmasked(<vscale x 1 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1805,8 +1805,8 @@ define <vscale x 2 x i64> @vxor_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1831,8 +1831,8 @@ define <vscale x 2 x i64> @vxor_vx_nxv2i64_unmasked(<vscale x 2 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1919,8 +1919,8 @@ define <vscale x 4 x i64> @vxor_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -1945,8 +1945,8 @@ define <vscale x 4 x i64> @vxor_vx_nxv4i64_unmasked(<vscale x 4 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2033,8 +2033,8 @@ define <vscale x 8 x i64> @vxor_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b, <vsca
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2059,8 +2059,8 @@ define <vscale x 8 x i64> @vxor_vx_nxv8i64_unmasked(<vscale x 8 x i64> %va, i64
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxor.ll b/llvm/test/CodeGen/RISCV/rvv/vxor.ll
index b08d4530d0085c7eee847ef9ddd010d39d3ddf44..a3cb2f42c8ecf8833d8451e195676020acaa6b71 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxor.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vxor.ll
@@ -1898,8 +1898,8 @@ define <vscale x 1 x i64> @intrinsic_vxor_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; RV32-LABEL: intrinsic_vxor_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v9, (a0), zero
@@ -1934,8 +1934,8 @@ define <vscale x 1 x i64> @intrinsic_vxor_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vxor_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, mu
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -1969,8 +1969,8 @@ define <vscale x 2 x i64> @intrinsic_vxor_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; RV32-LABEL: intrinsic_vxor_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
@@ -2005,8 +2005,8 @@ define <vscale x 2 x i64> @intrinsic_vxor_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vxor_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m2, ta, mu
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2040,8 +2040,8 @@ define <vscale x 4 x i64> @intrinsic_vxor_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; RV32-LABEL: intrinsic_vxor_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, ma
 ; RV32-NEXT:    vlse64.v v12, (a0), zero
@@ -2076,8 +2076,8 @@ define <vscale x 4 x i64> @intrinsic_vxor_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vxor_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m4, ta, mu
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2111,8 +2111,8 @@ define <vscale x 8 x i64> @intrinsic_vxor_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; RV32-LABEL: intrinsic_vxor_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vlse64.v v16, (a0), zero
@@ -2147,8 +2147,8 @@ define <vscale x 8 x i64> @intrinsic_vxor_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ; RV32-LABEL: intrinsic_vxor_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, mu
 ; RV32-NEXT:    vlse64.v v24, (a0), zero
diff --git a/llvm/test/CodeGen/RISCV/rvv/xsfvcp-xvv.ll b/llvm/test/CodeGen/RISCV/rvv/xsfvcp-xvv.ll
index 2c9100111fab6017958f277c5bd5e67ac44696bc..b09e9f0e3365c1301408f629dd29e0bcb77fb954 100644
--- a/llvm/test/CodeGen/RISCV/rvv/xsfvcp-xvv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/xsfvcp-xvv.ll
@@ -297,7 +297,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv8i64.nxv8i64.iXLen(iXLen, <vscale
 define <vscale x 1 x i8> @test_sf_vc_v_vvv_se_e8mf8(<vscale x 1 x i8> %vd, <vscale x 1 x i8> %vs2, <vscale x 1 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -310,7 +310,7 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv1i8.iXLen.nxv1i8.iXLen(i
 define <vscale x 2 x i8> @test_sf_vc_v_vvv_se_e8mf4(<vscale x 2 x i8> %vd, <vscale x 2 x i8> %vs2, <vscale x 2 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -323,7 +323,7 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv2i8.iXLen.nxv2i8.iXLen(i
 define <vscale x 4 x i8> @test_sf_vc_v_vvv_se_e8mf2(<vscale x 4 x i8> %vd, <vscale x 4 x i8> %vs2, <vscale x 4 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -336,7 +336,7 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv4i8.iXLen.nxv4i8.iXLen(i
 define <vscale x 8 x i8> @test_sf_vc_v_vvv_se_e8m1(<vscale x 8 x i8> %vd, <vscale x 8 x i8> %vs2, <vscale x 8 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -349,7 +349,7 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv8i8.iXLen.nxv8i8.iXLen(i
 define <vscale x 16 x i8> @test_sf_vc_v_vvv_se_e8m2(<vscale x 16 x i8> %vd, <vscale x 16 x i8> %vs2, <vscale x 16 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -362,7 +362,7 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv16i8.iXLen.nxv16i8.iXLe
 define <vscale x 32 x i8> @test_sf_vc_v_vvv_se_e8m4(<vscale x 32 x i8> %vd, <vscale x 32 x i8> %vs2, <vscale x 32 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -376,7 +376,7 @@ define <vscale x 64 x i8> @test_sf_vc_v_vvv_se_e8m8(<vscale x 64 x i8> %vd, <vsc
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e8m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -389,7 +389,7 @@ declare <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv64i8.iXLen.nxv64i8.iXLe
 define <vscale x 1 x i16> @test_sf_vc_v_vvv_se_e16mf4(<vscale x 1 x i16> %vd, <vscale x 1 x i16> %vs2, <vscale x 1 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -402,7 +402,7 @@ declare <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv1i16.iXLen.nxv1i16.iXLe
 define <vscale x 2 x i16> @test_sf_vc_v_vvv_se_e16mf2(<vscale x 2 x i16> %vd, <vscale x 2 x i16> %vs2, <vscale x 2 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -415,7 +415,7 @@ declare <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv2i16.iXLen.nxv2i16.iXLe
 define <vscale x 4 x i16> @test_sf_vc_v_vvv_se_e16m1(<vscale x 4 x i16> %vd, <vscale x 4 x i16> %vs2, <vscale x 4 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -428,7 +428,7 @@ declare <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv4i16.iXLen.nxv4i16.iXLe
 define <vscale x 8 x i16> @test_sf_vc_v_vvv_se_e16m2(<vscale x 8 x i16> %vd, <vscale x 8 x i16> %vs2, <vscale x 8 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -441,7 +441,7 @@ declare <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv8i16.iXLen.nxv8i16.iXLe
 define <vscale x 16 x i16> @test_sf_vc_v_vvv_se_e16m4(<vscale x 16 x i16> %vd, <vscale x 16 x i16> %vs2, <vscale x 16 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -455,7 +455,7 @@ define <vscale x 32 x i16> @test_sf_vc_v_vvv_se_e16m8(<vscale x 32 x i16> %vd, <
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -468,7 +468,7 @@ declare <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv32i16.iXLen.nxv32i16.i
 define <vscale x 1 x i32> @test_sf_vc_v_vvv_se_e32mf2(<vscale x 1 x i32> %vd, <vscale x 1 x i32> %vs2, <vscale x 1 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -481,7 +481,7 @@ declare <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv1i32.iXLen.nxv1i32.iXLe
 define <vscale x 2 x i32> @test_sf_vc_v_vvv_se_e32m1(<vscale x 2 x i32> %vd, <vscale x 2 x i32> %vs2, <vscale x 2 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -494,7 +494,7 @@ declare <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv2i32.iXLen.nxv2i32.iXLe
 define <vscale x 4 x i32> @test_sf_vc_v_vvv_se_e32m2(<vscale x 4 x i32> %vd, <vscale x 4 x i32> %vs2, <vscale x 4 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -507,7 +507,7 @@ declare <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv4i32.iXLen.nxv4i32.iXLe
 define <vscale x 8 x i32> @test_sf_vc_v_vvv_se_e32m4(<vscale x 8 x i32> %vd, <vscale x 8 x i32> %vs2, <vscale x 8 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -521,7 +521,7 @@ define <vscale x 16 x i32> @test_sf_vc_v_vvv_se_e32m8(<vscale x 16 x i32> %vd, <
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -534,7 +534,7 @@ declare <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv16i32.iXLen.nxv16i32.i
 define <vscale x 1 x i64> @test_sf_vc_v_vvv_se_e64m1(<vscale x 1 x i64> %vd, <vscale x 1 x i64> %vs2, <vscale x 1 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -547,7 +547,7 @@ declare <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv1i64.iXLen.nxv1i64.iXLe
 define <vscale x 2 x i64> @test_sf_vc_v_vvv_se_e64m2(<vscale x 2 x i64> %vd, <vscale x 2 x i64> %vs2, <vscale x 2 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -560,7 +560,7 @@ declare <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv2i64.iXLen.nxv2i64.iXLe
 define <vscale x 4 x i64> @test_sf_vc_v_vvv_se_e64m4(<vscale x 4 x i64> %vd, <vscale x 4 x i64> %vs2, <vscale x 4 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -574,7 +574,7 @@ define <vscale x 8 x i64> @test_sf_vc_v_vvv_se_e64m8(<vscale x 8 x i64> %vd, <vs
 ; CHECK-LABEL: test_sf_vc_v_vvv_se_e64m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -587,7 +587,7 @@ declare <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv8i64.iXLen.nxv8i64.iXLe
 define <vscale x 1 x i8> @test_sf_vc_v_vvv_e8mf8(<vscale x 1 x i8> %vd, <vscale x 1 x i8> %vs2, <vscale x 1 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -600,7 +600,7 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vvv.nxv1i8.iXLen.nxv1i8.iXLen(iXLe
 define <vscale x 2 x i8> @test_sf_vc_v_vvv_e8mf4(<vscale x 2 x i8> %vd, <vscale x 2 x i8> %vs2, <vscale x 2 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -613,7 +613,7 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vvv.nxv2i8.iXLen.nxv2i8.iXLen(iXLe
 define <vscale x 4 x i8> @test_sf_vc_v_vvv_e8mf2(<vscale x 4 x i8> %vd, <vscale x 4 x i8> %vs2, <vscale x 4 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -626,7 +626,7 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vvv.nxv4i8.iXLen.nxv4i8.iXLen(iXLe
 define <vscale x 8 x i8> @test_sf_vc_v_vvv_e8m1(<vscale x 8 x i8> %vd, <vscale x 8 x i8> %vs2, <vscale x 8 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -639,7 +639,7 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vvv.nxv8i8.iXLen.nxv8i8.iXLen(iXLe
 define <vscale x 16 x i8> @test_sf_vc_v_vvv_e8m2(<vscale x 16 x i8> %vd, <vscale x 16 x i8> %vs2, <vscale x 16 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -652,7 +652,7 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vvv.nxv16i8.iXLen.nxv16i8.iXLen(i
 define <vscale x 32 x i8> @test_sf_vc_v_vvv_e8m4(<vscale x 32 x i8> %vd, <vscale x 32 x i8> %vs2, <vscale x 32 x i8> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -666,7 +666,7 @@ define <vscale x 64 x i8> @test_sf_vc_v_vvv_e8m8(<vscale x 64 x i8> %vd, <vscale
 ; CHECK-LABEL: test_sf_vc_v_vvv_e8m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -679,7 +679,7 @@ declare <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vvv.nxv64i8.iXLen.nxv64i8.iXLen(i
 define <vscale x 1 x i16> @test_sf_vc_v_vvv_e16mf4(<vscale x 1 x i16> %vd, <vscale x 1 x i16> %vs2, <vscale x 1 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -692,7 +692,7 @@ declare <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvv.nxv1i16.iXLen.nxv1i16.iXLen(i
 define <vscale x 2 x i16> @test_sf_vc_v_vvv_e16mf2(<vscale x 2 x i16> %vd, <vscale x 2 x i16> %vs2, <vscale x 2 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -705,7 +705,7 @@ declare <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvv.nxv2i16.iXLen.nxv2i16.iXLen(i
 define <vscale x 4 x i16> @test_sf_vc_v_vvv_e16m1(<vscale x 4 x i16> %vd, <vscale x 4 x i16> %vs2, <vscale x 4 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -718,7 +718,7 @@ declare <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvv.nxv4i16.iXLen.nxv4i16.iXLen(i
 define <vscale x 8 x i16> @test_sf_vc_v_vvv_e16m2(<vscale x 8 x i16> %vd, <vscale x 8 x i16> %vs2, <vscale x 8 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -731,7 +731,7 @@ declare <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvv.nxv8i16.iXLen.nxv8i16.iXLen(i
 define <vscale x 16 x i16> @test_sf_vc_v_vvv_e16m4(<vscale x 16 x i16> %vd, <vscale x 16 x i16> %vs2, <vscale x 16 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -745,7 +745,7 @@ define <vscale x 32 x i16> @test_sf_vc_v_vvv_e16m8(<vscale x 32 x i16> %vd, <vsc
 ; CHECK-LABEL: test_sf_vc_v_vvv_e16m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -758,7 +758,7 @@ declare <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvv.nxv32i16.iXLen.nxv32i16.iXLe
 define <vscale x 1 x i32> @test_sf_vc_v_vvv_e32mf2(<vscale x 1 x i32> %vd, <vscale x 1 x i32> %vs2, <vscale x 1 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -771,7 +771,7 @@ declare <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvv.nxv1i32.iXLen.nxv1i32.iXLen(i
 define <vscale x 2 x i32> @test_sf_vc_v_vvv_e32m1(<vscale x 2 x i32> %vd, <vscale x 2 x i32> %vs2, <vscale x 2 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -784,7 +784,7 @@ declare <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvv.nxv2i32.iXLen.nxv2i32.iXLen(i
 define <vscale x 4 x i32> @test_sf_vc_v_vvv_e32m2(<vscale x 4 x i32> %vd, <vscale x 4 x i32> %vs2, <vscale x 4 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -797,7 +797,7 @@ declare <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvv.nxv4i32.iXLen.nxv4i32.iXLen(i
 define <vscale x 8 x i32> @test_sf_vc_v_vvv_e32m4(<vscale x 8 x i32> %vd, <vscale x 8 x i32> %vs2, <vscale x 8 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -811,7 +811,7 @@ define <vscale x 16 x i32> @test_sf_vc_v_vvv_e32m8(<vscale x 16 x i32> %vd, <vsc
 ; CHECK-LABEL: test_sf_vc_v_vvv_e32m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -824,7 +824,7 @@ declare <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvv.nxv16i32.iXLen.nxv16i32.iXLe
 define <vscale x 1 x i64> @test_sf_vc_v_vvv_e64m1(<vscale x 1 x i64> %vd, <vscale x 1 x i64> %vs2, <vscale x 1 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -837,7 +837,7 @@ declare <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvv.nxv1i64.iXLen.nxv1i64.iXLen(i
 define <vscale x 2 x i64> @test_sf_vc_v_vvv_e64m2(<vscale x 2 x i64> %vd, <vscale x 2 x i64> %vs2, <vscale x 2 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -850,7 +850,7 @@ declare <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvv.nxv2i64.iXLen.nxv2i64.iXLen(i
 define <vscale x 4 x i64> @test_sf_vc_v_vvv_e64m4(<vscale x 4 x i64> %vd, <vscale x 4 x i64> %vs2, <vscale x 4 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -864,7 +864,7 @@ define <vscale x 8 x i64> @test_sf_vc_v_vvv_e64m8(<vscale x 8 x i64> %vd, <vscal
 ; CHECK-LABEL: test_sf_vc_v_vvv_e64m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -1111,7 +1111,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.i32.nxv16i32.iXLen.iXLen(iXLen, <vscale x
 define <vscale x 1 x i8> @test_sf_vc_v_xvv_se_e8mf8(<vscale x 1 x i8> %vd, <vscale x 1 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1124,7 +1124,7 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv1i8.iXLen.i8.iXLen(iXLen
 define <vscale x 2 x i8> @test_sf_vc_v_xvv_se_e8mf4(<vscale x 2 x i8> %vd, <vscale x 2 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1137,7 +1137,7 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv2i8.iXLen.i8.iXLen(iXLen
 define <vscale x 4 x i8> @test_sf_vc_v_xvv_se_e8mf2(<vscale x 4 x i8> %vd, <vscale x 4 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1150,7 +1150,7 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv4i8.iXLen.i8.iXLen(iXLen
 define <vscale x 8 x i8> @test_sf_vc_v_xvv_se_e8m1(<vscale x 8 x i8> %vd, <vscale x 8 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1163,7 +1163,7 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv8i8.iXLen.i8.iXLen(iXLen
 define <vscale x 16 x i8> @test_sf_vc_v_xvv_se_e8m2(<vscale x 16 x i8> %vd, <vscale x 16 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1176,7 +1176,7 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv16i8.iXLen.i8.iXLen(iXL
 define <vscale x 32 x i8> @test_sf_vc_v_xvv_se_e8m4(<vscale x 32 x i8> %vd, <vscale x 32 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1189,7 +1189,7 @@ declare <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv32i8.iXLen.i8.iXLen(iXL
 define <vscale x 64 x i8> @test_sf_vc_v_xvv_se_e8m8(<vscale x 64 x i8> %vd, <vscale x 64 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1202,7 +1202,7 @@ declare <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv64i8.iXLen.i8.iXLen(iXL
 define <vscale x 1 x i16> @test_sf_vc_v_xvv_se_e16mf4(<vscale x 1 x i16> %vd, <vscale x 1 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1215,7 +1215,7 @@ declare <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv1i16.iXLen.i16.iXLen(iX
 define <vscale x 2 x i16> @test_sf_vc_v_xvv_se_e16mf2(<vscale x 2 x i16> %vd, <vscale x 2 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1228,7 +1228,7 @@ declare <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv2i16.iXLen.i16.iXLen(iX
 define <vscale x 4 x i16> @test_sf_vc_v_xvv_se_e16m1(<vscale x 4 x i16> %vd, <vscale x 4 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1241,7 +1241,7 @@ declare <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv4i16.iXLen.i16.iXLen(iX
 define <vscale x 8 x i16> @test_sf_vc_v_xvv_se_e16m2(<vscale x 8 x i16> %vd, <vscale x 8 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1254,7 +1254,7 @@ declare <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv8i16.iXLen.i16.iXLen(iX
 define <vscale x 16 x i16> @test_sf_vc_v_xvv_se_e16m4(<vscale x 16 x i16> %vd, <vscale x 16 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1267,7 +1267,7 @@ declare <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv16i16.iXLen.i16.iXLen(
 define <vscale x 32 x i16> @test_sf_vc_v_xvv_se_e16m8(<vscale x 32 x i16> %vd, <vscale x 32 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1280,7 +1280,7 @@ declare <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv32i16.iXLen.i16.iXLen(
 define <vscale x 1 x i32> @test_sf_vc_v_xvv_se_e32mf2(<vscale x 1 x i32> %vd, <vscale x 1 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1293,7 +1293,7 @@ declare <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv1i32.iXLen.i32.iXLen(iX
 define <vscale x 2 x i32> @test_sf_vc_v_xvv_se_e32m1(<vscale x 2 x i32> %vd, <vscale x 2 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1306,7 +1306,7 @@ declare <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv2i32.iXLen.i32.iXLen(iX
 define <vscale x 4 x i32> @test_sf_vc_v_xvv_se_e32m2(<vscale x 4 x i32> %vd, <vscale x 4 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1319,7 +1319,7 @@ declare <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv4i32.iXLen.i32.iXLen(iX
 define <vscale x 8 x i32> @test_sf_vc_v_xvv_se_e32m4(<vscale x 8 x i32> %vd, <vscale x 8 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1332,7 +1332,7 @@ declare <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv8i32.iXLen.i32.iXLen(iX
 define <vscale x 16 x i32> @test_sf_vc_v_xvv_se_e32m8(<vscale x 16 x i32> %vd, <vscale x 16 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1345,7 +1345,7 @@ declare <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv16i32.iXLen.i32.iXLen(
 define <vscale x 1 x i8> @test_sf_vc_v_xvv_e8mf8(<vscale x 1 x i8> %vd, <vscale x 1 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1358,7 +1358,7 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xvv.nxv1i8.iXLen.i8.iXLen(iXLen, <
 define <vscale x 2 x i8> @test_sf_vc_v_xvv_e8mf4(<vscale x 2 x i8> %vd, <vscale x 2 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1371,7 +1371,7 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xvv.nxv2i8.iXLen.i8.iXLen(iXLen, <
 define <vscale x 4 x i8> @test_sf_vc_v_xvv_e8mf2(<vscale x 4 x i8> %vd, <vscale x 4 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1384,7 +1384,7 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xvv.nxv4i8.iXLen.i8.iXLen(iXLen, <
 define <vscale x 8 x i8> @test_sf_vc_v_xvv_e8m1(<vscale x 8 x i8> %vd, <vscale x 8 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1397,7 +1397,7 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xvv.nxv8i8.iXLen.i8.iXLen(iXLen, <
 define <vscale x 16 x i8> @test_sf_vc_v_xvv_e8m2(<vscale x 16 x i8> %vd, <vscale x 16 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1410,7 +1410,7 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xvv.nxv16i8.iXLen.i8.iXLen(iXLen,
 define <vscale x 32 x i8> @test_sf_vc_v_xvv_e8m4(<vscale x 32 x i8> %vd, <vscale x 32 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1423,7 +1423,7 @@ declare <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xvv.nxv32i8.iXLen.i8.iXLen(iXLen,
 define <vscale x 64 x i8> @test_sf_vc_v_xvv_e8m8(<vscale x 64 x i8> %vd, <vscale x 64 x i8> %vs2, i8 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1436,7 +1436,7 @@ declare <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xvv.nxv64i8.iXLen.i8.iXLen(iXLen,
 define <vscale x 1 x i16> @test_sf_vc_v_xvv_e16mf4(<vscale x 1 x i16> %vd, <vscale x 1 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1449,7 +1449,7 @@ declare <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvv.nxv1i16.iXLen.i16.iXLen(iXLen
 define <vscale x 2 x i16> @test_sf_vc_v_xvv_e16mf2(<vscale x 2 x i16> %vd, <vscale x 2 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1462,7 +1462,7 @@ declare <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvv.nxv2i16.iXLen.i16.iXLen(iXLen
 define <vscale x 4 x i16> @test_sf_vc_v_xvv_e16m1(<vscale x 4 x i16> %vd, <vscale x 4 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1475,7 +1475,7 @@ declare <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvv.nxv4i16.iXLen.i16.iXLen(iXLen
 define <vscale x 8 x i16> @test_sf_vc_v_xvv_e16m2(<vscale x 8 x i16> %vd, <vscale x 8 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1488,7 +1488,7 @@ declare <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvv.nxv8i16.iXLen.i16.iXLen(iXLen
 define <vscale x 16 x i16> @test_sf_vc_v_xvv_e16m4(<vscale x 16 x i16> %vd, <vscale x 16 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1501,7 +1501,7 @@ declare <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvv.nxv16i16.iXLen.i16.iXLen(iXL
 define <vscale x 32 x i16> @test_sf_vc_v_xvv_e16m8(<vscale x 32 x i16> %vd, <vscale x 32 x i16> %vs2, i16 zeroext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1514,7 +1514,7 @@ declare <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvv.nxv32i16.iXLen.i16.iXLen(iXL
 define <vscale x 1 x i32> @test_sf_vc_v_xvv_e32mf2(<vscale x 1 x i32> %vd, <vscale x 1 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1527,7 +1527,7 @@ declare <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvv.nxv1i32.iXLen.i32.iXLen(iXLen
 define <vscale x 2 x i32> @test_sf_vc_v_xvv_e32m1(<vscale x 2 x i32> %vd, <vscale x 2 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1540,7 +1540,7 @@ declare <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvv.nxv2i32.iXLen.i32.iXLen(iXLen
 define <vscale x 4 x i32> @test_sf_vc_v_xvv_e32m2(<vscale x 4 x i32> %vd, <vscale x 4 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1553,7 +1553,7 @@ declare <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvv.nxv4i32.iXLen.i32.iXLen(iXLen
 define <vscale x 8 x i32> @test_sf_vc_v_xvv_e32m4(<vscale x 8 x i32> %vd, <vscale x 8 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1566,7 +1566,7 @@ declare <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvv.nxv8i32.iXLen.i32.iXLen(iXLen
 define <vscale x 16 x i32> @test_sf_vc_v_xvv_e32m8(<vscale x 16 x i32> %vd, <vscale x 16 x i32> %vs2, i32 signext %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_xvv_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1865,7 +1865,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv8i64.iXLen.iXLen(iXLen, <vscale x
 define <vscale x 1 x i8> @test_sf_vc_v_ivv_se_e8mf8(<vscale x 1 x i8> %vd, <vscale x 1 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1878,7 +1878,7 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv1i8.iXLen.iXLen.iXLen(iX
 define <vscale x 2 x i8> @test_sf_vc_v_ivv_se_e8mf4(<vscale x 2 x i8> %vd, <vscale x 2 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1891,7 +1891,7 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv2i8.iXLen.iXLen.iXLen(iX
 define <vscale x 4 x i8> @test_sf_vc_v_ivv_se_e8mf2(<vscale x 4 x i8> %vd, <vscale x 4 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1904,7 +1904,7 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv4i8.iXLen.iXLen.iXLen(iX
 define <vscale x 8 x i8> @test_sf_vc_v_ivv_se_e8m1(<vscale x 8 x i8> %vd, <vscale x 8 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1917,7 +1917,7 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv8i8.iXLen.iXLen.iXLen(iX
 define <vscale x 16 x i8> @test_sf_vc_v_ivv_se_e8m2(<vscale x 16 x i8> %vd, <vscale x 16 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1930,7 +1930,7 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv16i8.iXLen.iXLen.iXLen(
 define <vscale x 32 x i8> @test_sf_vc_v_ivv_se_e8m4(<vscale x 32 x i8> %vd, <vscale x 32 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1943,7 +1943,7 @@ declare <vscale x 32 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv32i8.iXLen.iXLen.iXLen(
 define <vscale x 64 x i8> @test_sf_vc_v_ivv_se_e8m8(<vscale x 64 x i8> %vd, <vscale x 64 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1956,7 +1956,7 @@ declare <vscale x 64 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv64i8.iXLen.iXLen.iXLen(
 define <vscale x 1 x i16> @test_sf_vc_v_ivv_se_e16mf4(<vscale x 1 x i16> %vd, <vscale x 1 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1969,7 +1969,7 @@ declare <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv1i16.iXLen.iXLen.iXLen(
 define <vscale x 2 x i16> @test_sf_vc_v_ivv_se_e16mf2(<vscale x 2 x i16> %vd, <vscale x 2 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1982,7 +1982,7 @@ declare <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv2i16.iXLen.iXLen.iXLen(
 define <vscale x 4 x i16> @test_sf_vc_v_ivv_se_e16m1(<vscale x 4 x i16> %vd, <vscale x 4 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -1995,7 +1995,7 @@ declare <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv4i16.iXLen.iXLen.iXLen(
 define <vscale x 8 x i16> @test_sf_vc_v_ivv_se_e16m2(<vscale x 8 x i16> %vd, <vscale x 8 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2008,7 +2008,7 @@ declare <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv8i16.iXLen.iXLen.iXLen(
 define <vscale x 16 x i16> @test_sf_vc_v_ivv_se_e16m4(<vscale x 16 x i16> %vd, <vscale x 16 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2021,7 +2021,7 @@ declare <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv16i16.iXLen.iXLen.iXLe
 define <vscale x 32 x i16> @test_sf_vc_v_ivv_se_e16m8(<vscale x 32 x i16> %vd, <vscale x 32 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2034,7 +2034,7 @@ declare <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv32i16.iXLen.iXLen.iXLe
 define <vscale x 1 x i32> @test_sf_vc_v_ivv_se_e32mf2(<vscale x 1 x i32> %vd, <vscale x 1 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2047,7 +2047,7 @@ declare <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv1i32.iXLen.iXLen.iXLen(
 define <vscale x 2 x i32> @test_sf_vc_v_ivv_se_e32m1(<vscale x 2 x i32> %vd, <vscale x 2 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2060,7 +2060,7 @@ declare <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv2i32.iXLen.iXLen.iXLen(
 define <vscale x 4 x i32> @test_sf_vc_v_ivv_se_e32m2(<vscale x 4 x i32> %vd, <vscale x 4 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2073,7 +2073,7 @@ declare <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv4i32.iXLen.iXLen.iXLen(
 define <vscale x 8 x i32> @test_sf_vc_v_ivv_se_e32m4(<vscale x 8 x i32> %vd, <vscale x 8 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2086,7 +2086,7 @@ declare <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv8i32.iXLen.iXLen.iXLen(
 define <vscale x 16 x i32> @test_sf_vc_v_ivv_se_e32m8(<vscale x 16 x i32> %vd, <vscale x 16 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2099,7 +2099,7 @@ declare <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv16i32.iXLen.iXLen.iXLe
 define <vscale x 1 x i64> @test_sf_vc_v_ivv_se_e64m1(<vscale x 1 x i64> %vd, <vscale x 1 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2112,7 +2112,7 @@ declare <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv1i64.iXLen.iXLen.iXLen(
 define <vscale x 2 x i64> @test_sf_vc_v_ivv_se_e64m2(<vscale x 2 x i64> %vd, <vscale x 2 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2125,7 +2125,7 @@ declare <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv2i64.iXLen.iXLen.iXLen(
 define <vscale x 4 x i64> @test_sf_vc_v_ivv_se_e64m4(<vscale x 4 x i64> %vd, <vscale x 4 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2138,7 +2138,7 @@ declare <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv4i64.iXLen.iXLen.iXLen(
 define <vscale x 8 x i64> @test_sf_vc_v_ivv_se_e64m8(<vscale x 8 x i64> %vd, <vscale x 8 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_se_e64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2151,7 +2151,7 @@ declare <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv8i64.iXLen.iXLen.iXLen(
 define <vscale x 1 x i8> @test_sf_vc_v_ivv_e8mf8(<vscale x 1 x i8> %vd, <vscale x 1 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2164,7 +2164,7 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vc.v.ivv.nxv1i8.iXLen.iXLen.iXLen(iXLen
 define <vscale x 2 x i8> @test_sf_vc_v_ivv_e8mf4(<vscale x 2 x i8> %vd, <vscale x 2 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2177,7 +2177,7 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vc.v.ivv.nxv2i8.iXLen.iXLen.iXLen(iXLen
 define <vscale x 4 x i8> @test_sf_vc_v_ivv_e8mf2(<vscale x 4 x i8> %vd, <vscale x 4 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2190,7 +2190,7 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vc.v.ivv.nxv4i8.iXLen.iXLen.iXLen(iXLen
 define <vscale x 8 x i8> @test_sf_vc_v_ivv_e8m1(<vscale x 8 x i8> %vd, <vscale x 8 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2203,7 +2203,7 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vc.v.ivv.nxv8i8.iXLen.iXLen.iXLen(iXLen
 define <vscale x 16 x i8> @test_sf_vc_v_ivv_e8m2(<vscale x 16 x i8> %vd, <vscale x 16 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2216,7 +2216,7 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vc.v.ivv.nxv16i8.iXLen.iXLen.iXLen(iXL
 define <vscale x 32 x i8> @test_sf_vc_v_ivv_e8m4(<vscale x 32 x i8> %vd, <vscale x 32 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2229,7 +2229,7 @@ declare <vscale x 32 x i8> @llvm.riscv.sf.vc.v.ivv.nxv32i8.iXLen.iXLen.iXLen(iXL
 define <vscale x 64 x i8> @test_sf_vc_v_ivv_e8m8(<vscale x 64 x i8> %vd, <vscale x 64 x i8> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2242,7 +2242,7 @@ declare <vscale x 64 x i8> @llvm.riscv.sf.vc.v.ivv.nxv64i8.iXLen.iXLen.iXLen(iXL
 define <vscale x 1 x i16> @test_sf_vc_v_ivv_e16mf4(<vscale x 1 x i16> %vd, <vscale x 1 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2255,7 +2255,7 @@ declare <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivv.nxv1i16.iXLen.iXLen.iXLen(iXL
 define <vscale x 2 x i16> @test_sf_vc_v_ivv_e16mf2(<vscale x 2 x i16> %vd, <vscale x 2 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2268,7 +2268,7 @@ declare <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivv.nxv2i16.iXLen.iXLen.iXLen(iXL
 define <vscale x 4 x i16> @test_sf_vc_v_ivv_e16m1(<vscale x 4 x i16> %vd, <vscale x 4 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2281,7 +2281,7 @@ declare <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivv.nxv4i16.iXLen.iXLen.iXLen(iXL
 define <vscale x 8 x i16> @test_sf_vc_v_ivv_e16m2(<vscale x 8 x i16> %vd, <vscale x 8 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2294,7 +2294,7 @@ declare <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivv.nxv8i16.iXLen.iXLen.iXLen(iXL
 define <vscale x 16 x i16> @test_sf_vc_v_ivv_e16m4(<vscale x 16 x i16> %vd, <vscale x 16 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2307,7 +2307,7 @@ declare <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivv.nxv16i16.iXLen.iXLen.iXLen(i
 define <vscale x 32 x i16> @test_sf_vc_v_ivv_e16m8(<vscale x 32 x i16> %vd, <vscale x 32 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2320,7 +2320,7 @@ declare <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivv.nxv32i16.iXLen.iXLen.iXLen(i
 define <vscale x 1 x i32> @test_sf_vc_v_ivv_e32mf2(<vscale x 1 x i32> %vd, <vscale x 1 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2333,7 +2333,7 @@ declare <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivv.nxv1i32.iXLen.iXLen.iXLen(iXL
 define <vscale x 2 x i32> @test_sf_vc_v_ivv_e32m1(<vscale x 2 x i32> %vd, <vscale x 2 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2346,7 +2346,7 @@ declare <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivv.nxv2i32.iXLen.iXLen.iXLen(iXL
 define <vscale x 4 x i32> @test_sf_vc_v_ivv_e32m2(<vscale x 4 x i32> %vd, <vscale x 4 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2359,7 +2359,7 @@ declare <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivv.nxv4i32.iXLen.iXLen.iXLen(iXL
 define <vscale x 8 x i32> @test_sf_vc_v_ivv_e32m4(<vscale x 8 x i32> %vd, <vscale x 8 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2372,7 +2372,7 @@ declare <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivv.nxv8i32.iXLen.iXLen.iXLen(iXL
 define <vscale x 16 x i32> @test_sf_vc_v_ivv_e32m8(<vscale x 16 x i32> %vd, <vscale x 16 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2385,7 +2385,7 @@ declare <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivv.nxv16i32.iXLen.iXLen.iXLen(i
 define <vscale x 1 x i64> @test_sf_vc_v_ivv_e64m1(<vscale x 1 x i64> %vd, <vscale x 1 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2398,7 +2398,7 @@ declare <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivv.nxv1i64.iXLen.iXLen.iXLen(iXL
 define <vscale x 2 x i64> @test_sf_vc_v_ivv_e64m2(<vscale x 2 x i64> %vd, <vscale x 2 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2411,7 +2411,7 @@ declare <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivv.nxv2i64.iXLen.iXLen.iXLen(iXL
 define <vscale x 4 x i64> @test_sf_vc_v_ivv_e64m4(<vscale x 4 x i64> %vd, <vscale x 4 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2424,7 +2424,7 @@ declare <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivv.nxv4i64.iXLen.iXLen.iXLen(iXL
 define <vscale x 8 x i64> @test_sf_vc_v_ivv_e64m8(<vscale x 8 x i64> %vd, <vscale x 8 x i64> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_ivv_e64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 10
 ; CHECK-NEXT:    ret
 entry:
@@ -2450,7 +2450,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv1f16.nxv1i16.nxv1i16.iXLen(iXLen,
 define <vscale x 1 x half> @test_sf_vc_fv_fvv_se_e16mf4(<vscale x 1 x half> %vd, <vscale x 1 x i16> %vs2, <vscale x 1 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2476,7 +2476,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv2f16.nxv2i16.nxv2i16.iXLen(iXLen,
 define <vscale x 2 x half> @test_sf_vc_fv_fvv_se_e16mf2(<vscale x 2 x half> %vd, <vscale x 2 x i16> %vs2, <vscale x 2 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2502,7 +2502,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv4f16.nxv4i16.nxv4i16.iXLen(iXLen,
 define <vscale x 4 x half> @test_sf_vc_fv_fvv_se_e16m1(<vscale x 4 x half> %vd, <vscale x 4 x i16> %vs2, <vscale x 4 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2528,7 +2528,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv8f16.nxv8i16.nxv8i16.iXLen(iXLen,
 define <vscale x 8 x half> @test_sf_vc_fv_fvv_se_e16m2(<vscale x 8 x half> %vd, <vscale x 8 x i16> %vs2, <vscale x 8 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -2554,7 +2554,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv16f16.nxv16i16.nxv16i16.iXLen(iXL
 define <vscale x 16 x half> @test_sf_vc_fv_fvv_se_e16m4(<vscale x 16 x half> %vd, <vscale x 16 x i16> %vs2, <vscale x 16 x i16> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -2582,7 +2582,7 @@ define <vscale x 32 x half> @test_sf_vc_fv_fvv_se_e16m8(<vscale x 32 x half> %vd
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -2608,7 +2608,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv1f32.nxv1i32.nxv1i32.iXLen(iXLen,
 define <vscale x 1 x float> @test_sf_vc_fv_fvv_se_e32mf2(<vscale x 1 x float> %vd, <vscale x 1 x i32> %vs2, <vscale x 1 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2634,7 +2634,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv2f32.nxv2i32.nxv2i32.iXLen(iXLen,
 define <vscale x 2 x float> @test_sf_vc_fv_fvv_se_e32m1(<vscale x 2 x float> %vd, <vscale x 2 x i32> %vs2, <vscale x 2 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2660,7 +2660,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv4f32.nxv4i32.nxv4i32.iXLen(iXLen,
 define <vscale x 4 x float> @test_sf_vc_fv_fvv_se_e32m2(<vscale x 4 x float> %vd, <vscale x 4 x i32> %vs2, <vscale x 4 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -2686,7 +2686,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv8f32.nxv8i32.nxv8i32.iXLen(iXLen,
 define <vscale x 8 x float> @test_sf_vc_fv_fvv_se_e32m4(<vscale x 8 x float> %vd, <vscale x 8 x i32> %vs2, <vscale x 8 x i32> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -2714,7 +2714,7 @@ define <vscale x 16 x float> @test_sf_vc_fv_fvv_se_e32m8(<vscale x 16 x float> %
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -2740,7 +2740,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv1f64.nxv1i64.nxv1i64.iXLen(iXLen,
 define <vscale x 1 x double> @test_sf_vc_fv_fvv_se_e64m1(<vscale x 1 x double> %vd, <vscale x 1 x i64> %vs2, <vscale x 1 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -2766,7 +2766,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv2f64.nxv2i64.nxv2i64.iXLen(iXLen,
 define <vscale x 2 x double> @test_sf_vc_fv_fvv_se_e64m2(<vscale x 2 x double> %vd, <vscale x 2 x i64> %vs2, <vscale x 2 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v10, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -2792,7 +2792,7 @@ declare void @llvm.riscv.sf.vc.vvv.se.iXLen.nxv4f64.nxv4i64.nxv4i64.iXLen(iXLen,
 define <vscale x 4 x double> @test_sf_vc_fv_fvv_se_e64m4(<vscale x 4 x double> %vd, <vscale x 4 x i64> %vs2, <vscale x 4 x i64> %vs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v12, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -2820,7 +2820,7 @@ define <vscale x 8 x double> @test_sf_vc_fv_fvv_se_e64m8(<vscale x 8 x double> %
 ; CHECK-LABEL: test_sf_vc_fv_fvv_se_e64m8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.vvv 3, v8, v16, v24
 ; CHECK-NEXT:    ret
 entry:
@@ -2846,7 +2846,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv1f16.nxv1i16.i16.iXLen(iXLen, <vs
 define <vscale x 1 x half> @test_sf_vc_v_fvvx_se_e16mf4(<vscale x 1 x half> %vd, <vscale x 1 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2872,7 +2872,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv2f16.nxv2i16.i16.iXLen(iXLen, <vs
 define <vscale x 2 x half> @test_sf_vc_v_fvvx_se_e16mf2(<vscale x 2 x half> %vd, <vscale x 2 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2898,7 +2898,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv4f16.nxv4i16.i16.iXLen(iXLen, <vs
 define <vscale x 4 x half> @test_sf_vc_v_fvvx_se_e16m1(<vscale x 4 x half> %vd, <vscale x 4 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2924,7 +2924,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv8f16.nxv8i16.i16.iXLen(iXLen, <vs
 define <vscale x 8 x half> @test_sf_vc_v_fvvx_se_e16m2(<vscale x 8 x half> %vd, <vscale x 8 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2950,7 +2950,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv16f16.nxv16i16.i16.iXLen(iXLen, <
 define <vscale x 16 x half> @test_sf_vc_v_fvvx_se_e16m4(<vscale x 16 x half> %vd, <vscale x 16 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2976,7 +2976,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv32f16.nxv32i16.i16.iXLen(iXLen, <
 define <vscale x 32 x half> @test_sf_vc_v_fvvx_se_e16m8(<vscale x 32 x half> %vd, <vscale x 32 x i16> %vs2, i16 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3002,7 +3002,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv1f32.nxv1i32.i32.iXLen(iXLen, <vs
 define <vscale x 1 x float> @test_sf_vc_v_fvvx_se_e32mf2(<vscale x 1 x float> %vd, <vscale x 1 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3028,7 +3028,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv2f32.nxv2i32.i32.iXLen(iXLen, <vs
 define <vscale x 2 x float> @test_sf_vc_v_fvvx_se_e32m1(<vscale x 2 x float> %vd, <vscale x 2 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v9, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3054,7 +3054,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv4f32.nxv4i32.i32.iXLen(iXLen, <vs
 define <vscale x 4 x float> @test_sf_vc_v_fvvx_se_e32m2(<vscale x 4 x float> %vd, <vscale x 4 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v10, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3080,7 +3080,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv8f32.nxv8i32.i32.iXLen(iXLen, <vs
 define <vscale x 8 x float> @test_sf_vc_v_fvvx_se_e32m4(<vscale x 8 x float> %vd, <vscale x 8 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v12, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3106,7 +3106,7 @@ declare void @llvm.riscv.sf.vc.xvv.se.iXLen.nxv16f32.nxv16i32.i32.iXLen(iXLen, <
 define <vscale x 16 x float> @test_sf_vc_v_fvvx_se_e32m8(<vscale x 16 x float> %vd, <vscale x 16 x i32> %vs2, i32 %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_v_fvvx_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.xvv 3, v8, v16, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -3132,7 +3132,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv1f16.nxv1i16.iXLen.iXLen(iXLen, <
 define <vscale x 1 x half> @test_sf_vc_fv_fvvi_se_e16mf4(<vscale x 1 x half> %vd, <vscale x 1 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3158,7 +3158,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv2f16.nxv2i16.iXLen.iXLen(iXLen, <
 define <vscale x 2 x half> @test_sf_vc_fv_fvvi_se_e16mf2(<vscale x 2 x half> %vd, <vscale x 2 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3184,7 +3184,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv4f16.nxv4i16.iXLen.iXLen(iXLen, <
 define <vscale x 4 x half> @test_sf_vc_fv_fvvi_se_e16m1(<vscale x 4 x half> %vd, <vscale x 4 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3210,7 +3210,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv8f16.nxv8i16.iXLen.iXLen(iXLen, <
 define <vscale x 8 x half> @test_sf_vc_fv_fvvi_se_e16m2(<vscale x 8 x half> %vd, <vscale x 8 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3236,7 +3236,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv16f16.nxv16i16.iXLen.iXLen(iXLen,
 define <vscale x 16 x half> @test_sf_vc_fv_fvvi_se_e16m4(<vscale x 16 x half> %vd, <vscale x 16 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3262,7 +3262,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv32f16.nxv32i16.iXLen.iXLen(iXLen,
 define <vscale x 32 x half> @test_sf_vc_fv_fvvi_se_e16m8(<vscale x 32 x half> %vd, <vscale x 32 x i16> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3288,7 +3288,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv1f32.nxv1i32.iXLen.iXLen(iXLen, <
 define <vscale x 1 x float> @test_sf_vc_fv_fvvi_se_e32mf2(<vscale x 1 x float> %vd, <vscale x 1 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3314,7 +3314,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv2f32.nxv2i32.iXLen.iXLen(iXLen, <
 define <vscale x 2 x float> @test_sf_vc_fv_fvvi_se_e32m1(<vscale x 2 x float> %vd, <vscale x 2 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v9, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3340,7 +3340,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv4f32.nxv4i32.iXLen.iXLen(iXLen, <
 define <vscale x 4 x float> @test_sf_vc_fv_fvvi_se_e32m2(<vscale x 4 x float> %vd, <vscale x 4 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v10, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3366,7 +3366,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv8f32.nxv8i32.iXLen.iXLen(iXLen, <
 define <vscale x 8 x float> @test_sf_vc_fv_fvvi_se_e32m4(<vscale x 8 x float> %vd, <vscale x 8 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v12, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3392,7 +3392,7 @@ declare void @llvm.riscv.sf.vc.ivv.se.iXLen.nxv16f32.nxv16i32.iXLen.iXLen(iXLen,
 define <vscale x 16 x float> @test_sf_vc_fv_fvvi_se_e32m8(<vscale x 16 x float> %vd, <vscale x 16 x i32> %vs2, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvi_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.ivv 3, v8, v16, 3
 ; CHECK-NEXT:    ret
 entry:
@@ -3418,7 +3418,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv1f16.nxv1i16.f16.iXLen(iXLen, <vs
 define <vscale x 1 x half> @test_sf_vc_fv_fvvf_se_e16mf4(<vscale x 1 x half> %vd, <vscale x 1 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3444,7 +3444,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv2f16.nxv2i16.f16.iXLen(iXLen, <vs
 define <vscale x 2 x half> @test_sf_vc_fv_fvvf_se_e16mf2(<vscale x 2 x half> %vd, <vscale x 2 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3470,7 +3470,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv4f16.nxv4i16.f16.iXLen(iXLen, <vs
 define <vscale x 4 x half> @test_sf_vc_fv_fvvf_se_e16m1(<vscale x 4 x half> %vd, <vscale x 4 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3496,7 +3496,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv8f16.nxv8i16.f16.iXLen(iXLen, <vs
 define <vscale x 8 x half> @test_sf_vc_fv_fvvf_se_e16m2(<vscale x 8 x half> %vd, <vscale x 8 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v10, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3522,7 +3522,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv16f16.nxv16i16.f16.iXLen(iXLen, <
 define <vscale x 16 x half> @test_sf_vc_fv_fvvf_se_e16m4(<vscale x 16 x half> %vd, <vscale x 16 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v12, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3548,7 +3548,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv32f16.nxv32i16.f16.iXLen(iXLen, <
 define <vscale x 32 x half> @test_sf_vc_fv_fvvf_se_e16m8(<vscale x 32 x half> %vd, <vscale x 32 x i16> %vs2, half %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v16, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3574,7 +3574,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv1f32.nxv1i32.f32.iXLen(iXLen, <vs
 define <vscale x 1 x float> @test_sf_vc_fv_fvvf_se_e32mf2(<vscale x 1 x float> %vd, <vscale x 1 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3600,7 +3600,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv2f32.nxv2i32.f32.iXLen(iXLen, <vs
 define <vscale x 2 x float> @test_sf_vc_fv_fvvf_se_e32m1(<vscale x 2 x float> %vd, <vscale x 2 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v9, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3626,7 +3626,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv4f32.nxv4i32.f32.iXLen(iXLen, <vs
 define <vscale x 4 x float> @test_sf_vc_fv_fvvf_se_e32m2(<vscale x 4 x float> %vd, <vscale x 4 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v10, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3652,7 +3652,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv8f32.nxv8i32.f32.iXLen(iXLen, <vs
 define <vscale x 8 x float> @test_sf_vc_fv_fvvf_se_e32m4(<vscale x 8 x float> %vd, <vscale x 8 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v12, fa0
 ; CHECK-NEXT:    ret
 entry:
@@ -3678,7 +3678,7 @@ declare void @llvm.riscv.sf.vc.fvv.se.iXLen.nxv16f32.nxv16i32.f32.iXLen(iXLen, <
 define <vscale x 16 x float> @test_sf_vc_fv_fvvf_se_e32m8(<vscale x 16 x float> %vd, <vscale x 16 x i32> %vs2, float %rs1, iXLen %vl) {
 ; CHECK-LABEL: test_sf_vc_fv_fvvf_se_e32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, tu, ma
 ; CHECK-NEXT:    sf.vc.v.fvv 1, v8, v16, fa0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
index f4c3b072c8fe1e71a1bbf97af3c42a6ea247541b..005a01bf1000a0a57baa869507f8b08e5e2d2fbb 100644
--- a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
+++ b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
@@ -70,10 +70,10 @@ define i128 @cmovcc128(i64 signext %a, i128 %b, i128 %c) nounwind {
 ; RV32I-NEXT:    addi a3, a3, 12
 ; RV32I-NEXT:  .LBB1_10: # %entry
 ; RV32I-NEXT:    lw a1, 0(a3)
-; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    sw a6, 8(a0)
-; RV32I-NEXT:    sw a5, 4(a0)
 ; RV32I-NEXT:    sw a2, 0(a0)
+; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw a6, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmovcc128:
@@ -153,10 +153,10 @@ define i128 @cmov128(i1 %a, i128 %b, i128 %c) nounwind {
 ; RV32I-NEXT:    addi a2, a2, 12
 ; RV32I-NEXT:  .LBB3_10: # %entry
 ; RV32I-NEXT:    lw a1, 0(a2)
-; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    sw a6, 8(a0)
-; RV32I-NEXT:    sw a5, 4(a0)
 ; RV32I-NEXT:    sw a4, 0(a0)
+; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw a6, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmov128:
@@ -221,8 +221,8 @@ define double @cmovdouble(i1 %a, double %b, double %c) nounwind {
 ; RV32I-NEXT:    sw a3, 8(sp)
 ; RV32I-NEXT:    sw a4, 12(sp)
 ; RV32I-NEXT:    fld fa5, 8(sp)
-; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    andi a0, a0, 1
+; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB5_2
 ; RV32I-NEXT:  # %bb.1:
diff --git a/llvm/test/CodeGen/RISCV/shifts.ll b/llvm/test/CodeGen/RISCV/shifts.ll
index dcc99ebaa5514ba4fd50b9def8d434d78e72eb80..2bec1fca5c43b48d1e082f8dbba60c014f13f3e9 100644
--- a/llvm/test/CodeGen/RISCV/shifts.ll
+++ b/llvm/test/CodeGen/RISCV/shifts.ll
@@ -157,14 +157,14 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a4, 4(a1)
 ; RV32I-NEXT:    lw a5, 8(a1)
 ; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw a1, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
 ; RV32I-NEXT:    srli a1, a2, 3
 ; RV32I-NEXT:    andi a1, a1, 12
 ; RV32I-NEXT:    mv a3, sp
@@ -188,10 +188,10 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sll a6, a6, a7
 ; RV32I-NEXT:    or a5, a5, a6
 ; RV32I-NEXT:    srl a1, a1, a2
-; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    sw a5, 8(a0)
-; RV32I-NEXT:    sw a4, 4(a0)
 ; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a4, 4(a0)
+; RV32I-NEXT:    sw a5, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -221,20 +221,20 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV32I-LABEL: ashr128:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    lw a3, 8(a1)
-; RV32I-NEXT:    lw a4, 12(a1)
-; RV32I-NEXT:    lw a5, 0(a1)
-; RV32I-NEXT:    lw a1, 4(a1)
 ; RV32I-NEXT:    lw a2, 0(a2)
-; RV32I-NEXT:    sw a4, 12(sp)
-; RV32I-NEXT:    sw a3, 8(sp)
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    sw a5, 0(sp)
-; RV32I-NEXT:    srai a4, a4, 31
-; RV32I-NEXT:    sw a4, 28(sp)
-; RV32I-NEXT:    sw a4, 24(sp)
-; RV32I-NEXT:    sw a4, 20(sp)
-; RV32I-NEXT:    sw a4, 16(sp)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    srai a1, a1, 31
+; RV32I-NEXT:    sw a1, 16(sp)
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a1, 24(sp)
+; RV32I-NEXT:    sw a1, 28(sp)
 ; RV32I-NEXT:    srli a1, a2, 3
 ; RV32I-NEXT:    andi a1, a1, 12
 ; RV32I-NEXT:    mv a3, sp
@@ -258,10 +258,10 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sll a6, a6, a7
 ; RV32I-NEXT:    or a5, a5, a6
 ; RV32I-NEXT:    sra a1, a1, a2
-; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    sw a5, 8(a0)
-; RV32I-NEXT:    sw a4, 4(a0)
 ; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a4, 4(a0)
+; RV32I-NEXT:    sw a5, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -296,14 +296,14 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    lw a4, 4(a1)
 ; RV32I-NEXT:    lw a5, 8(a1)
 ; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a1, 28(sp)
-; RV32I-NEXT:    sw a5, 24(sp)
-; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw a5, 24(sp)
+; RV32I-NEXT:    sw a1, 28(sp)
 ; RV32I-NEXT:    srli a1, a2, 3
 ; RV32I-NEXT:    andi a1, a1, 12
 ; RV32I-NEXT:    addi a3, sp, 16
@@ -328,9 +328,9 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    or a3, a3, a5
 ; RV32I-NEXT:    sll a2, a4, a2
 ; RV32I-NEXT:    sw a2, 0(a0)
-; RV32I-NEXT:    sw a3, 12(a0)
-; RV32I-NEXT:    sw a1, 8(a0)
 ; RV32I-NEXT:    sw a6, 4(a0)
+; RV32I-NEXT:    sw a1, 8(a0)
+; RV32I-NEXT:    sw a3, 12(a0)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -446,10 +446,10 @@ define i128 @fshr128_minsize(i128 %a, i128 %b) minsize nounwind {
 ; RV32I-NEXT:    slli a5, a5, 1
 ; RV32I-NEXT:    sll a2, a5, t2
 ; RV32I-NEXT:    or a1, a2, a1
-; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    sw a3, 8(a0)
-; RV32I-NEXT:    sw t0, 4(a0)
 ; RV32I-NEXT:    sw a7, 0(a0)
+; RV32I-NEXT:    sw t0, 4(a0)
+; RV32I-NEXT:    sw a3, 8(a0)
+; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB10_13:
 ; RV32I-NEXT:    mv a5, t0
diff --git a/llvm/test/CodeGen/RISCV/split-offsets.ll b/llvm/test/CodeGen/RISCV/split-offsets.ll
index 8d065daa2067c4a99eeba58a7e347f2120503963..284b559eeec5f56f5a2e90b332100a5d8adaa67d 100644
--- a/llvm/test/CodeGen/RISCV/split-offsets.ll
+++ b/llvm/test/CodeGen/RISCV/split-offsets.ll
@@ -17,8 +17,8 @@ define void @test1(ptr %sp, ptr %t, i32 %n) {
 ; RV32I-NEXT:    add a1, a1, a2
 ; RV32I-NEXT:    add a0, a0, a2
 ; RV32I-NEXT:    li a2, 2
-; RV32I-NEXT:    sw a2, 0(a0)
 ; RV32I-NEXT:    li a3, 1
+; RV32I-NEXT:    sw a2, 0(a0)
 ; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    sw a3, 0(a1)
 ; RV32I-NEXT:    sw a2, 4(a1)
@@ -32,8 +32,8 @@ define void @test1(ptr %sp, ptr %t, i32 %n) {
 ; RV64I-NEXT:    add a1, a1, a2
 ; RV64I-NEXT:    add a0, a0, a2
 ; RV64I-NEXT:    li a2, 2
-; RV64I-NEXT:    sw a2, 0(a0)
 ; RV64I-NEXT:    li a3, 1
+; RV64I-NEXT:    sw a2, 0(a0)
 ; RV64I-NEXT:    sw a3, 4(a0)
 ; RV64I-NEXT:    sw a3, 0(a1)
 ; RV64I-NEXT:    sw a2, 4(a1)
@@ -128,9 +128,9 @@ define void @test3(ptr %t) {
 ; RV32I-NEXT:    addi a1, a1, -1920
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    li a1, 2
+; RV32I-NEXT:    li a2, 3
 ; RV32I-NEXT:    sw a1, 4(a0)
-; RV32I-NEXT:    li a1, 3
-; RV32I-NEXT:    sw a1, 8(a0)
+; RV32I-NEXT:    sw a2, 8(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test3:
@@ -139,9 +139,9 @@ define void @test3(ptr %t) {
 ; RV64I-NEXT:    addiw a1, a1, -1920
 ; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    li a1, 2
+; RV64I-NEXT:    li a2, 3
 ; RV64I-NEXT:    sw a1, 4(a0)
-; RV64I-NEXT:    li a1, 3
-; RV64I-NEXT:    sw a1, 8(a0)
+; RV64I-NEXT:    sw a2, 8(a0)
 ; RV64I-NEXT:    ret
 entry:
   %splitgep = getelementptr i8, ptr %t, i64 80000
diff --git a/llvm/test/CodeGen/RISCV/split-store.ll b/llvm/test/CodeGen/RISCV/split-store.ll
index 367d3fe2c595feabeb4a238a1847d5deb6ef04ba..a9d29f1840cfc1285546aed726a2d8db404a211b 100644
--- a/llvm/test/CodeGen/RISCV/split-store.ll
+++ b/llvm/test/CodeGen/RISCV/split-store.ll
@@ -117,14 +117,14 @@ define void @int8_float_pair(i8 signext %tmp1, float %tmp2, ptr %ref.tmp) {
 define void @int32_int32_pair(i32 %tmp1, i32 %tmp2, ptr %ref.tmp) {
 ; RV32-LABEL: int32_int32_pair:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw a1, 4(a2)
 ; RV32-NEXT:    sw a0, 0(a2)
+; RV32-NEXT:    sw a1, 4(a2)
 ; RV32-NEXT:    ret
 ;
 ; RV32D-LABEL: int32_int32_pair:
 ; RV32D:       # %bb.0:
-; RV32D-NEXT:    sw a1, 4(a2)
 ; RV32D-NEXT:    sw a0, 0(a2)
+; RV32D-NEXT:    sw a1, 4(a2)
 ; RV32D-NEXT:    ret
 ;
 ; RV64-LABEL: int32_int32_pair:
diff --git a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
index 5d00e90366c3be7684c5bb0901dbbff1a846833d..307a0531cf029667ae24aea45ab48acfdfb08260 100644
--- a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
@@ -354,19 +354,19 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV32-NEXT:    neg a3, a2
 ; RV32-NEXT:    addi a1, a1, -1
 ; RV32-NEXT:    addi a0, a0, -1
-; RV32-NEXT:    sw a3, 0(s0)
-; RV32-NEXT:    andi a3, a0, 7
-; RV32-NEXT:    sb a3, 12(s0)
-; RV32-NEXT:    slli a3, a1, 1
-; RV32-NEXT:    or a2, a3, a2
-; RV32-NEXT:    sw a2, 4(s0)
-; RV32-NEXT:    srli a2, a1, 31
+; RV32-NEXT:    andi a4, a0, 7
+; RV32-NEXT:    slli a5, a1, 1
+; RV32-NEXT:    or a2, a5, a2
+; RV32-NEXT:    srli a5, a1, 31
 ; RV32-NEXT:    andi a1, a1, 1
 ; RV32-NEXT:    slli a1, a1, 1
 ; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    or a0, a2, a0
+; RV32-NEXT:    or a0, a5, a0
 ; RV32-NEXT:    or a0, a0, a1
+; RV32-NEXT:    sw a3, 0(s0)
+; RV32-NEXT:    sw a2, 4(s0)
 ; RV32-NEXT:    sw a0, 8(s0)
+; RV32-NEXT:    sb a4, 12(s0)
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -431,15 +431,15 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV64-NEXT:    slli a4, a2, 31
 ; RV64-NEXT:    srli a4, a4, 62
 ; RV64-NEXT:    or a3, a4, a3
-; RV64-NEXT:    sw a3, 8(s0)
 ; RV64-NEXT:    slli a1, a1, 29
 ; RV64-NEXT:    srli a1, a1, 61
-; RV64-NEXT:    sb a1, 12(s0)
 ; RV64-NEXT:    slli a0, a0, 31
 ; RV64-NEXT:    srli a0, a0, 31
 ; RV64-NEXT:    slli a2, a2, 33
 ; RV64-NEXT:    or a0, a0, a2
 ; RV64-NEXT:    sd a0, 0(s0)
+; RV64-NEXT:    sw a3, 8(s0)
+; RV64-NEXT:    sb a1, 12(s0)
 ; RV64-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -506,19 +506,19 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV32M-NEXT:    neg a3, a2
 ; RV32M-NEXT:    addi a1, a1, -1
 ; RV32M-NEXT:    addi a0, a0, -1
-; RV32M-NEXT:    sw a3, 0(s0)
-; RV32M-NEXT:    andi a3, a0, 7
-; RV32M-NEXT:    sb a3, 12(s0)
-; RV32M-NEXT:    slli a3, a1, 1
-; RV32M-NEXT:    or a2, a3, a2
-; RV32M-NEXT:    sw a2, 4(s0)
-; RV32M-NEXT:    srli a2, a1, 31
+; RV32M-NEXT:    andi a4, a0, 7
+; RV32M-NEXT:    slli a5, a1, 1
+; RV32M-NEXT:    or a2, a5, a2
+; RV32M-NEXT:    srli a5, a1, 31
 ; RV32M-NEXT:    andi a1, a1, 1
 ; RV32M-NEXT:    slli a1, a1, 1
 ; RV32M-NEXT:    slli a0, a0, 2
-; RV32M-NEXT:    or a0, a2, a0
+; RV32M-NEXT:    or a0, a5, a0
 ; RV32M-NEXT:    or a0, a0, a1
+; RV32M-NEXT:    sw a3, 0(s0)
+; RV32M-NEXT:    sw a2, 4(s0)
 ; RV32M-NEXT:    sw a0, 8(s0)
+; RV32M-NEXT:    sb a4, 12(s0)
 ; RV32M-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32M-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32M-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -586,14 +586,14 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV64M-NEXT:    slli a1, a1, 31
 ; RV64M-NEXT:    srli a1, a1, 31
 ; RV64M-NEXT:    or a1, a1, a4
-; RV64M-NEXT:    sd a1, 0(a0)
-; RV64M-NEXT:    slli a1, a2, 2
+; RV64M-NEXT:    slli a4, a2, 2
 ; RV64M-NEXT:    slli a3, a3, 31
 ; RV64M-NEXT:    srli a3, a3, 62
-; RV64M-NEXT:    or a1, a3, a1
-; RV64M-NEXT:    sw a1, 8(a0)
+; RV64M-NEXT:    or a3, a3, a4
 ; RV64M-NEXT:    slli a2, a2, 29
 ; RV64M-NEXT:    srli a2, a2, 61
+; RV64M-NEXT:    sd a1, 0(a0)
+; RV64M-NEXT:    sw a3, 8(a0)
 ; RV64M-NEXT:    sb a2, 12(a0)
 ; RV64M-NEXT:    ret
 ;
@@ -629,9 +629,9 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    slli a1, a3, 31
 ; RV32MV-NEXT:    srai a1, a1, 31
 ; RV32MV-NEXT:    li a2, 1
+; RV32MV-NEXT:    li a3, -1
+; RV32MV-NEXT:    sw a3, 16(sp)
 ; RV32MV-NEXT:    sw a2, 20(sp)
-; RV32MV-NEXT:    li a2, -1
-; RV32MV-NEXT:    sw a2, 16(sp)
 ; RV32MV-NEXT:    li a2, 6
 ; RV32MV-NEXT:    li a3, 0
 ; RV32MV-NEXT:    call __moddi3
@@ -683,35 +683,35 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    vmsne.vv v0, v8, v12
 ; RV32MV-NEXT:    vmv.v.i v8, 0
 ; RV32MV-NEXT:    vmerge.vim v8, v8, -1, v0
-; RV32MV-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32MV-NEXT:    vse32.v v8, (s0)
+; RV32MV-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32MV-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32MV-NEXT:    vmv.x.s a0, v10
 ; RV32MV-NEXT:    vslidedown.vi v10, v8, 2
 ; RV32MV-NEXT:    vmv.x.s a1, v10
 ; RV32MV-NEXT:    slli a2, a1, 1
 ; RV32MV-NEXT:    sub a2, a2, a0
-; RV32MV-NEXT:    sw a2, 4(s0)
 ; RV32MV-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32MV-NEXT:    vslidedown.vi v10, v8, 4
 ; RV32MV-NEXT:    vmv.x.s a0, v10
-; RV32MV-NEXT:    srli a2, a0, 30
+; RV32MV-NEXT:    srli a3, a0, 30
 ; RV32MV-NEXT:    vslidedown.vi v10, v8, 5
-; RV32MV-NEXT:    vmv.x.s a3, v10
-; RV32MV-NEXT:    slli a3, a3, 2
-; RV32MV-NEXT:    or a2, a3, a2
-; RV32MV-NEXT:    andi a2, a2, 7
-; RV32MV-NEXT:    sb a2, 12(s0)
-; RV32MV-NEXT:    srli a1, a1, 31
+; RV32MV-NEXT:    vmv.x.s a4, v10
+; RV32MV-NEXT:    slli a4, a4, 2
+; RV32MV-NEXT:    or a3, a4, a3
 ; RV32MV-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32MV-NEXT:    vse32.v v8, (s0)
+; RV32MV-NEXT:    andi a3, a3, 7
+; RV32MV-NEXT:    srli a1, a1, 31
 ; RV32MV-NEXT:    vslidedown.vi v8, v8, 3
-; RV32MV-NEXT:    vmv.x.s a2, v8
-; RV32MV-NEXT:    andi a2, a2, 1
-; RV32MV-NEXT:    slli a2, a2, 1
 ; RV32MV-NEXT:    slli a0, a0, 2
 ; RV32MV-NEXT:    or a0, a1, a0
-; RV32MV-NEXT:    or a0, a0, a2
+; RV32MV-NEXT:    vmv.x.s a1, v8
+; RV32MV-NEXT:    andi a1, a1, 1
+; RV32MV-NEXT:    slli a1, a1, 1
+; RV32MV-NEXT:    or a0, a0, a1
+; RV32MV-NEXT:    sw a2, 4(s0)
 ; RV32MV-NEXT:    sw a0, 8(s0)
+; RV32MV-NEXT:    sb a3, 12(s0)
 ; RV32MV-NEXT:    csrr a0, vlenb
 ; RV32MV-NEXT:    slli a0, a0, 1
 ; RV32MV-NEXT:    add sp, sp, a0
@@ -785,20 +785,20 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV64MV-NEXT:    vmv.x.s a2, v10
 ; RV64MV-NEXT:    slli a3, a2, 31
 ; RV64MV-NEXT:    srli a3, a3, 61
-; RV64MV-NEXT:    sb a3, 12(a0)
-; RV64MV-NEXT:    vmv.x.s a3, v8
-; RV64MV-NEXT:    and a1, a3, a1
+; RV64MV-NEXT:    vmv.x.s a4, v8
+; RV64MV-NEXT:    and a1, a4, a1
 ; RV64MV-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64MV-NEXT:    vslidedown.vi v8, v8, 1
-; RV64MV-NEXT:    vmv.x.s a3, v8
-; RV64MV-NEXT:    slli a4, a3, 33
-; RV64MV-NEXT:    or a1, a1, a4
-; RV64MV-NEXT:    sd a1, 0(a0)
+; RV64MV-NEXT:    vmv.x.s a4, v8
+; RV64MV-NEXT:    slli a5, a4, 33
+; RV64MV-NEXT:    or a1, a1, a5
 ; RV64MV-NEXT:    slli a2, a2, 2
-; RV64MV-NEXT:    slli a3, a3, 31
-; RV64MV-NEXT:    srli a3, a3, 62
-; RV64MV-NEXT:    or a2, a3, a2
+; RV64MV-NEXT:    slli a4, a4, 31
+; RV64MV-NEXT:    srli a4, a4, 62
+; RV64MV-NEXT:    or a2, a4, a2
+; RV64MV-NEXT:    sd a1, 0(a0)
 ; RV64MV-NEXT:    sw a2, 8(a0)
+; RV64MV-NEXT:    sb a3, 12(a0)
 ; RV64MV-NEXT:    ret
   %ld = load <3 x i33>, ptr %X
   %srem = srem <3 x i33> %ld, <i33 6, i33 7, i33 -5>
diff --git a/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll b/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
index 90443051d4b574d37e3654564b1a03f39bd09b63..c7e57021b90dc17282ee33c58cb80a265b87685b 100644
--- a/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
+++ b/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
@@ -38,10 +38,10 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    li a1, -1003
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    sh a0, 6(s3)
-; RV32I-NEXT:    sh s1, 4(s3)
-; RV32I-NEXT:    sh s0, 2(s3)
 ; RV32I-NEXT:    sh s4, 0(s3)
+; RV32I-NEXT:    sh s0, 2(s3)
+; RV32I-NEXT:    sh s1, 4(s3)
+; RV32I-NEXT:    sh a0, 6(s3)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -95,10 +95,10 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    li a6, -1003
 ; RV32IM-NEXT:    mul a5, a5, a6
 ; RV32IM-NEXT:    sub a1, a1, a5
-; RV32IM-NEXT:    sh a1, 6(a0)
-; RV32IM-NEXT:    sh a4, 4(a0)
-; RV32IM-NEXT:    sh a3, 2(a0)
 ; RV32IM-NEXT:    sh a2, 0(a0)
+; RV32IM-NEXT:    sh a3, 2(a0)
+; RV32IM-NEXT:    sh a4, 4(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: fold_srem_vec_1:
@@ -130,10 +130,10 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    li a1, -1003
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    sh a0, 6(s3)
-; RV64I-NEXT:    sh s1, 4(s3)
-; RV64I-NEXT:    sh s0, 2(s3)
 ; RV64I-NEXT:    sh s4, 0(s3)
+; RV64I-NEXT:    sh s0, 2(s3)
+; RV64I-NEXT:    sh s1, 4(s3)
+; RV64I-NEXT:    sh a0, 6(s3)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -187,10 +187,10 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    li a6, -1003
 ; RV64IM-NEXT:    mul a2, a2, a6
 ; RV64IM-NEXT:    subw a1, a1, a2
-; RV64IM-NEXT:    sh a1, 6(a0)
-; RV64IM-NEXT:    sh a5, 4(a0)
-; RV64IM-NEXT:    sh a4, 2(a0)
 ; RV64IM-NEXT:    sh a3, 0(a0)
+; RV64IM-NEXT:    sh a4, 2(a0)
+; RV64IM-NEXT:    sh a5, 4(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i16> %x, <i16 95, i16 -124, i16 98, i16 -1003>
   ret <4 x i16> %1
@@ -226,10 +226,10 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    li a1, 95
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    sh a0, 6(s3)
-; RV32I-NEXT:    sh s1, 4(s3)
-; RV32I-NEXT:    sh s0, 2(s3)
 ; RV32I-NEXT:    sh s4, 0(s3)
+; RV32I-NEXT:    sh s0, 2(s3)
+; RV32I-NEXT:    sh s1, 4(s3)
+; RV32I-NEXT:    sh a0, 6(s3)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -276,10 +276,10 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    add a5, a5, a6
 ; RV32IM-NEXT:    mul a5, a5, a7
 ; RV32IM-NEXT:    sub a1, a1, a5
-; RV32IM-NEXT:    sh a1, 6(a0)
-; RV32IM-NEXT:    sh a4, 4(a0)
-; RV32IM-NEXT:    sh a3, 2(a0)
 ; RV32IM-NEXT:    sh a2, 0(a0)
+; RV32IM-NEXT:    sh a3, 2(a0)
+; RV32IM-NEXT:    sh a4, 4(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: fold_srem_vec_2:
@@ -311,10 +311,10 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    li a1, 95
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    sh a0, 6(s3)
-; RV64I-NEXT:    sh s1, 4(s3)
-; RV64I-NEXT:    sh s0, 2(s3)
 ; RV64I-NEXT:    sh s4, 0(s3)
+; RV64I-NEXT:    sh s0, 2(s3)
+; RV64I-NEXT:    sh s1, 4(s3)
+; RV64I-NEXT:    sh a0, 6(s3)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -361,10 +361,10 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    add a2, a2, a6
 ; RV64IM-NEXT:    mul a2, a2, a7
 ; RV64IM-NEXT:    subw a1, a1, a2
-; RV64IM-NEXT:    sh a1, 6(a0)
-; RV64IM-NEXT:    sh a5, 4(a0)
-; RV64IM-NEXT:    sh a4, 2(a0)
 ; RV64IM-NEXT:    sh a3, 0(a0)
+; RV64IM-NEXT:    sh a4, 2(a0)
+; RV64IM-NEXT:    sh a5, 4(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
   ret <4 x i16> %1
@@ -426,10 +426,10 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    add s2, s7, s2
 ; RV32I-NEXT:    add s3, s6, s3
 ; RV32I-NEXT:    add s4, s5, s4
-; RV32I-NEXT:    sh s4, 6(s0)
-; RV32I-NEXT:    sh s3, 4(s0)
-; RV32I-NEXT:    sh s2, 2(s0)
 ; RV32I-NEXT:    sh a0, 0(s0)
+; RV32I-NEXT:    sh s2, 2(s0)
+; RV32I-NEXT:    sh s3, 4(s0)
+; RV32I-NEXT:    sh s4, 6(s0)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -484,10 +484,10 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    sub a1, a1, t2
 ; RV32IM-NEXT:    add a2, a2, a6
 ; RV32IM-NEXT:    sub a2, a2, t0
-; RV32IM-NEXT:    sh a2, 6(a0)
-; RV32IM-NEXT:    sh a1, 4(a0)
-; RV32IM-NEXT:    sh a4, 2(a0)
 ; RV32IM-NEXT:    sh a3, 0(a0)
+; RV32IM-NEXT:    sh a4, 2(a0)
+; RV32IM-NEXT:    sh a1, 4(a0)
+; RV32IM-NEXT:    sh a2, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: combine_srem_sdiv:
@@ -543,10 +543,10 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    add s2, s7, s2
 ; RV64I-NEXT:    add s3, s6, s3
 ; RV64I-NEXT:    add s4, s5, s4
-; RV64I-NEXT:    sh s4, 6(s0)
-; RV64I-NEXT:    sh s3, 4(s0)
-; RV64I-NEXT:    sh s2, 2(s0)
 ; RV64I-NEXT:    sh a0, 0(s0)
+; RV64I-NEXT:    sh s2, 2(s0)
+; RV64I-NEXT:    sh s3, 4(s0)
+; RV64I-NEXT:    sh s4, 6(s0)
 ; RV64I-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 56(sp) # 8-byte Folded Reload
@@ -601,10 +601,10 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    subw a1, a1, t2
 ; RV64IM-NEXT:    add a2, a2, a6
 ; RV64IM-NEXT:    subw a2, a2, t0
-; RV64IM-NEXT:    sh a2, 6(a0)
-; RV64IM-NEXT:    sh a1, 4(a0)
-; RV64IM-NEXT:    sh a4, 2(a0)
 ; RV64IM-NEXT:    sh a3, 0(a0)
+; RV64IM-NEXT:    sh a4, 2(a0)
+; RV64IM-NEXT:    sh a1, 4(a0)
+; RV64IM-NEXT:    sh a2, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
   %2 = sdiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
@@ -641,10 +641,10 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    sub s3, a4, a1
 ; RV32I-NEXT:    li a1, 95
 ; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    sh a0, 6(s0)
-; RV32I-NEXT:    sh s3, 4(s0)
-; RV32I-NEXT:    sh s2, 2(s0)
 ; RV32I-NEXT:    sh s1, 0(s0)
+; RV32I-NEXT:    sh s2, 2(s0)
+; RV32I-NEXT:    sh s3, 4(s0)
+; RV32I-NEXT:    sh a0, 6(s0)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -681,9 +681,9 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    add a5, a3, a5
 ; RV32IM-NEXT:    andi a5, a5, -8
 ; RV32IM-NEXT:    sub a3, a3, a5
-; RV32IM-NEXT:    sh a3, 4(a0)
-; RV32IM-NEXT:    sh a2, 2(a0)
 ; RV32IM-NEXT:    sh a1, 0(a0)
+; RV32IM-NEXT:    sh a2, 2(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
 ; RV32IM-NEXT:    sh a4, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
@@ -714,10 +714,10 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    subw s3, a4, a1
 ; RV64I-NEXT:    li a1, 95
 ; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    sh a0, 6(s0)
-; RV64I-NEXT:    sh s3, 4(s0)
-; RV64I-NEXT:    sh s2, 2(s0)
 ; RV64I-NEXT:    sh s1, 0(s0)
+; RV64I-NEXT:    sh s2, 2(s0)
+; RV64I-NEXT:    sh s3, 4(s0)
+; RV64I-NEXT:    sh a0, 6(s0)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -754,9 +754,9 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    add a3, a4, a3
 ; RV64IM-NEXT:    andi a3, a3, -8
 ; RV64IM-NEXT:    subw a4, a4, a3
-; RV64IM-NEXT:    sh a4, 4(a0)
-; RV64IM-NEXT:    sh a5, 2(a0)
 ; RV64IM-NEXT:    sh a1, 0(a0)
+; RV64IM-NEXT:    sh a5, 2(a0)
+; RV64IM-NEXT:    sh a4, 4(a0)
 ; RV64IM-NEXT:    sh a2, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
@@ -789,10 +789,10 @@ define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    addi a1, a0, 1327
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    sh a0, 6(s2)
-; RV32I-NEXT:    sh s0, 4(s2)
-; RV32I-NEXT:    sh s3, 2(s2)
 ; RV32I-NEXT:    sh zero, 0(s2)
+; RV32I-NEXT:    sh s3, 2(s2)
+; RV32I-NEXT:    sh s0, 4(s2)
+; RV32I-NEXT:    sh a0, 6(s2)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -837,9 +837,9 @@ define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    mul a4, a4, a5
 ; RV32IM-NEXT:    sub a1, a1, a4
 ; RV32IM-NEXT:    sh zero, 0(a0)
-; RV32IM-NEXT:    sh a1, 6(a0)
-; RV32IM-NEXT:    sh a3, 4(a0)
 ; RV32IM-NEXT:    sh a2, 2(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: dont_fold_srem_one:
@@ -866,10 +866,10 @@ define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    sh a0, 6(s2)
-; RV64I-NEXT:    sh s0, 4(s2)
-; RV64I-NEXT:    sh s3, 2(s2)
 ; RV64I-NEXT:    sh zero, 0(s2)
+; RV64I-NEXT:    sh s3, 2(s2)
+; RV64I-NEXT:    sh s0, 4(s2)
+; RV64I-NEXT:    sh a0, 6(s2)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -913,9 +913,9 @@ define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    mul a2, a2, a5
 ; RV64IM-NEXT:    subw a1, a1, a2
 ; RV64IM-NEXT:    sh zero, 0(a0)
-; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    sh a4, 2(a0)
 ; RV64IM-NEXT:    sh a3, 4(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
   ret <4 x i16> %1
@@ -947,10 +947,10 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    addi a1, a0, 1327
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    sh a0, 6(s0)
-; RV32I-NEXT:    sh s2, 4(s0)
 ; RV32I-NEXT:    sh zero, 0(s0)
 ; RV32I-NEXT:    sh s3, 2(s0)
+; RV32I-NEXT:    sh s2, 4(s0)
+; RV32I-NEXT:    sh a0, 6(s0)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -991,8 +991,8 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    sub a2, a2, a4
 ; RV32IM-NEXT:    sh zero, 0(a0)
 ; RV32IM-NEXT:    sh a2, 2(a0)
-; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: dont_fold_urem_i16_smax:
@@ -1019,10 +1019,10 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    sh a0, 6(s0)
-; RV64I-NEXT:    sh s2, 4(s0)
 ; RV64I-NEXT:    sh zero, 0(s0)
 ; RV64I-NEXT:    sh s3, 2(s0)
+; RV64I-NEXT:    sh s2, 4(s0)
+; RV64I-NEXT:    sh a0, 6(s0)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -1063,8 +1063,8 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    subw a4, a4, a2
 ; RV64IM-NEXT:    sh zero, 0(a0)
 ; RV64IM-NEXT:    sh a4, 2(a0)
-; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    sh a3, 4(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i16> %x, <i16 1, i16 32768, i16 23, i16 5423>
   ret <4 x i16> %1
@@ -1085,15 +1085,15 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    lw s0, 16(a1)
-; RV32I-NEXT:    lw s1, 20(a1)
-; RV32I-NEXT:    lw s2, 24(a1)
-; RV32I-NEXT:    lw s3, 28(a1)
+; RV32I-NEXT:    lw s1, 16(a1)
+; RV32I-NEXT:    lw s2, 20(a1)
+; RV32I-NEXT:    lw s3, 24(a1)
+; RV32I-NEXT:    lw s4, 28(a1)
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    lw s4, 8(a1)
-; RV32I-NEXT:    lw s5, 12(a1)
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    lw s5, 8(a1)
+; RV32I-NEXT:    lw s6, 12(a1)
+; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    li a2, 1
 ; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    mv a1, a4
@@ -1102,33 +1102,33 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    mv s8, a1
 ; RV32I-NEXT:    li a2, 654
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s5
+; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    mv a1, s6
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __moddi3
-; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    mv s5, a1
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s6, a1
 ; RV32I-NEXT:    li a2, 23
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __moddi3
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    mv s1, a1
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a2, a0, 1327
-; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    mv a1, s4
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __moddi3
-; RV32I-NEXT:    sw a1, 28(s6)
-; RV32I-NEXT:    sw a0, 24(s6)
-; RV32I-NEXT:    sw s1, 20(s6)
-; RV32I-NEXT:    sw s0, 16(s6)
-; RV32I-NEXT:    sw s5, 12(s6)
-; RV32I-NEXT:    sw s4, 8(s6)
-; RV32I-NEXT:    sw s8, 4(s6)
-; RV32I-NEXT:    sw s7, 0(s6)
+; RV32I-NEXT:    sw s1, 16(s0)
+; RV32I-NEXT:    sw s2, 20(s0)
+; RV32I-NEXT:    sw a0, 24(s0)
+; RV32I-NEXT:    sw a1, 28(s0)
+; RV32I-NEXT:    sw s7, 0(s0)
+; RV32I-NEXT:    sw s8, 4(s0)
+; RV32I-NEXT:    sw s5, 8(s0)
+; RV32I-NEXT:    sw s6, 12(s0)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -1155,15 +1155,15 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
-; RV32IM-NEXT:    lw s0, 16(a1)
-; RV32IM-NEXT:    lw s1, 20(a1)
-; RV32IM-NEXT:    lw s2, 24(a1)
-; RV32IM-NEXT:    lw s3, 28(a1)
+; RV32IM-NEXT:    lw s1, 16(a1)
+; RV32IM-NEXT:    lw s2, 20(a1)
+; RV32IM-NEXT:    lw s3, 24(a1)
+; RV32IM-NEXT:    lw s4, 28(a1)
 ; RV32IM-NEXT:    lw a3, 0(a1)
 ; RV32IM-NEXT:    lw a4, 4(a1)
-; RV32IM-NEXT:    lw s4, 8(a1)
-; RV32IM-NEXT:    lw s5, 12(a1)
-; RV32IM-NEXT:    mv s6, a0
+; RV32IM-NEXT:    lw s5, 8(a1)
+; RV32IM-NEXT:    lw s6, 12(a1)
+; RV32IM-NEXT:    mv s0, a0
 ; RV32IM-NEXT:    li a2, 1
 ; RV32IM-NEXT:    mv a0, a3
 ; RV32IM-NEXT:    mv a1, a4
@@ -1172,33 +1172,33 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    mv s7, a0
 ; RV32IM-NEXT:    mv s8, a1
 ; RV32IM-NEXT:    li a2, 654
-; RV32IM-NEXT:    mv a0, s4
-; RV32IM-NEXT:    mv a1, s5
+; RV32IM-NEXT:    mv a0, s5
+; RV32IM-NEXT:    mv a1, s6
 ; RV32IM-NEXT:    li a3, 0
 ; RV32IM-NEXT:    call __moddi3
-; RV32IM-NEXT:    mv s4, a0
-; RV32IM-NEXT:    mv s5, a1
+; RV32IM-NEXT:    mv s5, a0
+; RV32IM-NEXT:    mv s6, a1
 ; RV32IM-NEXT:    li a2, 23
-; RV32IM-NEXT:    mv a0, s0
-; RV32IM-NEXT:    mv a1, s1
+; RV32IM-NEXT:    mv a0, s1
+; RV32IM-NEXT:    mv a1, s2
 ; RV32IM-NEXT:    li a3, 0
 ; RV32IM-NEXT:    call __moddi3
-; RV32IM-NEXT:    mv s0, a0
-; RV32IM-NEXT:    mv s1, a1
+; RV32IM-NEXT:    mv s1, a0
+; RV32IM-NEXT:    mv s2, a1
 ; RV32IM-NEXT:    lui a0, 1
 ; RV32IM-NEXT:    addi a2, a0, 1327
-; RV32IM-NEXT:    mv a0, s2
-; RV32IM-NEXT:    mv a1, s3
+; RV32IM-NEXT:    mv a0, s3
+; RV32IM-NEXT:    mv a1, s4
 ; RV32IM-NEXT:    li a3, 0
 ; RV32IM-NEXT:    call __moddi3
-; RV32IM-NEXT:    sw a1, 28(s6)
-; RV32IM-NEXT:    sw a0, 24(s6)
-; RV32IM-NEXT:    sw s1, 20(s6)
-; RV32IM-NEXT:    sw s0, 16(s6)
-; RV32IM-NEXT:    sw s5, 12(s6)
-; RV32IM-NEXT:    sw s4, 8(s6)
-; RV32IM-NEXT:    sw s8, 4(s6)
-; RV32IM-NEXT:    sw s7, 0(s6)
+; RV32IM-NEXT:    sw s1, 16(s0)
+; RV32IM-NEXT:    sw s2, 20(s0)
+; RV32IM-NEXT:    sw a0, 24(s0)
+; RV32IM-NEXT:    sw a1, 28(s0)
+; RV32IM-NEXT:    sw s7, 0(s0)
+; RV32IM-NEXT:    sw s8, 4(s0)
+; RV32IM-NEXT:    sw s5, 8(s0)
+; RV32IM-NEXT:    sw s6, 12(s0)
 ; RV32IM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -1236,10 +1236,10 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    sd a0, 24(s2)
-; RV64I-NEXT:    sd s0, 16(s2)
-; RV64I-NEXT:    sd s3, 8(s2)
 ; RV64I-NEXT:    sd zero, 0(s2)
+; RV64I-NEXT:    sd s3, 8(s2)
+; RV64I-NEXT:    sd s0, 16(s2)
+; RV64I-NEXT:    sd a0, 24(s2)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -1283,9 +1283,9 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV64IM-NEXT:    mul a2, a2, a5
 ; RV64IM-NEXT:    sub a1, a1, a2
 ; RV64IM-NEXT:    sd zero, 0(a0)
-; RV64IM-NEXT:    sd a1, 24(a0)
 ; RV64IM-NEXT:    sd a4, 8(a0)
 ; RV64IM-NEXT:    sd a3, 16(a0)
+; RV64IM-NEXT:    sd a1, 24(a0)
 ; RV64IM-NEXT:    ret
   %1 = srem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
   ret <4 x i64> %1
diff --git a/llvm/test/CodeGen/RISCV/stack-slot-size.ll b/llvm/test/CodeGen/RISCV/stack-slot-size.ll
index 1388eaac3a67e7d705fe5c41b484d32f72cd821d..71ee6d8160a9da8e9dc9b3e18a6f55f15be89ba7 100644
--- a/llvm/test/CodeGen/RISCV/stack-slot-size.ll
+++ b/llvm/test/CodeGen/RISCV/stack-slot-size.ll
@@ -21,11 +21,11 @@ define i32 @caller129() nounwind {
 ; RV32I-NEXT:    li a0, 42
 ; RV32I-NEXT:    sw a0, 24(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    mv a0, sp
 ; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    call callee129
 ; RV32I-NEXT:    lw a0, 24(sp)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
@@ -38,10 +38,10 @@ define i32 @caller129() nounwind {
 ; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    li a0, 42
 ; RV64I-NEXT:    sw a0, 36(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    mv a0, sp
 ; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    call callee129
 ; RV64I-NEXT:    lw a0, 36(sp)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
@@ -62,11 +62,11 @@ define i32 @caller160() nounwind {
 ; RV32I-NEXT:    li a0, 42
 ; RV32I-NEXT:    sw a0, 24(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    mv a0, sp
 ; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    call callee160
 ; RV32I-NEXT:    lw a0, 24(sp)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
@@ -79,10 +79,10 @@ define i32 @caller160() nounwind {
 ; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    li a0, 42
 ; RV64I-NEXT:    sw a0, 36(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    mv a0, sp
 ; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    call callee160
 ; RV64I-NEXT:    lw a0, 36(sp)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
@@ -102,13 +102,13 @@ define i32 @caller161() nounwind {
 ; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    li a0, 42
 ; RV32I-NEXT:    sw a0, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    mv a0, sp
 ; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    call callee161
 ; RV32I-NEXT:    lw a0, 24(sp)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
@@ -121,10 +121,10 @@ define i32 @caller161() nounwind {
 ; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    li a0, 42
 ; RV64I-NEXT:    sw a0, 36(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    mv a0, sp
 ; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    call callee161
 ; RV64I-NEXT:    lw a0, 36(sp)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/stack-store-check.ll b/llvm/test/CodeGen/RISCV/stack-store-check.ll
index b51a759a87b85928a32a863dac0c7fe82d85317b..052ccbf6e06f369d69235ed1bbf2b1191c439d61 100644
--- a/llvm/test/CodeGen/RISCV/stack-store-check.ll
+++ b/llvm/test/CodeGen/RISCV/stack-store-check.ll
@@ -14,292 +14,294 @@
 define void @main() local_unnamed_addr nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -688
-; CHECK-NEXT:    sw ra, 684(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s0, 680(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s1, 676(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s2, 672(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s3, 668(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s4, 664(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s5, 660(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s6, 656(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s7, 652(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s8, 648(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s9, 644(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s10, 640(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s11, 636(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    addi sp, sp, -704
+; CHECK-NEXT:    sw ra, 700(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s0, 696(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s1, 692(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s2, 688(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s3, 684(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s4, 680(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s5, 676(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s6, 672(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s7, 668(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s8, 664(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s9, 660(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s10, 656(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s11, 652(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lui a0, %hi(U)
-; CHECK-NEXT:    lw s6, %lo(U)(a0)
-; CHECK-NEXT:    lw s7, %lo(U+4)(a0)
-; CHECK-NEXT:    lw s8, %lo(U+8)(a0)
-; CHECK-NEXT:    lw s0, %lo(U+12)(a0)
-; CHECK-NEXT:    sw zero, 612(sp)
-; CHECK-NEXT:    sw zero, 608(sp)
-; CHECK-NEXT:    sw zero, 604(sp)
-; CHECK-NEXT:    sw zero, 600(sp)
-; CHECK-NEXT:    sw s0, 596(sp)
-; CHECK-NEXT:    sw s8, 592(sp)
-; CHECK-NEXT:    sw s7, 588(sp)
-; CHECK-NEXT:    addi a0, sp, 616
-; CHECK-NEXT:    addi a1, sp, 600
-; CHECK-NEXT:    addi a2, sp, 584
-; CHECK-NEXT:    sw s6, 584(sp)
+; CHECK-NEXT:    lw s9, %lo(U)(a0)
+; CHECK-NEXT:    lw s10, %lo(U+4)(a0)
+; CHECK-NEXT:    lw s11, %lo(U+8)(a0)
+; CHECK-NEXT:    lw s5, %lo(U+12)(a0)
+; CHECK-NEXT:    sw zero, 616(sp)
+; CHECK-NEXT:    sw zero, 620(sp)
+; CHECK-NEXT:    sw zero, 624(sp)
+; CHECK-NEXT:    sw zero, 628(sp)
+; CHECK-NEXT:    addi a0, sp, 632
+; CHECK-NEXT:    addi a1, sp, 616
+; CHECK-NEXT:    addi a2, sp, 600
+; CHECK-NEXT:    sw s9, 600(sp)
+; CHECK-NEXT:    sw s10, 604(sp)
+; CHECK-NEXT:    sw s11, 608(sp)
+; CHECK-NEXT:    sw s5, 612(sp)
 ; CHECK-NEXT:    call __subtf3
-; CHECK-NEXT:    lw s1, 616(sp)
-; CHECK-NEXT:    lw s2, 620(sp)
-; CHECK-NEXT:    lw s3, 624(sp)
-; CHECK-NEXT:    lw s4, 628(sp)
-; CHECK-NEXT:    sw s0, 548(sp)
-; CHECK-NEXT:    sw s8, 544(sp)
-; CHECK-NEXT:    sw s7, 540(sp)
-; CHECK-NEXT:    sw s6, 536(sp)
-; CHECK-NEXT:    sw s4, 564(sp)
-; CHECK-NEXT:    sw s3, 560(sp)
-; CHECK-NEXT:    sw s2, 556(sp)
-; CHECK-NEXT:    addi a0, sp, 568
-; CHECK-NEXT:    addi a1, sp, 552
-; CHECK-NEXT:    addi a2, sp, 536
-; CHECK-NEXT:    sw s1, 552(sp)
+; CHECK-NEXT:    lw s1, 632(sp)
+; CHECK-NEXT:    lw s2, 636(sp)
+; CHECK-NEXT:    lw s3, 640(sp)
+; CHECK-NEXT:    lw s4, 644(sp)
+; CHECK-NEXT:    sw s9, 552(sp)
+; CHECK-NEXT:    sw s10, 556(sp)
+; CHECK-NEXT:    sw s11, 560(sp)
+; CHECK-NEXT:    sw s5, 564(sp)
+; CHECK-NEXT:    addi a0, sp, 584
+; CHECK-NEXT:    addi a1, sp, 568
+; CHECK-NEXT:    addi a2, sp, 552
+; CHECK-NEXT:    sw s1, 568(sp)
+; CHECK-NEXT:    sw s2, 572(sp)
+; CHECK-NEXT:    sw s3, 576(sp)
+; CHECK-NEXT:    sw s4, 580(sp)
 ; CHECK-NEXT:    call __subtf3
-; CHECK-NEXT:    lw a0, 568(sp)
-; CHECK-NEXT:    sw a0, 40(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lw a0, 572(sp)
-; CHECK-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lw a0, 576(sp)
-; CHECK-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lw a0, 580(sp)
+; CHECK-NEXT:    lw a0, 584(sp)
+; CHECK-NEXT:    sw a0, 52(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 588(sp)
 ; CHECK-NEXT:    sw a0, 48(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw zero, 500(sp)
-; CHECK-NEXT:    sw zero, 496(sp)
-; CHECK-NEXT:    sw zero, 492(sp)
-; CHECK-NEXT:    sw zero, 488(sp)
-; CHECK-NEXT:    sw s0, 516(sp)
-; CHECK-NEXT:    sw s8, 512(sp)
-; CHECK-NEXT:    sw s7, 508(sp)
-; CHECK-NEXT:    addi a0, sp, 520
-; CHECK-NEXT:    addi a1, sp, 504
-; CHECK-NEXT:    addi a2, sp, 488
-; CHECK-NEXT:    sw s6, 504(sp)
+; CHECK-NEXT:    lw a0, 592(sp)
+; CHECK-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 596(sp)
+; CHECK-NEXT:    sw a0, 40(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw zero, 504(sp)
+; CHECK-NEXT:    sw zero, 508(sp)
+; CHECK-NEXT:    sw zero, 512(sp)
+; CHECK-NEXT:    sw zero, 516(sp)
+; CHECK-NEXT:    addi a0, sp, 536
+; CHECK-NEXT:    addi a1, sp, 520
+; CHECK-NEXT:    addi a2, sp, 504
+; CHECK-NEXT:    sw s9, 520(sp)
+; CHECK-NEXT:    sw s10, 524(sp)
+; CHECK-NEXT:    sw s11, 528(sp)
+; CHECK-NEXT:    sw s5, 532(sp)
 ; CHECK-NEXT:    call __addtf3
-; CHECK-NEXT:    lw s9, 520(sp)
-; CHECK-NEXT:    lw s11, 524(sp)
-; CHECK-NEXT:    lw s5, 528(sp)
-; CHECK-NEXT:    lw s10, 532(sp)
-; CHECK-NEXT:    sw s10, 16(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw s0, 536(sp)
+; CHECK-NEXT:    lw s6, 540(sp)
+; CHECK-NEXT:    lw s7, 544(sp)
+; CHECK-NEXT:    lw s8, 548(sp)
 ; CHECK-NEXT:    lui a0, %hi(Y1)
 ; CHECK-NEXT:    lw a1, %lo(Y1)(a0)
-; CHECK-NEXT:    sw a1, 52(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 20(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a2, %lo(Y1+4)(a0)
-; CHECK-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a2, 16(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a3, %lo(Y1+8)(a0)
-; CHECK-NEXT:    sw a3, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a3, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, %lo(Y1+12)(a0)
-; CHECK-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw a0, 308(sp)
-; CHECK-NEXT:    sw a3, 304(sp)
-; CHECK-NEXT:    sw a2, 300(sp)
-; CHECK-NEXT:    sw a1, 296(sp)
-; CHECK-NEXT:    sw s4, 324(sp)
-; CHECK-NEXT:    sw s3, 320(sp)
-; CHECK-NEXT:    sw s2, 316(sp)
-; CHECK-NEXT:    addi a0, sp, 328
-; CHECK-NEXT:    addi a1, sp, 312
-; CHECK-NEXT:    addi a2, sp, 296
-; CHECK-NEXT:    sw s1, 312(sp)
+; CHECK-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 312(sp)
+; CHECK-NEXT:    sw a2, 316(sp)
+; CHECK-NEXT:    sw a3, 320(sp)
+; CHECK-NEXT:    sw a0, 324(sp)
+; CHECK-NEXT:    addi a0, sp, 344
+; CHECK-NEXT:    addi a1, sp, 328
+; CHECK-NEXT:    addi a2, sp, 312
+; CHECK-NEXT:    sw s1, 328(sp)
+; CHECK-NEXT:    sw s2, 332(sp)
+; CHECK-NEXT:    sw s3, 336(sp)
+; CHECK-NEXT:    sw s4, 340(sp)
 ; CHECK-NEXT:    call __multf3
-; CHECK-NEXT:    lw a0, 328(sp)
-; CHECK-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lw a0, 332(sp)
-; CHECK-NEXT:    sw a0, 36(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lw a0, 336(sp)
-; CHECK-NEXT:    sw a0, 32(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lw a0, 340(sp)
-; CHECK-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw s0, 468(sp)
-; CHECK-NEXT:    sw s8, 464(sp)
-; CHECK-NEXT:    sw s7, 460(sp)
-; CHECK-NEXT:    sw s6, 456(sp)
-; CHECK-NEXT:    sw s10, 452(sp)
-; CHECK-NEXT:    sw s5, 448(sp)
-; CHECK-NEXT:    sw s11, 444(sp)
-; CHECK-NEXT:    addi a0, sp, 472
-; CHECK-NEXT:    addi a1, sp, 456
-; CHECK-NEXT:    addi a2, sp, 440
-; CHECK-NEXT:    sw s9, 440(sp)
+; CHECK-NEXT:    lw a0, 344(sp)
+; CHECK-NEXT:    sw a0, 68(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 348(sp)
+; CHECK-NEXT:    sw a0, 64(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 352(sp)
+; CHECK-NEXT:    sw a0, 60(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 356(sp)
+; CHECK-NEXT:    sw a0, 56(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s9, 472(sp)
+; CHECK-NEXT:    sw s10, 476(sp)
+; CHECK-NEXT:    sw s11, 480(sp)
+; CHECK-NEXT:    sw s5, 484(sp)
+; CHECK-NEXT:    addi a0, sp, 488
+; CHECK-NEXT:    addi a1, sp, 472
+; CHECK-NEXT:    addi a2, sp, 456
+; CHECK-NEXT:    sw s0, 456(sp)
+; CHECK-NEXT:    sw s6, 460(sp)
+; CHECK-NEXT:    sw s7, 464(sp)
+; CHECK-NEXT:    sw s8, 468(sp)
 ; CHECK-NEXT:    call __addtf3
-; CHECK-NEXT:    lw a3, 472(sp)
-; CHECK-NEXT:    lw a0, 476(sp)
-; CHECK-NEXT:    lw a1, 480(sp)
-; CHECK-NEXT:    lw a2, 484(sp)
-; CHECK-NEXT:    sw zero, 420(sp)
-; CHECK-NEXT:    sw zero, 416(sp)
-; CHECK-NEXT:    sw zero, 412(sp)
-; CHECK-NEXT:    sw zero, 408(sp)
-; CHECK-NEXT:    sw a2, 404(sp)
-; CHECK-NEXT:    sw a1, 400(sp)
-; CHECK-NEXT:    sw a0, 396(sp)
-; CHECK-NEXT:    addi a0, sp, 424
-; CHECK-NEXT:    addi a1, sp, 408
-; CHECK-NEXT:    addi a2, sp, 392
-; CHECK-NEXT:    sw a3, 392(sp)
+; CHECK-NEXT:    lw a3, 488(sp)
+; CHECK-NEXT:    lw a4, 492(sp)
+; CHECK-NEXT:    lw a5, 496(sp)
+; CHECK-NEXT:    lw a6, 500(sp)
+; CHECK-NEXT:    sw zero, 424(sp)
+; CHECK-NEXT:    sw zero, 428(sp)
+; CHECK-NEXT:    sw zero, 432(sp)
+; CHECK-NEXT:    sw zero, 436(sp)
+; CHECK-NEXT:    addi a0, sp, 440
+; CHECK-NEXT:    addi a1, sp, 424
+; CHECK-NEXT:    addi a2, sp, 408
+; CHECK-NEXT:    sw a3, 408(sp)
+; CHECK-NEXT:    sw a4, 412(sp)
+; CHECK-NEXT:    sw a5, 416(sp)
+; CHECK-NEXT:    sw a6, 420(sp)
 ; CHECK-NEXT:    call __subtf3
-; CHECK-NEXT:    lw a0, 432(sp)
-; CHECK-NEXT:    lw a1, 436(sp)
-; CHECK-NEXT:    lw a2, 424(sp)
-; CHECK-NEXT:    lw a3, 428(sp)
+; CHECK-NEXT:    lw a0, 448(sp)
+; CHECK-NEXT:    lw a1, 452(sp)
+; CHECK-NEXT:    lw a2, 440(sp)
+; CHECK-NEXT:    lw a3, 444(sp)
 ; CHECK-NEXT:    lui a4, %hi(X)
 ; CHECK-NEXT:    sw a1, %lo(X+12)(a4)
 ; CHECK-NEXT:    sw a0, %lo(X+8)(a4)
 ; CHECK-NEXT:    sw a3, %lo(X+4)(a4)
 ; CHECK-NEXT:    sw a2, %lo(X)(a4)
-; CHECK-NEXT:    lw s8, 4(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw s8, 212(sp)
-; CHECK-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw s4, 208(sp)
-; CHECK-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw s3, 204(sp)
-; CHECK-NEXT:    lw a0, 52(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 200(sp)
-; CHECK-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 228(sp)
-; CHECK-NEXT:    lw s10, 20(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw s5, 216(sp)
+; CHECK-NEXT:    lw s9, 16(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw s9, 220(sp)
+; CHECK-NEXT:    lw s10, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw s10, 224(sp)
-; CHECK-NEXT:    lw s2, 28(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw s2, 220(sp)
-; CHECK-NEXT:    addi a0, sp, 232
-; CHECK-NEXT:    addi a1, sp, 216
-; CHECK-NEXT:    addi a2, sp, 200
-; CHECK-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw s0, 216(sp)
+; CHECK-NEXT:    lw s11, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw s11, 228(sp)
+; CHECK-NEXT:    addi a0, sp, 248
+; CHECK-NEXT:    addi a1, sp, 232
+; CHECK-NEXT:    addi a2, sp, 216
+; CHECK-NEXT:    lw s1, 52(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw s1, 232(sp)
+; CHECK-NEXT:    lw s2, 48(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw s2, 236(sp)
+; CHECK-NEXT:    lw s3, 44(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw s3, 240(sp)
+; CHECK-NEXT:    lw s4, 40(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw s4, 244(sp)
 ; CHECK-NEXT:    call __multf3
-; CHECK-NEXT:    lw s1, 232(sp)
-; CHECK-NEXT:    lw a0, 236(sp)
-; CHECK-NEXT:    sw a0, 0(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lw s6, 240(sp)
-; CHECK-NEXT:    lw s7, 244(sp)
-; CHECK-NEXT:    sw zero, 356(sp)
-; CHECK-NEXT:    sw zero, 352(sp)
-; CHECK-NEXT:    sw zero, 348(sp)
-; CHECK-NEXT:    sw zero, 344(sp)
-; CHECK-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 372(sp)
-; CHECK-NEXT:    sw s5, 368(sp)
-; CHECK-NEXT:    sw s11, 364(sp)
-; CHECK-NEXT:    addi a0, sp, 376
-; CHECK-NEXT:    addi a1, sp, 360
-; CHECK-NEXT:    addi a2, sp, 344
-; CHECK-NEXT:    sw s9, 360(sp)
+; CHECK-NEXT:    lw a0, 248(sp)
+; CHECK-NEXT:    sw a0, 36(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 252(sp)
+; CHECK-NEXT:    sw a0, 32(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 256(sp)
+; CHECK-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    lw a0, 260(sp)
+; CHECK-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw zero, 360(sp)
+; CHECK-NEXT:    sw zero, 364(sp)
+; CHECK-NEXT:    sw zero, 368(sp)
+; CHECK-NEXT:    sw zero, 372(sp)
+; CHECK-NEXT:    addi a0, sp, 392
+; CHECK-NEXT:    addi a1, sp, 376
+; CHECK-NEXT:    addi a2, sp, 360
+; CHECK-NEXT:    sw s0, 376(sp)
+; CHECK-NEXT:    sw s6, 380(sp)
+; CHECK-NEXT:    sw s7, 384(sp)
+; CHECK-NEXT:    sw s8, 388(sp)
 ; CHECK-NEXT:    call __multf3
-; CHECK-NEXT:    lw a0, 384(sp)
-; CHECK-NEXT:    lw a1, 388(sp)
-; CHECK-NEXT:    lw a2, 376(sp)
-; CHECK-NEXT:    lw a3, 380(sp)
+; CHECK-NEXT:    lw a0, 400(sp)
+; CHECK-NEXT:    lw a1, 404(sp)
+; CHECK-NEXT:    lw a2, 392(sp)
+; CHECK-NEXT:    lw a3, 396(sp)
 ; CHECK-NEXT:    lui a4, %hi(S)
 ; CHECK-NEXT:    sw a1, %lo(S+12)(a4)
 ; CHECK-NEXT:    sw a0, %lo(S+8)(a4)
 ; CHECK-NEXT:    sw a3, %lo(S+4)(a4)
 ; CHECK-NEXT:    sw a2, %lo(S)(a4)
-; CHECK-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 260(sp)
-; CHECK-NEXT:    sw s10, 256(sp)
-; CHECK-NEXT:    sw s2, 252(sp)
-; CHECK-NEXT:    sw s0, 248(sp)
-; CHECK-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 276(sp)
-; CHECK-NEXT:    lw a0, 32(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 272(sp)
-; CHECK-NEXT:    lw a0, 36(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 268(sp)
-; CHECK-NEXT:    addi a0, sp, 280
-; CHECK-NEXT:    addi a1, sp, 264
-; CHECK-NEXT:    addi a2, sp, 248
-; CHECK-NEXT:    lw a3, 44(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a3, 264(sp)
+; CHECK-NEXT:    sw s1, 264(sp)
+; CHECK-NEXT:    sw s2, 268(sp)
+; CHECK-NEXT:    sw s3, 272(sp)
+; CHECK-NEXT:    sw s4, 276(sp)
+; CHECK-NEXT:    addi a0, sp, 296
+; CHECK-NEXT:    addi a1, sp, 280
+; CHECK-NEXT:    addi a2, sp, 264
+; CHECK-NEXT:    lw a3, 68(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 280(sp)
+; CHECK-NEXT:    lw a3, 64(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 284(sp)
+; CHECK-NEXT:    lw a3, 60(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 288(sp)
+; CHECK-NEXT:    lw a3, 56(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 292(sp)
 ; CHECK-NEXT:    call __subtf3
-; CHECK-NEXT:    lw a0, 288(sp)
-; CHECK-NEXT:    lw a1, 292(sp)
-; CHECK-NEXT:    lw a2, 280(sp)
-; CHECK-NEXT:    lw a3, 284(sp)
+; CHECK-NEXT:    lw a0, 304(sp)
+; CHECK-NEXT:    lw a1, 308(sp)
+; CHECK-NEXT:    lw a2, 296(sp)
+; CHECK-NEXT:    lw a3, 300(sp)
 ; CHECK-NEXT:    lui a4, %hi(T)
 ; CHECK-NEXT:    sw a1, %lo(T+12)(a4)
 ; CHECK-NEXT:    sw a0, %lo(T+8)(a4)
 ; CHECK-NEXT:    sw a3, %lo(T+4)(a4)
 ; CHECK-NEXT:    sw a2, %lo(T)(a4)
-; CHECK-NEXT:    sw zero, 164(sp)
-; CHECK-NEXT:    sw zero, 160(sp)
-; CHECK-NEXT:    sw zero, 156(sp)
-; CHECK-NEXT:    sw zero, 152(sp)
-; CHECK-NEXT:    sw s7, 180(sp)
-; CHECK-NEXT:    sw s6, 176(sp)
-; CHECK-NEXT:    lw a0, 0(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 172(sp)
-; CHECK-NEXT:    addi a0, sp, 184
-; CHECK-NEXT:    addi a1, sp, 168
-; CHECK-NEXT:    addi a2, sp, 152
-; CHECK-NEXT:    sw s1, 168(sp)
+; CHECK-NEXT:    sw zero, 168(sp)
+; CHECK-NEXT:    sw zero, 172(sp)
+; CHECK-NEXT:    sw zero, 176(sp)
+; CHECK-NEXT:    sw zero, 180(sp)
+; CHECK-NEXT:    addi a0, sp, 200
+; CHECK-NEXT:    addi a1, sp, 184
+; CHECK-NEXT:    addi a2, sp, 168
+; CHECK-NEXT:    lw a3, 36(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 184(sp)
+; CHECK-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 188(sp)
+; CHECK-NEXT:    lw a3, 28(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 192(sp)
+; CHECK-NEXT:    lw a3, 24(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a3, 196(sp)
 ; CHECK-NEXT:    call __addtf3
-; CHECK-NEXT:    lw a0, 192(sp)
-; CHECK-NEXT:    lw a1, 196(sp)
-; CHECK-NEXT:    lw a2, 184(sp)
-; CHECK-NEXT:    lw a3, 188(sp)
+; CHECK-NEXT:    lw a0, 208(sp)
+; CHECK-NEXT:    lw a1, 212(sp)
+; CHECK-NEXT:    lw a2, 200(sp)
+; CHECK-NEXT:    lw a3, 204(sp)
 ; CHECK-NEXT:    lui a4, %hi(Y)
 ; CHECK-NEXT:    sw a1, %lo(Y+12)(a4)
 ; CHECK-NEXT:    sw a0, %lo(Y+8)(a4)
 ; CHECK-NEXT:    sw a3, %lo(Y+4)(a4)
 ; CHECK-NEXT:    sw a2, %lo(Y)(a4)
-; CHECK-NEXT:    sw zero, 116(sp)
-; CHECK-NEXT:    sw zero, 112(sp)
-; CHECK-NEXT:    sw zero, 108(sp)
-; CHECK-NEXT:    sw zero, 104(sp)
-; CHECK-NEXT:    sw s8, 132(sp)
-; CHECK-NEXT:    sw s4, 128(sp)
-; CHECK-NEXT:    sw s3, 124(sp)
-; CHECK-NEXT:    addi a0, sp, 136
-; CHECK-NEXT:    addi a1, sp, 120
-; CHECK-NEXT:    addi a2, sp, 104
-; CHECK-NEXT:    lw a3, 52(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a3, 120(sp)
+; CHECK-NEXT:    sw zero, 120(sp)
+; CHECK-NEXT:    sw zero, 124(sp)
+; CHECK-NEXT:    sw zero, 128(sp)
+; CHECK-NEXT:    sw zero, 132(sp)
+; CHECK-NEXT:    addi a0, sp, 152
+; CHECK-NEXT:    addi a1, sp, 136
+; CHECK-NEXT:    addi a2, sp, 120
+; CHECK-NEXT:    sw s5, 136(sp)
+; CHECK-NEXT:    sw s9, 140(sp)
+; CHECK-NEXT:    sw s10, 144(sp)
+; CHECK-NEXT:    sw s11, 148(sp)
 ; CHECK-NEXT:    call __multf3
-; CHECK-NEXT:    lw a3, 136(sp)
-; CHECK-NEXT:    lw a0, 140(sp)
-; CHECK-NEXT:    lw a1, 144(sp)
-; CHECK-NEXT:    lw a2, 148(sp)
-; CHECK-NEXT:    lui a4, 786400
-; CHECK-NEXT:    sw a4, 68(sp)
-; CHECK-NEXT:    sw zero, 64(sp)
-; CHECK-NEXT:    sw zero, 60(sp)
-; CHECK-NEXT:    sw zero, 56(sp)
-; CHECK-NEXT:    sw a2, 84(sp)
-; CHECK-NEXT:    sw a1, 80(sp)
-; CHECK-NEXT:    sw a0, 76(sp)
-; CHECK-NEXT:    addi a0, sp, 88
-; CHECK-NEXT:    addi a1, sp, 72
-; CHECK-NEXT:    addi a2, sp, 56
-; CHECK-NEXT:    sw a3, 72(sp)
+; CHECK-NEXT:    lw a3, 152(sp)
+; CHECK-NEXT:    lw a4, 156(sp)
+; CHECK-NEXT:    lw a5, 160(sp)
+; CHECK-NEXT:    lw a6, 164(sp)
+; CHECK-NEXT:    lui a0, 786400
+; CHECK-NEXT:    sw zero, 72(sp)
+; CHECK-NEXT:    sw zero, 76(sp)
+; CHECK-NEXT:    sw zero, 80(sp)
+; CHECK-NEXT:    sw a0, 84(sp)
+; CHECK-NEXT:    addi a0, sp, 104
+; CHECK-NEXT:    addi a1, sp, 88
+; CHECK-NEXT:    addi a2, sp, 72
+; CHECK-NEXT:    sw a3, 88(sp)
+; CHECK-NEXT:    sw a4, 92(sp)
+; CHECK-NEXT:    sw a5, 96(sp)
+; CHECK-NEXT:    sw a6, 100(sp)
 ; CHECK-NEXT:    call __addtf3
-; CHECK-NEXT:    lw a0, 96(sp)
-; CHECK-NEXT:    lw a1, 100(sp)
-; CHECK-NEXT:    lw a2, 88(sp)
-; CHECK-NEXT:    lw a3, 92(sp)
+; CHECK-NEXT:    lw a0, 112(sp)
+; CHECK-NEXT:    lw a1, 116(sp)
+; CHECK-NEXT:    lw a2, 104(sp)
+; CHECK-NEXT:    lw a3, 108(sp)
 ; CHECK-NEXT:    lui a4, %hi(Y1)
 ; CHECK-NEXT:    sw a0, %lo(Y1+8)(a4)
 ; CHECK-NEXT:    sw a1, %lo(Y1+12)(a4)
 ; CHECK-NEXT:    sw a2, %lo(Y1)(a4)
 ; CHECK-NEXT:    sw a3, %lo(Y1+4)(a4)
-; CHECK-NEXT:    lw ra, 684(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s0, 680(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s1, 676(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s2, 672(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s3, 668(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s4, 664(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s5, 660(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s6, 656(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s7, 652(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s8, 648(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s9, 644(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s10, 640(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    lw s11, 636(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    addi sp, sp, 688
+; CHECK-NEXT:    lw ra, 700(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s0, 696(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s1, 692(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s2, 688(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s3, 684(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s4, 680(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s5, 676(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s6, 672(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s7, 668(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s8, 664(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s9, 660(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s10, 656(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s11, 652(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 704
 ; CHECK-NEXT:    ret
   %1 = load fp128, ptr @U, align 16
   %2 = fsub fp128 0xL00000000000000000000000000000000, %1
diff --git a/llvm/test/CodeGen/RISCV/tail-calls.ll b/llvm/test/CodeGen/RISCV/tail-calls.ll
index 4dd6ed68ff9811284ae3ae2032ac009aab28d1b1..f7ef01b0958d87f2865ff13e0d7cb43e3bcc4cd4 100644
--- a/llvm/test/CodeGen/RISCV/tail-calls.ll
+++ b/llvm/test/CodeGen/RISCV/tail-calls.ll
@@ -213,12 +213,12 @@ define i32 @caller_args(i32 %a, i32 %b, i32 %c, i32 %dd, i32 %e, i32 %ff, i32 %g
 ; CHECK-NEXT:    lw t3, 44(sp)
 ; CHECK-NEXT:    lw t4, 48(sp)
 ; CHECK-NEXT:    lw t5, 52(sp)
-; CHECK-NEXT:    sw t5, 20(sp)
 ; CHECK-NEXT:    sw t4, 16(sp)
-; CHECK-NEXT:    sw t3, 12(sp)
-; CHECK-NEXT:    sw t2, 8(sp)
-; CHECK-NEXT:    sw t1, 4(sp)
+; CHECK-NEXT:    sw t5, 20(sp)
 ; CHECK-NEXT:    sw t0, 0(sp)
+; CHECK-NEXT:    sw t1, 4(sp)
+; CHECK-NEXT:    sw t2, 8(sp)
+; CHECK-NEXT:    sw t3, 12(sp)
 ; CHECK-NEXT:    call callee_args
 ; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
@@ -235,15 +235,15 @@ define i32 @caller_args(i32 %a, i32 %b, i32 %c, i32 %dd, i32 %e, i32 %ff, i32 %g
 ; CHECK-LARGE-ZICFILP-NEXT:    lw t4, 44(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:    lw t2, 48(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:    lw t5, 52(sp)
-; CHECK-LARGE-ZICFILP-NEXT:    sw t5, 20(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:    sw t2, 16(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t5, 20(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi8:
 ; CHECK-LARGE-ZICFILP-NEXT:    auipc t2, %pcrel_hi(.LCPI6_0)
 ; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi8)(t2)
-; CHECK-LARGE-ZICFILP-NEXT:    sw t4, 12(sp)
-; CHECK-LARGE-ZICFILP-NEXT:    sw t3, 8(sp)
-; CHECK-LARGE-ZICFILP-NEXT:    sw t1, 4(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:    sw t0, 0(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t1, 4(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t3, 8(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t4, 12(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
 ; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 32
@@ -260,12 +260,12 @@ define void @caller_indirect_args() nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -32
 ; CHECK-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    lui a0, 262128
-; CHECK-NEXT:    sw a0, 12(sp)
-; CHECK-NEXT:    sw zero, 8(sp)
-; CHECK-NEXT:    sw zero, 4(sp)
+; CHECK-NEXT:    lui a1, 262128
 ; CHECK-NEXT:    mv a0, sp
 ; CHECK-NEXT:    sw zero, 0(sp)
+; CHECK-NEXT:    sw zero, 4(sp)
+; CHECK-NEXT:    sw zero, 8(sp)
+; CHECK-NEXT:    sw a1, 12(sp)
 ; CHECK-NEXT:    call callee_indirect_args
 ; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
@@ -276,15 +276,15 @@ define void @caller_indirect_args() nounwind {
 ; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
 ; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -32
 ; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; CHECK-LARGE-ZICFILP-NEXT:    lui a0, 262128
-; CHECK-LARGE-ZICFILP-NEXT:    sw a0, 12(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    lui a1, 262128
 ; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi9:
 ; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI7_0)
 ; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi9)(a0)
-; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 8(sp)
-; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 4(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:    mv a0, sp
 ; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 0(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 4(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 8(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw a1, 12(sp)
 ; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
 ; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
index 0ca9fc065ff2d4db0ff2b6f58b61e1ea13d74dfa..a2f5e446b63bcd09de52a6d26d674783908bc77c 100644
--- a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
@@ -247,9 +247,9 @@ define void @store_i8(ptr %p, i8 %v) {
 define void @store_i16(ptr %p, i16 %v) {
 ; SLOW-LABEL: store_i16:
 ; SLOW:       # %bb.0:
+; SLOW-NEXT:    srli a2, a1, 8
 ; SLOW-NEXT:    sb a1, 0(a0)
-; SLOW-NEXT:    srli a1, a1, 8
-; SLOW-NEXT:    sb a1, 1(a0)
+; SLOW-NEXT:    sb a2, 1(a0)
 ; SLOW-NEXT:    ret
 ;
 ; FAST-LABEL: store_i16:
@@ -263,18 +263,18 @@ define void @store_i16(ptr %p, i16 %v) {
 define void @store_i24(ptr %p, i24 %v) {
 ; SLOW-LABEL: store_i24:
 ; SLOW:       # %bb.0:
-; SLOW-NEXT:    sb a1, 0(a0)
 ; SLOW-NEXT:    srli a2, a1, 8
+; SLOW-NEXT:    srli a3, a1, 16
+; SLOW-NEXT:    sb a1, 0(a0)
 ; SLOW-NEXT:    sb a2, 1(a0)
-; SLOW-NEXT:    srli a1, a1, 16
-; SLOW-NEXT:    sb a1, 2(a0)
+; SLOW-NEXT:    sb a3, 2(a0)
 ; SLOW-NEXT:    ret
 ;
 ; FAST-LABEL: store_i24:
 ; FAST:       # %bb.0:
+; FAST-NEXT:    srli a2, a1, 16
 ; FAST-NEXT:    sh a1, 0(a0)
-; FAST-NEXT:    srli a1, a1, 16
-; FAST-NEXT:    sb a1, 2(a0)
+; FAST-NEXT:    sb a2, 2(a0)
 ; FAST-NEXT:    ret
   store i24 %v, ptr %p, align 1
   ret void
@@ -283,13 +283,13 @@ define void @store_i24(ptr %p, i24 %v) {
 define void @store_i32(ptr %p, i32 %v) {
 ; SLOW-LABEL: store_i32:
 ; SLOW:       # %bb.0:
-; SLOW-NEXT:    sb a1, 0(a0)
 ; SLOW-NEXT:    srli a2, a1, 24
+; SLOW-NEXT:    srli a3, a1, 16
+; SLOW-NEXT:    srli a4, a1, 8
+; SLOW-NEXT:    sb a1, 0(a0)
+; SLOW-NEXT:    sb a4, 1(a0)
+; SLOW-NEXT:    sb a3, 2(a0)
 ; SLOW-NEXT:    sb a2, 3(a0)
-; SLOW-NEXT:    srli a2, a1, 16
-; SLOW-NEXT:    sb a2, 2(a0)
-; SLOW-NEXT:    srli a1, a1, 8
-; SLOW-NEXT:    sb a1, 1(a0)
 ; SLOW-NEXT:    ret
 ;
 ; FAST-LABEL: store_i32:
@@ -303,82 +303,82 @@ define void @store_i32(ptr %p, i32 %v) {
 define void @store_i64(ptr %p, i64 %v) {
 ; RV32I-LABEL: store_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sb a2, 4(a0)
-; RV32I-NEXT:    sb a1, 0(a0)
 ; RV32I-NEXT:    srli a3, a2, 24
+; RV32I-NEXT:    srli a4, a2, 16
+; RV32I-NEXT:    srli a5, a2, 8
+; RV32I-NEXT:    sb a2, 4(a0)
+; RV32I-NEXT:    sb a5, 5(a0)
+; RV32I-NEXT:    sb a4, 6(a0)
 ; RV32I-NEXT:    sb a3, 7(a0)
-; RV32I-NEXT:    srli a3, a2, 16
-; RV32I-NEXT:    sb a3, 6(a0)
-; RV32I-NEXT:    srli a2, a2, 8
-; RV32I-NEXT:    sb a2, 5(a0)
 ; RV32I-NEXT:    srli a2, a1, 24
+; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    srli a4, a1, 8
+; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    sb a4, 1(a0)
+; RV32I-NEXT:    sb a3, 2(a0)
 ; RV32I-NEXT:    sb a2, 3(a0)
-; RV32I-NEXT:    srli a2, a1, 16
-; RV32I-NEXT:    sb a2, 2(a0)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: store_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sb a1, 0(a0)
 ; RV64I-NEXT:    srli a2, a1, 56
+; RV64I-NEXT:    srli a3, a1, 48
+; RV64I-NEXT:    srli a4, a1, 40
+; RV64I-NEXT:    srli a5, a1, 32
+; RV64I-NEXT:    sb a5, 4(a0)
+; RV64I-NEXT:    sb a4, 5(a0)
+; RV64I-NEXT:    sb a3, 6(a0)
 ; RV64I-NEXT:    sb a2, 7(a0)
-; RV64I-NEXT:    srli a2, a1, 48
-; RV64I-NEXT:    sb a2, 6(a0)
-; RV64I-NEXT:    srli a2, a1, 40
-; RV64I-NEXT:    sb a2, 5(a0)
-; RV64I-NEXT:    srli a2, a1, 32
-; RV64I-NEXT:    sb a2, 4(a0)
 ; RV64I-NEXT:    srli a2, a1, 24
+; RV64I-NEXT:    srli a3, a1, 16
+; RV64I-NEXT:    srli a4, a1, 8
+; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    sb a4, 1(a0)
+; RV64I-NEXT:    sb a3, 2(a0)
 ; RV64I-NEXT:    sb a2, 3(a0)
-; RV64I-NEXT:    srli a2, a1, 16
-; RV64I-NEXT:    sb a2, 2(a0)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 1(a0)
 ; RV64I-NEXT:    ret
 ;
 ; RV32IZBKB-LABEL: store_i64:
 ; RV32IZBKB:       # %bb.0:
-; RV32IZBKB-NEXT:    sb a2, 4(a0)
-; RV32IZBKB-NEXT:    sb a1, 0(a0)
 ; RV32IZBKB-NEXT:    srli a3, a2, 24
+; RV32IZBKB-NEXT:    srli a4, a2, 16
+; RV32IZBKB-NEXT:    srli a5, a2, 8
+; RV32IZBKB-NEXT:    sb a2, 4(a0)
+; RV32IZBKB-NEXT:    sb a5, 5(a0)
+; RV32IZBKB-NEXT:    sb a4, 6(a0)
 ; RV32IZBKB-NEXT:    sb a3, 7(a0)
-; RV32IZBKB-NEXT:    srli a3, a2, 16
-; RV32IZBKB-NEXT:    sb a3, 6(a0)
-; RV32IZBKB-NEXT:    srli a2, a2, 8
-; RV32IZBKB-NEXT:    sb a2, 5(a0)
 ; RV32IZBKB-NEXT:    srli a2, a1, 24
+; RV32IZBKB-NEXT:    srli a3, a1, 16
+; RV32IZBKB-NEXT:    srli a4, a1, 8
+; RV32IZBKB-NEXT:    sb a1, 0(a0)
+; RV32IZBKB-NEXT:    sb a4, 1(a0)
+; RV32IZBKB-NEXT:    sb a3, 2(a0)
 ; RV32IZBKB-NEXT:    sb a2, 3(a0)
-; RV32IZBKB-NEXT:    srli a2, a1, 16
-; RV32IZBKB-NEXT:    sb a2, 2(a0)
-; RV32IZBKB-NEXT:    srli a1, a1, 8
-; RV32IZBKB-NEXT:    sb a1, 1(a0)
 ; RV32IZBKB-NEXT:    ret
 ;
 ; RV64IZBKB-LABEL: store_i64:
 ; RV64IZBKB:       # %bb.0:
-; RV64IZBKB-NEXT:    sb a1, 0(a0)
 ; RV64IZBKB-NEXT:    srli a2, a1, 56
+; RV64IZBKB-NEXT:    srli a3, a1, 48
+; RV64IZBKB-NEXT:    srli a4, a1, 40
+; RV64IZBKB-NEXT:    srli a5, a1, 32
+; RV64IZBKB-NEXT:    sb a5, 4(a0)
+; RV64IZBKB-NEXT:    sb a4, 5(a0)
+; RV64IZBKB-NEXT:    sb a3, 6(a0)
 ; RV64IZBKB-NEXT:    sb a2, 7(a0)
-; RV64IZBKB-NEXT:    srli a2, a1, 48
-; RV64IZBKB-NEXT:    sb a2, 6(a0)
-; RV64IZBKB-NEXT:    srli a2, a1, 40
-; RV64IZBKB-NEXT:    sb a2, 5(a0)
-; RV64IZBKB-NEXT:    srli a2, a1, 32
-; RV64IZBKB-NEXT:    sb a2, 4(a0)
 ; RV64IZBKB-NEXT:    srli a2, a1, 24
+; RV64IZBKB-NEXT:    srli a3, a1, 16
+; RV64IZBKB-NEXT:    srli a4, a1, 8
+; RV64IZBKB-NEXT:    sb a1, 0(a0)
+; RV64IZBKB-NEXT:    sb a4, 1(a0)
+; RV64IZBKB-NEXT:    sb a3, 2(a0)
 ; RV64IZBKB-NEXT:    sb a2, 3(a0)
-; RV64IZBKB-NEXT:    srli a2, a1, 16
-; RV64IZBKB-NEXT:    sb a2, 2(a0)
-; RV64IZBKB-NEXT:    srli a1, a1, 8
-; RV64IZBKB-NEXT:    sb a1, 1(a0)
 ; RV64IZBKB-NEXT:    ret
 ;
 ; RV32I-FAST-LABEL: store_i64:
 ; RV32I-FAST:       # %bb.0:
-; RV32I-FAST-NEXT:    sw a2, 4(a0)
 ; RV32I-FAST-NEXT:    sw a1, 0(a0)
+; RV32I-FAST-NEXT:    sw a2, 4(a0)
 ; RV32I-FAST-NEXT:    ret
 ;
 ; RV64I-FAST-LABEL: store_i64:
@@ -543,31 +543,31 @@ define void @store_large_constant(ptr %x) {
 ; SLOW-LABEL: store_large_constant:
 ; SLOW:       # %bb.0:
 ; SLOW-NEXT:    li a1, -2
+; SLOW-NEXT:    li a2, 220
+; SLOW-NEXT:    li a3, 186
+; SLOW-NEXT:    li a4, 152
+; SLOW-NEXT:    sb a4, 4(a0)
+; SLOW-NEXT:    sb a3, 5(a0)
+; SLOW-NEXT:    sb a2, 6(a0)
 ; SLOW-NEXT:    sb a1, 7(a0)
-; SLOW-NEXT:    li a1, 220
-; SLOW-NEXT:    sb a1, 6(a0)
-; SLOW-NEXT:    li a1, 186
-; SLOW-NEXT:    sb a1, 5(a0)
-; SLOW-NEXT:    li a1, 152
-; SLOW-NEXT:    sb a1, 4(a0)
 ; SLOW-NEXT:    li a1, 118
+; SLOW-NEXT:    li a2, 84
+; SLOW-NEXT:    li a3, 50
+; SLOW-NEXT:    li a4, 16
+; SLOW-NEXT:    sb a4, 0(a0)
+; SLOW-NEXT:    sb a3, 1(a0)
+; SLOW-NEXT:    sb a2, 2(a0)
 ; SLOW-NEXT:    sb a1, 3(a0)
-; SLOW-NEXT:    li a1, 84
-; SLOW-NEXT:    sb a1, 2(a0)
-; SLOW-NEXT:    li a1, 50
-; SLOW-NEXT:    sb a1, 1(a0)
-; SLOW-NEXT:    li a1, 16
-; SLOW-NEXT:    sb a1, 0(a0)
 ; SLOW-NEXT:    ret
 ;
 ; RV32I-FAST-LABEL: store_large_constant:
 ; RV32I-FAST:       # %bb.0:
 ; RV32I-FAST-NEXT:    lui a1, 1043916
 ; RV32I-FAST-NEXT:    addi a1, a1, -1384
+; RV32I-FAST-NEXT:    lui a2, 484675
+; RV32I-FAST-NEXT:    addi a2, a2, 528
+; RV32I-FAST-NEXT:    sw a2, 0(a0)
 ; RV32I-FAST-NEXT:    sw a1, 4(a0)
-; RV32I-FAST-NEXT:    lui a1, 484675
-; RV32I-FAST-NEXT:    addi a1, a1, 528
-; RV32I-FAST-NEXT:    sw a1, 0(a0)
 ; RV32I-FAST-NEXT:    ret
 ;
 ; RV64I-FAST-LABEL: store_large_constant:
diff --git a/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll b/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
index 5a5ae66b5fa767bc3817e29b702fe01328628337..0ee067b673da9ac0581515eb8729362abd5b2977 100644
--- a/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
@@ -362,14 +362,14 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV32-NEXT:    addi s1, s1, -1
 ; RV32-NEXT:    slli a1, s1, 21
 ; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 4(s0)
-; RV32-NEXT:    andi a1, s3, 2047
+; RV32-NEXT:    andi a2, s3, 2047
 ; RV32-NEXT:    andi a0, a0, 2047
 ; RV32-NEXT:    slli a0, a0, 11
 ; RV32-NEXT:    slli s1, s1, 22
 ; RV32-NEXT:    or a0, a0, s1
-; RV32-NEXT:    or a0, a1, a0
+; RV32-NEXT:    or a0, a2, a0
 ; RV32-NEXT:    sw a0, 0(s0)
+; RV32-NEXT:    sb a1, 4(s0)
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -423,10 +423,10 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV64-NEXT:    slli a0, a0, 22
 ; RV64-NEXT:    or a0, a2, a0
 ; RV64-NEXT:    or a0, a1, a0
+; RV64-NEXT:    slli a1, a0, 31
+; RV64-NEXT:    srli a1, a1, 63
 ; RV64-NEXT:    sw a0, 0(s0)
-; RV64-NEXT:    slli a0, a0, 31
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 4(s0)
+; RV64-NEXT:    sb a1, 4(s0)
 ; RV64-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -467,7 +467,6 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV32M-NEXT:    addi a1, a1, -1
 ; RV32M-NEXT:    slli a4, a1, 21
 ; RV32M-NEXT:    srli a4, a4, 31
-; RV32M-NEXT:    sb a4, 4(a0)
 ; RV32M-NEXT:    andi a2, a2, 2047
 ; RV32M-NEXT:    andi a3, a3, 2047
 ; RV32M-NEXT:    slli a3, a3, 11
@@ -475,6 +474,7 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV32M-NEXT:    or a1, a3, a1
 ; RV32M-NEXT:    or a1, a2, a1
 ; RV32M-NEXT:    sw a1, 0(a0)
+; RV32M-NEXT:    sb a4, 4(a0)
 ; RV32M-NEXT:    ret
 ;
 ; RV64M-LABEL: test_urem_vec:
@@ -513,10 +513,10 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV64M-NEXT:    slli a2, a2, 22
 ; RV64M-NEXT:    or a2, a3, a2
 ; RV64M-NEXT:    or a1, a1, a2
+; RV64M-NEXT:    slli a2, a1, 31
+; RV64M-NEXT:    srli a2, a2, 63
 ; RV64M-NEXT:    sw a1, 0(a0)
-; RV64M-NEXT:    slli a1, a1, 31
-; RV64M-NEXT:    srli a1, a1, 63
-; RV64M-NEXT:    sb a1, 4(a0)
+; RV64M-NEXT:    sb a2, 4(a0)
 ; RV64M-NEXT:    ret
 ;
 ; RV32MV-LABEL: test_urem_vec:
@@ -568,17 +568,17 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    vmv.x.s a1, v9
 ; RV32MV-NEXT:    slli a2, a1, 21
 ; RV32MV-NEXT:    srli a2, a2, 31
-; RV32MV-NEXT:    sb a2, 4(a0)
-; RV32MV-NEXT:    vmv.x.s a2, v8
-; RV32MV-NEXT:    andi a2, a2, 2047
+; RV32MV-NEXT:    vmv.x.s a3, v8
+; RV32MV-NEXT:    andi a3, a3, 2047
 ; RV32MV-NEXT:    vslidedown.vi v8, v8, 1
+; RV32MV-NEXT:    slli a1, a1, 22
+; RV32MV-NEXT:    or a1, a3, a1
 ; RV32MV-NEXT:    vmv.x.s a3, v8
 ; RV32MV-NEXT:    andi a3, a3, 2047
 ; RV32MV-NEXT:    slli a3, a3, 11
-; RV32MV-NEXT:    slli a1, a1, 22
-; RV32MV-NEXT:    or a1, a2, a1
 ; RV32MV-NEXT:    or a1, a1, a3
 ; RV32MV-NEXT:    sw a1, 0(a0)
+; RV32MV-NEXT:    sb a2, 4(a0)
 ; RV32MV-NEXT:    ret
 ;
 ; RV64MV-LABEL: test_urem_vec:
@@ -636,10 +636,10 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV64MV-NEXT:    slli a3, a3, 22
 ; RV64MV-NEXT:    or a1, a1, a3
 ; RV64MV-NEXT:    or a1, a1, a2
+; RV64MV-NEXT:    slli a2, a1, 31
+; RV64MV-NEXT:    srli a2, a2, 63
 ; RV64MV-NEXT:    sw a1, 0(a0)
-; RV64MV-NEXT:    slli a1, a1, 31
-; RV64MV-NEXT:    srli a1, a1, 63
-; RV64MV-NEXT:    sb a1, 4(a0)
+; RV64MV-NEXT:    sb a2, 4(a0)
 ; RV64MV-NEXT:    ret
   %ld = load <3 x i11>, ptr %X
   %urem = urem <3 x i11> %ld, <i11 6, i11 7, i11 -5>
diff --git a/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll b/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
index b0e790ed606350285cec0e03e1b36f434cfa7b9d..01f06474f78c23fb3a007376cbcfb4981a72ed03 100644
--- a/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
+++ b/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
@@ -39,10 +39,10 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    li a1, 1003
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    call __umodsi3
-; RV32I-NEXT:    sh a0, 6(s3)
-; RV32I-NEXT:    sh s1, 4(s3)
-; RV32I-NEXT:    sh s0, 2(s3)
 ; RV32I-NEXT:    sh s4, 0(s3)
+; RV32I-NEXT:    sh s0, 2(s3)
+; RV32I-NEXT:    sh s1, 4(s3)
+; RV32I-NEXT:    sh a0, 6(s3)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -83,10 +83,10 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    li a6, 1003
 ; RV32IM-NEXT:    mul a5, a5, a6
 ; RV32IM-NEXT:    sub a1, a1, a5
-; RV32IM-NEXT:    sh a1, 6(a0)
-; RV32IM-NEXT:    sh a4, 4(a0)
 ; RV32IM-NEXT:    sh a2, 0(a0)
 ; RV32IM-NEXT:    sh a3, 2(a0)
+; RV32IM-NEXT:    sh a4, 4(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: fold_urem_vec_1:
@@ -118,10 +118,10 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    li a1, 1003
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    sh a0, 6(s3)
-; RV64I-NEXT:    sh s1, 4(s3)
-; RV64I-NEXT:    sh s0, 2(s3)
 ; RV64I-NEXT:    sh s4, 0(s3)
+; RV64I-NEXT:    sh s0, 2(s3)
+; RV64I-NEXT:    sh s1, 4(s3)
+; RV64I-NEXT:    sh a0, 6(s3)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -162,10 +162,10 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    li a6, 1003
 ; RV64IM-NEXT:    mul a3, a3, a6
 ; RV64IM-NEXT:    subw a1, a1, a3
-; RV64IM-NEXT:    sh a1, 6(a0)
-; RV64IM-NEXT:    sh a5, 4(a0)
 ; RV64IM-NEXT:    sh a4, 0(a0)
 ; RV64IM-NEXT:    sh a2, 2(a0)
+; RV64IM-NEXT:    sh a5, 4(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = urem <4 x i16> %x, <i16 95, i16 124, i16 98, i16 1003>
   ret <4 x i16> %1
@@ -201,10 +201,10 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    li a1, 95
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    call __umodsi3
-; RV32I-NEXT:    sh a0, 6(s3)
-; RV32I-NEXT:    sh s1, 4(s3)
-; RV32I-NEXT:    sh s0, 2(s3)
 ; RV32I-NEXT:    sh s4, 0(s3)
+; RV32I-NEXT:    sh s0, 2(s3)
+; RV32I-NEXT:    sh s1, 4(s3)
+; RV32I-NEXT:    sh a0, 6(s3)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -235,10 +235,10 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    mulhu a5, a1, a5
 ; RV32IM-NEXT:    mul a5, a5, a7
 ; RV32IM-NEXT:    sub a1, a1, a5
-; RV32IM-NEXT:    sh a1, 6(a0)
-; RV32IM-NEXT:    sh a4, 4(a0)
-; RV32IM-NEXT:    sh a3, 2(a0)
 ; RV32IM-NEXT:    sh a2, 0(a0)
+; RV32IM-NEXT:    sh a3, 2(a0)
+; RV32IM-NEXT:    sh a4, 4(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: fold_urem_vec_2:
@@ -270,10 +270,10 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    li a1, 95
 ; RV64I-NEXT:    mv a0, s2
 ; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    sh a0, 6(s3)
-; RV64I-NEXT:    sh s1, 4(s3)
-; RV64I-NEXT:    sh s0, 2(s3)
 ; RV64I-NEXT:    sh s4, 0(s3)
+; RV64I-NEXT:    sh s0, 2(s3)
+; RV64I-NEXT:    sh s1, 4(s3)
+; RV64I-NEXT:    sh a0, 6(s3)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -304,10 +304,10 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    mulhu a2, a1, a2
 ; RV64IM-NEXT:    mul a2, a2, a7
 ; RV64IM-NEXT:    subw a1, a1, a2
-; RV64IM-NEXT:    sh a1, 6(a0)
-; RV64IM-NEXT:    sh a5, 4(a0)
-; RV64IM-NEXT:    sh a4, 2(a0)
 ; RV64IM-NEXT:    sh a3, 0(a0)
+; RV64IM-NEXT:    sh a4, 2(a0)
+; RV64IM-NEXT:    sh a5, 4(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
   ret <4 x i16> %1
@@ -369,10 +369,10 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    add s2, s7, s2
 ; RV32I-NEXT:    add s3, s6, s3
 ; RV32I-NEXT:    add s4, s5, s4
-; RV32I-NEXT:    sh s4, 6(s0)
-; RV32I-NEXT:    sh s3, 4(s0)
-; RV32I-NEXT:    sh s2, 2(s0)
 ; RV32I-NEXT:    sh a0, 0(s0)
+; RV32I-NEXT:    sh s2, 2(s0)
+; RV32I-NEXT:    sh s3, 4(s0)
+; RV32I-NEXT:    sh s4, 6(s0)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -411,10 +411,10 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    sub a1, a1, t2
 ; RV32IM-NEXT:    add a2, a2, a6
 ; RV32IM-NEXT:    sub a2, a2, t0
-; RV32IM-NEXT:    sh a2, 6(a0)
-; RV32IM-NEXT:    sh a1, 4(a0)
-; RV32IM-NEXT:    sh a4, 2(a0)
 ; RV32IM-NEXT:    sh a3, 0(a0)
+; RV32IM-NEXT:    sh a4, 2(a0)
+; RV32IM-NEXT:    sh a1, 4(a0)
+; RV32IM-NEXT:    sh a2, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: combine_urem_udiv:
@@ -470,10 +470,10 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    add s2, s7, s2
 ; RV64I-NEXT:    add s3, s6, s3
 ; RV64I-NEXT:    add s4, s5, s4
-; RV64I-NEXT:    sh s4, 6(s0)
-; RV64I-NEXT:    sh s3, 4(s0)
-; RV64I-NEXT:    sh s2, 2(s0)
 ; RV64I-NEXT:    sh a0, 0(s0)
+; RV64I-NEXT:    sh s2, 2(s0)
+; RV64I-NEXT:    sh s3, 4(s0)
+; RV64I-NEXT:    sh s4, 6(s0)
 ; RV64I-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 56(sp) # 8-byte Folded Reload
@@ -512,10 +512,10 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    subw a1, a1, t2
 ; RV64IM-NEXT:    add a2, a2, a6
 ; RV64IM-NEXT:    subw a2, a2, t0
-; RV64IM-NEXT:    sh a2, 6(a0)
-; RV64IM-NEXT:    sh a1, 4(a0)
-; RV64IM-NEXT:    sh a4, 2(a0)
 ; RV64IM-NEXT:    sh a3, 0(a0)
+; RV64IM-NEXT:    sh a4, 2(a0)
+; RV64IM-NEXT:    sh a1, 4(a0)
+; RV64IM-NEXT:    sh a2, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = urem <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
   %2 = udiv <4 x i16> %x, <i16 95, i16 95, i16 95, i16 95>
@@ -544,10 +544,10 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    andi a1, s1, 63
 ; RV32I-NEXT:    andi a2, s2, 31
 ; RV32I-NEXT:    andi a3, s3, 7
-; RV32I-NEXT:    sh a0, 6(s0)
-; RV32I-NEXT:    sh a3, 4(s0)
-; RV32I-NEXT:    sh a2, 2(s0)
 ; RV32I-NEXT:    sh a1, 0(s0)
+; RV32I-NEXT:    sh a2, 2(s0)
+; RV32I-NEXT:    sh a3, 4(s0)
+; RV32I-NEXT:    sh a0, 6(s0)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -571,9 +571,9 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    andi a1, a1, 63
 ; RV32IM-NEXT:    andi a2, a2, 31
 ; RV32IM-NEXT:    andi a3, a3, 7
-; RV32IM-NEXT:    sh a3, 4(a0)
-; RV32IM-NEXT:    sh a2, 2(a0)
 ; RV32IM-NEXT:    sh a1, 0(a0)
+; RV32IM-NEXT:    sh a2, 2(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
 ; RV32IM-NEXT:    sh a4, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
@@ -596,10 +596,10 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    andi a1, s1, 63
 ; RV64I-NEXT:    andi a2, s2, 31
 ; RV64I-NEXT:    andi a3, s3, 7
-; RV64I-NEXT:    sh a0, 6(s0)
-; RV64I-NEXT:    sh a3, 4(s0)
-; RV64I-NEXT:    sh a2, 2(s0)
 ; RV64I-NEXT:    sh a1, 0(s0)
+; RV64I-NEXT:    sh a2, 2(s0)
+; RV64I-NEXT:    sh a3, 4(s0)
+; RV64I-NEXT:    sh a0, 6(s0)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -623,9 +623,9 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    andi a1, a1, 63
 ; RV64IM-NEXT:    andi a5, a5, 31
 ; RV64IM-NEXT:    andi a4, a4, 7
-; RV64IM-NEXT:    sh a4, 4(a0)
-; RV64IM-NEXT:    sh a5, 2(a0)
 ; RV64IM-NEXT:    sh a1, 0(a0)
+; RV64IM-NEXT:    sh a5, 2(a0)
+; RV64IM-NEXT:    sh a4, 4(a0)
 ; RV64IM-NEXT:    sh a2, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = urem <4 x i16> %x, <i16 64, i16 32, i16 8, i16 95>
@@ -658,10 +658,10 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    addi a1, a0, 1327
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __umodsi3
-; RV32I-NEXT:    sh a0, 6(s2)
-; RV32I-NEXT:    sh s0, 4(s2)
-; RV32I-NEXT:    sh s3, 2(s2)
 ; RV32I-NEXT:    sh zero, 0(s2)
+; RV32I-NEXT:    sh s3, 2(s2)
+; RV32I-NEXT:    sh s0, 4(s2)
+; RV32I-NEXT:    sh a0, 6(s2)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -695,9 +695,9 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
 ; RV32IM-NEXT:    mul a4, a4, a5
 ; RV32IM-NEXT:    sub a1, a1, a4
 ; RV32IM-NEXT:    sh zero, 0(a0)
-; RV32IM-NEXT:    sh a1, 6(a0)
-; RV32IM-NEXT:    sh a3, 4(a0)
 ; RV32IM-NEXT:    sh a2, 2(a0)
+; RV32IM-NEXT:    sh a3, 4(a0)
+; RV32IM-NEXT:    sh a1, 6(a0)
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: dont_fold_urem_one:
@@ -724,10 +724,10 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    sh a0, 6(s2)
-; RV64I-NEXT:    sh s0, 4(s2)
-; RV64I-NEXT:    sh s3, 2(s2)
 ; RV64I-NEXT:    sh zero, 0(s2)
+; RV64I-NEXT:    sh s3, 2(s2)
+; RV64I-NEXT:    sh s0, 4(s2)
+; RV64I-NEXT:    sh a0, 6(s2)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -761,9 +761,9 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
 ; RV64IM-NEXT:    mul a2, a2, a5
 ; RV64IM-NEXT:    subw a1, a1, a2
 ; RV64IM-NEXT:    sh zero, 0(a0)
-; RV64IM-NEXT:    sh a1, 6(a0)
-; RV64IM-NEXT:    sh a4, 4(a0)
 ; RV64IM-NEXT:    sh a3, 2(a0)
+; RV64IM-NEXT:    sh a4, 4(a0)
+; RV64IM-NEXT:    sh a1, 6(a0)
 ; RV64IM-NEXT:    ret
   %1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
   ret <4 x i16> %1
@@ -793,15 +793,15 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    lw s0, 16(a1)
-; RV32I-NEXT:    lw s1, 20(a1)
-; RV32I-NEXT:    lw s2, 24(a1)
-; RV32I-NEXT:    lw s3, 28(a1)
+; RV32I-NEXT:    lw s1, 16(a1)
+; RV32I-NEXT:    lw s2, 20(a1)
+; RV32I-NEXT:    lw s3, 24(a1)
+; RV32I-NEXT:    lw s4, 28(a1)
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    lw s4, 8(a1)
-; RV32I-NEXT:    lw s5, 12(a1)
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    lw s5, 8(a1)
+; RV32I-NEXT:    lw s6, 12(a1)
+; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    li a2, 1
 ; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    mv a1, a4
@@ -810,33 +810,33 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    mv s8, a1
 ; RV32I-NEXT:    li a2, 654
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s5
+; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    mv a1, s6
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __umoddi3
-; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    mv s5, a1
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s6, a1
 ; RV32I-NEXT:    li a2, 23
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __umoddi3
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    mv s1, a1
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a2, a0, 1327
-; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    mv a1, s4
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __umoddi3
-; RV32I-NEXT:    sw a1, 28(s6)
-; RV32I-NEXT:    sw a0, 24(s6)
-; RV32I-NEXT:    sw s1, 20(s6)
-; RV32I-NEXT:    sw s0, 16(s6)
-; RV32I-NEXT:    sw s5, 12(s6)
-; RV32I-NEXT:    sw s4, 8(s6)
-; RV32I-NEXT:    sw s8, 4(s6)
-; RV32I-NEXT:    sw s7, 0(s6)
+; RV32I-NEXT:    sw s1, 16(s0)
+; RV32I-NEXT:    sw s2, 20(s0)
+; RV32I-NEXT:    sw a0, 24(s0)
+; RV32I-NEXT:    sw a1, 28(s0)
+; RV32I-NEXT:    sw s7, 0(s0)
+; RV32I-NEXT:    sw s8, 4(s0)
+; RV32I-NEXT:    sw s5, 8(s0)
+; RV32I-NEXT:    sw s6, 12(s0)
 ; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -863,15 +863,15 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
-; RV32IM-NEXT:    lw s0, 16(a1)
-; RV32IM-NEXT:    lw s1, 20(a1)
-; RV32IM-NEXT:    lw s2, 24(a1)
-; RV32IM-NEXT:    lw s3, 28(a1)
+; RV32IM-NEXT:    lw s1, 16(a1)
+; RV32IM-NEXT:    lw s2, 20(a1)
+; RV32IM-NEXT:    lw s3, 24(a1)
+; RV32IM-NEXT:    lw s4, 28(a1)
 ; RV32IM-NEXT:    lw a3, 0(a1)
 ; RV32IM-NEXT:    lw a4, 4(a1)
-; RV32IM-NEXT:    lw s4, 8(a1)
-; RV32IM-NEXT:    lw s5, 12(a1)
-; RV32IM-NEXT:    mv s6, a0
+; RV32IM-NEXT:    lw s5, 8(a1)
+; RV32IM-NEXT:    lw s6, 12(a1)
+; RV32IM-NEXT:    mv s0, a0
 ; RV32IM-NEXT:    li a2, 1
 ; RV32IM-NEXT:    mv a0, a3
 ; RV32IM-NEXT:    mv a1, a4
@@ -880,33 +880,33 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    mv s7, a0
 ; RV32IM-NEXT:    mv s8, a1
 ; RV32IM-NEXT:    li a2, 654
-; RV32IM-NEXT:    mv a0, s4
-; RV32IM-NEXT:    mv a1, s5
+; RV32IM-NEXT:    mv a0, s5
+; RV32IM-NEXT:    mv a1, s6
 ; RV32IM-NEXT:    li a3, 0
 ; RV32IM-NEXT:    call __umoddi3
-; RV32IM-NEXT:    mv s4, a0
-; RV32IM-NEXT:    mv s5, a1
+; RV32IM-NEXT:    mv s5, a0
+; RV32IM-NEXT:    mv s6, a1
 ; RV32IM-NEXT:    li a2, 23
-; RV32IM-NEXT:    mv a0, s0
-; RV32IM-NEXT:    mv a1, s1
+; RV32IM-NEXT:    mv a0, s1
+; RV32IM-NEXT:    mv a1, s2
 ; RV32IM-NEXT:    li a3, 0
 ; RV32IM-NEXT:    call __umoddi3
-; RV32IM-NEXT:    mv s0, a0
-; RV32IM-NEXT:    mv s1, a1
+; RV32IM-NEXT:    mv s1, a0
+; RV32IM-NEXT:    mv s2, a1
 ; RV32IM-NEXT:    lui a0, 1
 ; RV32IM-NEXT:    addi a2, a0, 1327
-; RV32IM-NEXT:    mv a0, s2
-; RV32IM-NEXT:    mv a1, s3
+; RV32IM-NEXT:    mv a0, s3
+; RV32IM-NEXT:    mv a1, s4
 ; RV32IM-NEXT:    li a3, 0
 ; RV32IM-NEXT:    call __umoddi3
-; RV32IM-NEXT:    sw a1, 28(s6)
-; RV32IM-NEXT:    sw a0, 24(s6)
-; RV32IM-NEXT:    sw s1, 20(s6)
-; RV32IM-NEXT:    sw s0, 16(s6)
-; RV32IM-NEXT:    sw s5, 12(s6)
-; RV32IM-NEXT:    sw s4, 8(s6)
-; RV32IM-NEXT:    sw s8, 4(s6)
-; RV32IM-NEXT:    sw s7, 0(s6)
+; RV32IM-NEXT:    sw s1, 16(s0)
+; RV32IM-NEXT:    sw s2, 20(s0)
+; RV32IM-NEXT:    sw a0, 24(s0)
+; RV32IM-NEXT:    sw a1, 28(s0)
+; RV32IM-NEXT:    sw s7, 0(s0)
+; RV32IM-NEXT:    sw s8, 4(s0)
+; RV32IM-NEXT:    sw s5, 8(s0)
+; RV32IM-NEXT:    sw s6, 12(s0)
 ; RV32IM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
@@ -944,10 +944,10 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    sd a0, 24(s2)
-; RV64I-NEXT:    sd s0, 16(s2)
-; RV64I-NEXT:    sd s3, 8(s2)
 ; RV64I-NEXT:    sd zero, 0(s2)
+; RV64I-NEXT:    sd s3, 8(s2)
+; RV64I-NEXT:    sd s0, 16(s2)
+; RV64I-NEXT:    sd a0, 24(s2)
 ; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
@@ -988,9 +988,9 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV64IM-NEXT:    mul a2, a2, a5
 ; RV64IM-NEXT:    sub a1, a1, a2
 ; RV64IM-NEXT:    sd zero, 0(a0)
-; RV64IM-NEXT:    sd a1, 24(a0)
 ; RV64IM-NEXT:    sd a4, 8(a0)
 ; RV64IM-NEXT:    sd a3, 16(a0)
+; RV64IM-NEXT:    sd a1, 24(a0)
 ; RV64IM-NEXT:    ret
   %1 = urem <4 x i64> %x, <i64 1, i64 654, i64 23, i64 5423>
   ret <4 x i64> %1
diff --git a/llvm/test/CodeGen/RISCV/vararg-ilp32e.ll b/llvm/test/CodeGen/RISCV/vararg-ilp32e.ll
index 18bb4f5ad0f0c0681a1d80940dfd105703d90eb2..7c4c98a12f4b8181b97c76d0431367a21424d4f9 100644
--- a/llvm/test/CodeGen/RISCV/vararg-ilp32e.ll
+++ b/llvm/test/CodeGen/RISCV/vararg-ilp32e.ll
@@ -64,10 +64,10 @@ define void @va_double(i32 %n, ...) {
 ; ILP32E-NEXT:    sw ra, 4(sp) # 4-byte Folded Spill
 ; ILP32E-NEXT:    .cfi_offset ra, -28
 ; ILP32E-NEXT:    sw a5, 28(sp)
-; ILP32E-NEXT:    sw a4, 24(sp)
-; ILP32E-NEXT:    sw a3, 20(sp)
-; ILP32E-NEXT:    sw a2, 16(sp)
 ; ILP32E-NEXT:    sw a1, 12(sp)
+; ILP32E-NEXT:    sw a2, 16(sp)
+; ILP32E-NEXT:    sw a3, 20(sp)
+; ILP32E-NEXT:    sw a4, 24(sp)
 ; ILP32E-NEXT:    addi a0, sp, 12
 ; ILP32E-NEXT:    sw a0, 0(sp)
 ; ILP32E-NEXT:    addi a0, sp, 19
@@ -98,10 +98,10 @@ define void @va_double(i32 %n, ...) {
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 12
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 24
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 4
 ; ILP32E-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 11
diff --git a/llvm/test/CodeGen/RISCV/vararg.ll b/llvm/test/CodeGen/RISCV/vararg.ll
index d2c30c543907026fcd347a7700d2ac622e4b5707..c53e5a78d1988c52dd3332013731f5f96ae94774 100644
--- a/llvm/test/CodeGen/RISCV/vararg.ll
+++ b/llvm/test/CodeGen/RISCV/vararg.ll
@@ -55,13 +55,13 @@ define i32 @va1(ptr %fmt, ...) {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
 ; ILP32-ILP32F-FPELIM-NEXT:    .cfi_def_cfa_offset 48
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a0, a1
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 36(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
@@ -78,13 +78,13 @@ define i32 @va1(ptr %fmt, ...) {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    .cfi_def_cfa s0, 32
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a0, a1
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -97,13 +97,13 @@ define i32 @va1(ptr %fmt, ...) {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    .cfi_def_cfa_offset 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a0, a1
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
@@ -115,10 +115,10 @@ define i32 @va1(ptr %fmt, ...) {
 ; ILP32E-FPELIM-NEXT:    .cfi_def_cfa_offset 28
 ; ILP32E-FPELIM-NEXT:    mv a0, a1
 ; ILP32E-FPELIM-NEXT:    sw a5, 24(sp)
-; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
-; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
 ; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
+; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
 ; ILP32E-FPELIM-NEXT:    addi a1, sp, 12
 ; ILP32E-FPELIM-NEXT:    sw a1, 0(sp)
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, 28
@@ -136,10 +136,10 @@ define i32 @va1(ptr %fmt, ...) {
 ; ILP32E-WITHFP-NEXT:    .cfi_def_cfa s0, 24
 ; ILP32E-WITHFP-NEXT:    mv a0, a1
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32E-WITHFP-NEXT:    sw a1, -12(s0)
 ; ILP32E-WITHFP-NEXT:    lw ra, 8(sp) # 4-byte Folded Reload
@@ -152,15 +152,15 @@ define i32 @va1(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    .cfi_def_cfa_offset 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 28
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lw a0, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -175,15 +175,15 @@ define i32 @va1(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa s0, 64
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 12
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
@@ -193,14 +193,14 @@ define i32 @va1(ptr %fmt, ...) {
 ; LP64E-FPELIM:       # %bb.0:
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -56
 ; LP64E-FPELIM-NEXT:    .cfi_def_cfa_offset 56
-; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
 ; LP64E-FPELIM-NEXT:    addi a0, sp, 20
 ; LP64E-FPELIM-NEXT:    sd a0, 0(sp)
+; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
 ; LP64E-FPELIM-NEXT:    lw a0, 16(sp)
 ; LP64E-FPELIM-NEXT:    sd a5, 48(sp)
-; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
 ; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
+; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
 ; LP64E-FPELIM-NEXT:    addi sp, sp, 56
 ; LP64E-FPELIM-NEXT:    ret
 ;
@@ -214,14 +214,14 @@ define i32 @va1(ptr %fmt, ...) {
 ; LP64E-WITHFP-NEXT:    .cfi_offset s0, -64
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 24
 ; LP64E-WITHFP-NEXT:    .cfi_def_cfa s0, 48
-; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64E-WITHFP-NEXT:    addi a0, s0, 12
 ; LP64E-WITHFP-NEXT:    sd a0, -24(s0)
+; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64E-WITHFP-NEXT:    lw a0, 8(s0)
 ; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    ld ra, 16(sp) # 8-byte Folded Reload
 ; LP64E-WITHFP-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; LP64E-WITHFP-NEXT:    addi sp, sp, 72
@@ -241,13 +241,13 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a0, a1
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 36(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
@@ -260,13 +260,13 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a0, a1
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -278,13 +278,13 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a0, a1
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
@@ -295,10 +295,10 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, -28
 ; ILP32E-FPELIM-NEXT:    mv a0, a1
 ; ILP32E-FPELIM-NEXT:    sw a5, 24(sp)
-; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
-; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
 ; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
+; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
 ; ILP32E-FPELIM-NEXT:    addi a1, sp, 12
 ; ILP32E-FPELIM-NEXT:    sw a1, 0(sp)
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, 28
@@ -312,10 +312,10 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 12
 ; ILP32E-WITHFP-NEXT:    mv a0, a1
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32E-WITHFP-NEXT:    sw a1, -12(s0)
 ; ILP32E-WITHFP-NEXT:    lw ra, 8(sp) # 4-byte Folded Reload
@@ -327,13 +327,13 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv a0, a1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 32
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
@@ -346,13 +346,13 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, a1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -365,10 +365,10 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -56
 ; LP64E-FPELIM-NEXT:    mv a0, a1
 ; LP64E-FPELIM-NEXT:    sd a5, 48(sp)
-; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
-; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
 ; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
+; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
 ; LP64E-FPELIM-NEXT:    addi a1, sp, 24
 ; LP64E-FPELIM-NEXT:    sd a1, 0(sp)
 ; LP64E-FPELIM-NEXT:    addi sp, sp, 56
@@ -382,10 +382,10 @@ define i32 @va1_va_arg(ptr %fmt, ...) nounwind {
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 24
 ; LP64E-WITHFP-NEXT:    mv a0, a1
 ; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    addi a1, s0, 16
 ; LP64E-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64E-WITHFP-NEXT:    ld ra, 16(sp) # 8-byte Folded Reload
@@ -410,13 +410,13 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-FPELIM-NEXT:    mv s1, a1
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, -16(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a1, 15
@@ -440,13 +440,13 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    mv s1, a1
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a1, 15
@@ -470,13 +470,13 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi s0, sp, 16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv s1, a1
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 28(s0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 24(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 20(s0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(s0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 12(s0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 8(s0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 24(s0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 28(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 4(s0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 8(s0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 12(s0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, s0, 8
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, -16(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a1, 15
@@ -501,10 +501,10 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; ILP32E-FPELIM-NEXT:    addi s0, sp, 16
 ; ILP32E-FPELIM-NEXT:    mv s1, a1
 ; ILP32E-FPELIM-NEXT:    sw a5, 20(s0)
-; ILP32E-FPELIM-NEXT:    sw a4, 16(s0)
-; ILP32E-FPELIM-NEXT:    sw a3, 12(s0)
-; ILP32E-FPELIM-NEXT:    sw a2, 8(s0)
 ; ILP32E-FPELIM-NEXT:    sw a1, 4(s0)
+; ILP32E-FPELIM-NEXT:    sw a2, 8(s0)
+; ILP32E-FPELIM-NEXT:    sw a3, 12(s0)
+; ILP32E-FPELIM-NEXT:    sw a4, 16(s0)
 ; ILP32E-FPELIM-NEXT:    addi a0, s0, 8
 ; ILP32E-FPELIM-NEXT:    sw a0, -16(s0)
 ; ILP32E-FPELIM-NEXT:    addi a0, a1, 3
@@ -529,10 +529,10 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32E-WITHFP-NEXT:    mv s1, a1
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 8
 ; ILP32E-WITHFP-NEXT:    sw a0, -16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, a1, 3
@@ -556,13 +556,13 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv s1, a1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(s0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, s0, 16
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, -32(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a1, 32
@@ -588,13 +588,13 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv s1, a1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a1, 32
@@ -621,10 +621,10 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LP64E-FPELIM-NEXT:    addi s0, sp, 32
 ; LP64E-FPELIM-NEXT:    mv s1, a1
 ; LP64E-FPELIM-NEXT:    sd a5, 40(s0)
-; LP64E-FPELIM-NEXT:    sd a4, 32(s0)
-; LP64E-FPELIM-NEXT:    sd a3, 24(s0)
-; LP64E-FPELIM-NEXT:    sd a2, 16(s0)
 ; LP64E-FPELIM-NEXT:    sd a1, 8(s0)
+; LP64E-FPELIM-NEXT:    sd a2, 16(s0)
+; LP64E-FPELIM-NEXT:    sd a3, 24(s0)
+; LP64E-FPELIM-NEXT:    sd a4, 32(s0)
 ; LP64E-FPELIM-NEXT:    addi a0, s0, 16
 ; LP64E-FPELIM-NEXT:    sd a0, -32(s0)
 ; LP64E-FPELIM-NEXT:    slli a0, a1, 32
@@ -651,10 +651,10 @@ define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64E-WITHFP-NEXT:    mv s1, a1
 ; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    addi a0, s0, 16
 ; LP64E-WITHFP-NEXT:    sd a0, -32(s0)
 ; LP64E-WITHFP-NEXT:    slli a0, a1, 32
@@ -812,13 +812,13 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va2:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 36(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 20
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 27
@@ -836,13 +836,13 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
@@ -859,13 +859,13 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va2:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 20
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 27
@@ -881,10 +881,10 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; ILP32E-FPELIM:       # %bb.0:
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, -28
 ; ILP32E-FPELIM-NEXT:    sw a5, 24(sp)
-; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
-; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
 ; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
+; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 8
 ; ILP32E-FPELIM-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 15
@@ -903,10 +903,10 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; ILP32E-WITHFP-NEXT:    sw s0, 4(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 12
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 4
 ; ILP32E-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 11
@@ -924,13 +924,13 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv a0, a1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 39
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
@@ -943,13 +943,13 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, a1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 23
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -962,10 +962,10 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -56
 ; LP64E-FPELIM-NEXT:    mv a0, a1
 ; LP64E-FPELIM-NEXT:    sd a5, 48(sp)
-; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
-; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
 ; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
+; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
 ; LP64E-FPELIM-NEXT:    addi a1, sp, 31
 ; LP64E-FPELIM-NEXT:    sd a1, 0(sp)
 ; LP64E-FPELIM-NEXT:    addi sp, sp, 56
@@ -979,10 +979,10 @@ define i64 @va2(ptr %fmt, ...) nounwind {
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 24
 ; LP64E-WITHFP-NEXT:    mv a0, a1
 ; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    addi a1, s0, 23
 ; LP64E-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64E-WITHFP-NEXT:    ld ra, 16(sp) # 8-byte Folded Reload
@@ -1009,13 +1009,13 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va2_va_arg:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 36(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 27
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a1, a0, -8
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a1, 4
@@ -1033,13 +1033,13 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a1, a0, -8
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a1, 4
@@ -1056,13 +1056,13 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va2_va_arg:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 27
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -8
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, a0, 8
@@ -1078,10 +1078,10 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32E-FPELIM:       # %bb.0:
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, -28
 ; ILP32E-FPELIM-NEXT:    sw a5, 24(sp)
-; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
-; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
 ; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
+; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 15
 ; ILP32E-FPELIM-NEXT:    andi a1, a0, -8
 ; ILP32E-FPELIM-NEXT:    addi a0, a1, 4
@@ -1100,10 +1100,10 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; ILP32E-WITHFP-NEXT:    sw s0, 4(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 12
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32E-WITHFP-NEXT:    andi a1, a0, -8
 ; ILP32E-WITHFP-NEXT:    addi a0, a1, 4
@@ -1121,13 +1121,13 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv a0, a1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 32
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
@@ -1140,13 +1140,13 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, a1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -1159,10 +1159,10 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -56
 ; LP64E-FPELIM-NEXT:    mv a0, a1
 ; LP64E-FPELIM-NEXT:    sd a5, 48(sp)
-; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
-; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
 ; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
+; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
 ; LP64E-FPELIM-NEXT:    addi a1, sp, 24
 ; LP64E-FPELIM-NEXT:    sd a1, 0(sp)
 ; LP64E-FPELIM-NEXT:    addi sp, sp, 56
@@ -1176,10 +1176,10 @@ define i64 @va2_va_arg(ptr %fmt, ...) nounwind {
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 24
 ; LP64E-WITHFP-NEXT:    mv a0, a1
 ; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    addi a1, s0, 16
 ; LP64E-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64E-WITHFP-NEXT:    ld ra, 16(sp) # 8-byte Folded Reload
@@ -1317,10 +1317,10 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -32
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 24(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 20(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 24(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 12
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 19
@@ -1343,10 +1343,10 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 16(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 24
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
@@ -1368,10 +1368,10 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -32
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 28(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 24(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 20(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 24(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 12
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 4(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 19
@@ -1390,9 +1390,9 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32E-FPELIM-LABEL: va3:
 ; ILP32E-FPELIM:       # %bb.0:
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, -20
-; ILP32E-FPELIM-NEXT:    sw a5, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw a4, 12(sp)
 ; ILP32E-FPELIM-NEXT:    sw a3, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 12(sp)
+; ILP32E-FPELIM-NEXT:    sw a5, 16(sp)
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 8
 ; ILP32E-FPELIM-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 15
@@ -1414,9 +1414,9 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32E-WITHFP-NEXT:    sw ra, 8(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    sw s0, 4(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 12
-; ILP32E-WITHFP-NEXT:    sw a5, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a3, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a5, 12(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 4
 ; ILP32E-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 11
@@ -1437,12 +1437,12 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va3:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -64
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 48(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, sp, 31
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, a1, a2
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 8(sp)
@@ -1455,12 +1455,12 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, s0, 15
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a0, a1, a2
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, -24(s0)
@@ -1472,10 +1472,10 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64E-FPELIM-LABEL: va3:
 ; LP64E-FPELIM:       # %bb.0:
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -40
-; LP64E-FPELIM-NEXT:    sd a5, 32(sp)
-; LP64E-FPELIM-NEXT:    sd a4, 24(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 16(sp)
 ; LP64E-FPELIM-NEXT:    sd a2, 8(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a4, 24(sp)
+; LP64E-FPELIM-NEXT:    sd a5, 32(sp)
 ; LP64E-FPELIM-NEXT:    addi a3, sp, 23
 ; LP64E-FPELIM-NEXT:    add a0, a1, a2
 ; LP64E-FPELIM-NEXT:    sd a3, 0(sp)
@@ -1488,10 +1488,10 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64E-WITHFP-NEXT:    sd ra, 16(sp) # 8-byte Folded Spill
 ; LP64E-WITHFP-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 24
-; LP64E-WITHFP-NEXT:    sd a5, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 16(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 8(s0)
 ; LP64E-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64E-WITHFP-NEXT:    addi a3, s0, 15
 ; LP64E-WITHFP-NEXT:    add a0, a1, a2
 ; LP64E-WITHFP-NEXT:    sd a3, -24(s0)
@@ -1521,10 +1521,10 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -32
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 24(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 20(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 24(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 19
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -8
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a3, a0, 4
@@ -1547,10 +1547,10 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 16(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 24
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -8
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a3, a0, 4
@@ -1572,10 +1572,10 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 35
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -8
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, a0, 8
@@ -1594,9 +1594,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32E-FPELIM-LABEL: va3_va_arg:
 ; ILP32E-FPELIM:       # %bb.0:
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, -20
-; ILP32E-FPELIM-NEXT:    sw a5, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw a4, 12(sp)
 ; ILP32E-FPELIM-NEXT:    sw a3, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 12(sp)
+; ILP32E-FPELIM-NEXT:    sw a5, 16(sp)
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 15
 ; ILP32E-FPELIM-NEXT:    andi a0, a0, -8
 ; ILP32E-FPELIM-NEXT:    addi a3, a0, 4
@@ -1618,9 +1618,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32E-WITHFP-NEXT:    sw ra, 8(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    sw s0, 4(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 12
-; ILP32E-WITHFP-NEXT:    sw a5, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a3, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a5, 12(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32E-WITHFP-NEXT:    andi a0, a0, -8
 ; ILP32E-WITHFP-NEXT:    addi a3, a0, 4
@@ -1641,12 +1641,12 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va3_va_arg:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -64
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 48(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, sp, 24
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, a1, a2
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 8(sp)
@@ -1659,12 +1659,12 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a0, a1, a2
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, -24(s0)
@@ -1676,10 +1676,10 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64E-FPELIM-LABEL: va3_va_arg:
 ; LP64E-FPELIM:       # %bb.0:
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -40
-; LP64E-FPELIM-NEXT:    sd a5, 32(sp)
-; LP64E-FPELIM-NEXT:    sd a4, 24(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 16(sp)
 ; LP64E-FPELIM-NEXT:    sd a2, 8(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a4, 24(sp)
+; LP64E-FPELIM-NEXT:    sd a5, 32(sp)
 ; LP64E-FPELIM-NEXT:    addi a3, sp, 16
 ; LP64E-FPELIM-NEXT:    add a0, a1, a2
 ; LP64E-FPELIM-NEXT:    sd a3, 0(sp)
@@ -1692,10 +1692,10 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64E-WITHFP-NEXT:    sd ra, 16(sp) # 8-byte Folded Spill
 ; LP64E-WITHFP-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 24
-; LP64E-WITHFP-NEXT:    sd a5, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 16(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 8(s0)
 ; LP64E-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64E-WITHFP-NEXT:    addi a3, s0, 8
 ; LP64E-WITHFP-NEXT:    add a0, a1, a2
 ; LP64E-WITHFP-NEXT:    sd a3, -24(s0)
@@ -1859,13 +1859,13 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    mv s0, a1
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 36(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 0(sp)
@@ -1902,13 +1902,13 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 32
 ; ILP32-ILP32F-WITHFP-NEXT:    mv s1, a1
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -20(s0)
@@ -1944,13 +1944,13 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv s0, a1
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 4(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 0(sp)
@@ -1986,10 +1986,10 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32E-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32E-FPELIM-NEXT:    mv s0, a1
 ; ILP32E-FPELIM-NEXT:    sw a5, 36(sp)
-; ILP32E-FPELIM-NEXT:    sw a4, 32(sp)
-; ILP32E-FPELIM-NEXT:    sw a3, 28(sp)
-; ILP32E-FPELIM-NEXT:    sw a2, 24(sp)
 ; ILP32E-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32E-FPELIM-NEXT:    sw a3, 28(sp)
+; ILP32E-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32E-FPELIM-NEXT:    addi a0, sp, 24
 ; ILP32E-FPELIM-NEXT:    sw a0, 4(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 0(sp)
@@ -2027,10 +2027,10 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 20
 ; ILP32E-WITHFP-NEXT:    mv s1, a1
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a0, s0, 8
 ; ILP32E-WITHFP-NEXT:    sw a0, -16(s0)
 ; ILP32E-WITHFP-NEXT:    sw a0, -20(s0)
@@ -2066,13 +2066,13 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv s0, a1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 88(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 80(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 72(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 64(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 56(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 48(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 80(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 88(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 48(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 56(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 64(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 48
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 0(sp)
@@ -2109,13 +2109,13 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 48
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv s1, a1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -40(s0)
@@ -2152,10 +2152,10 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64E-FPELIM-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64E-FPELIM-NEXT:    mv s0, a1
 ; LP64E-FPELIM-NEXT:    sd a5, 72(sp)
-; LP64E-FPELIM-NEXT:    sd a4, 64(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 56(sp)
-; LP64E-FPELIM-NEXT:    sd a2, 48(sp)
 ; LP64E-FPELIM-NEXT:    sd a1, 40(sp)
+; LP64E-FPELIM-NEXT:    sd a2, 48(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 56(sp)
+; LP64E-FPELIM-NEXT:    sd a4, 64(sp)
 ; LP64E-FPELIM-NEXT:    addi a0, sp, 48
 ; LP64E-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64E-FPELIM-NEXT:    sd a0, 0(sp)
@@ -2193,10 +2193,10 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 40
 ; LP64E-WITHFP-NEXT:    mv s1, a1
 ; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    addi a0, s0, 16
 ; LP64E-WITHFP-NEXT:    sd a0, -32(s0)
 ; LP64E-WITHFP-NEXT:    sd a0, -40(s0)
@@ -2255,30 +2255,27 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -64
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    li a0, 17
+; ILP32-ILP32F-FPELIM-NEXT:    li a1, 16
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 24(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    li a0, 16
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    li a0, 15
+; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 262236
+; ILP32-ILP32F-FPELIM-NEXT:    addi a1, a1, 655
+; ILP32-ILP32F-FPELIM-NEXT:    lui a2, 377487
+; ILP32-ILP32F-FPELIM-NEXT:    addi a2, a2, 1475
+; ILP32-ILP32F-FPELIM-NEXT:    li a3, 14
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 0(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 8(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 16(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 262236
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, 655
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 377487
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, 1475
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 8(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    li a0, 14
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 0(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 262153
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, 491
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 44(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    addi a5, a0, 491
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 545260
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, -1967
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    addi t0, a0, -1967
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 964690
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, -328
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 36(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    addi t1, a0, -328
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 335544
-; ILP32-ILP32F-FPELIM-NEXT:    addi a5, a0, 1311
+; ILP32-ILP32F-FPELIM-NEXT:    addi t2, a0, 1311
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 688509
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a6, a0, -2048
 ; ILP32-ILP32F-FPELIM-NEXT:    li a0, 1
@@ -2287,7 +2284,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    li a3, 12
 ; ILP32-ILP32F-FPELIM-NEXT:    li a4, 13
 ; ILP32-ILP32F-FPELIM-NEXT:    li a7, 4
-; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 32(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw t2, 32(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw t1, 36(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw t0, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 64
@@ -2300,30 +2300,27 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 64
 ; ILP32-ILP32F-WITHFP-NEXT:    li a0, 17
+; ILP32-ILP32F-WITHFP-NEXT:    li a1, 16
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 20(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 24(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    li a0, 16
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 20(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    li a0, 15
+; ILP32-ILP32F-WITHFP-NEXT:    lui a1, 262236
+; ILP32-ILP32F-WITHFP-NEXT:    addi a1, a1, 655
+; ILP32-ILP32F-WITHFP-NEXT:    lui a2, 377487
+; ILP32-ILP32F-WITHFP-NEXT:    addi a2, a2, 1475
+; ILP32-ILP32F-WITHFP-NEXT:    li a3, 14
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 0(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 16(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 262236
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, 655
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 377487
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, 1475
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    li a0, 14
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 0(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 262153
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, 491
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -20(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    addi a5, a0, 491
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 545260
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, -1967
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    addi t0, a0, -1967
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 964690
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, -328
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -28(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    addi t1, a0, -328
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 335544
-; ILP32-ILP32F-WITHFP-NEXT:    addi a5, a0, 1311
+; ILP32-ILP32F-WITHFP-NEXT:    addi t2, a0, 1311
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 688509
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a6, a0, -2048
 ; ILP32-ILP32F-WITHFP-NEXT:    li a0, 1
@@ -2332,7 +2329,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    li a3, 12
 ; ILP32-ILP32F-WITHFP-NEXT:    li a4, 13
 ; ILP32-ILP32F-WITHFP-NEXT:    li a7, 4
-; ILP32-ILP32F-WITHFP-NEXT:    sw a5, -32(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw t2, -32(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw t1, -28(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw t0, -24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a5, -20(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
@@ -2345,29 +2345,26 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 262236
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 655
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 377487
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, a1, 1475
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a2, 17
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a3, 16
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a2, 15
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a3, 14
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 0(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 8(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 377487
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 1475
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 8(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a0, 17
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 24(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a0, 16
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 20(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a0, 15
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 16(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a0, 14
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 0(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 262153
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 491
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 44(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a5, a0, 491
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 545260
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, -1967
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi t0, a0, -1967
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 964690
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, -328
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 36(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi t1, a0, -328
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 335544
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a5, a0, 1311
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi t2, a0, 1311
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 688509
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a6, a0, -2048
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a0, 1
@@ -2376,7 +2373,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a3, 12
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a4, 13
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    li a7, 4
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 32(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw t2, 32(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw t1, 36(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw t0, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 64
@@ -2390,32 +2390,29 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32E-FPELIM-NEXT:    addi s0, sp, 64
 ; ILP32E-FPELIM-NEXT:    andi sp, sp, -16
 ; ILP32E-FPELIM-NEXT:    li a0, 17
+; ILP32E-FPELIM-NEXT:    li a1, 16
+; ILP32E-FPELIM-NEXT:    li a2, 15
+; ILP32E-FPELIM-NEXT:    sw a2, 16(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 20(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 24(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 16
-; ILP32E-FPELIM-NEXT:    sw a0, 20(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 15
-; ILP32E-FPELIM-NEXT:    sw a0, 16(sp)
 ; ILP32E-FPELIM-NEXT:    lui a0, 262236
 ; ILP32E-FPELIM-NEXT:    addi a0, a0, 655
+; ILP32E-FPELIM-NEXT:    lui a1, 377487
+; ILP32E-FPELIM-NEXT:    addi a1, a1, 1475
+; ILP32E-FPELIM-NEXT:    li a2, 14
+; ILP32E-FPELIM-NEXT:    li a3, 4
+; ILP32E-FPELIM-NEXT:    sw a3, 0(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 4(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 12(sp)
-; ILP32E-FPELIM-NEXT:    lui a0, 377487
-; ILP32E-FPELIM-NEXT:    addi a0, a0, 1475
-; ILP32E-FPELIM-NEXT:    sw a0, 8(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 14
-; ILP32E-FPELIM-NEXT:    sw a0, 4(sp)
-; ILP32E-FPELIM-NEXT:    li a0, 4
-; ILP32E-FPELIM-NEXT:    sw a0, 0(sp)
 ; ILP32E-FPELIM-NEXT:    lui a0, 262153
-; ILP32E-FPELIM-NEXT:    addi a0, a0, 491
-; ILP32E-FPELIM-NEXT:    sw a0, 44(sp)
+; ILP32E-FPELIM-NEXT:    addi a6, a0, 491
 ; ILP32E-FPELIM-NEXT:    lui a0, 545260
-; ILP32E-FPELIM-NEXT:    addi a0, a0, -1967
-; ILP32E-FPELIM-NEXT:    sw a0, 40(sp)
+; ILP32E-FPELIM-NEXT:    addi a7, a0, -1967
 ; ILP32E-FPELIM-NEXT:    lui a0, 964690
-; ILP32E-FPELIM-NEXT:    addi a0, a0, -328
-; ILP32E-FPELIM-NEXT:    sw a0, 36(sp)
+; ILP32E-FPELIM-NEXT:    addi t0, a0, -328
 ; ILP32E-FPELIM-NEXT:    lui a0, 335544
-; ILP32E-FPELIM-NEXT:    addi a6, a0, 1311
+; ILP32E-FPELIM-NEXT:    addi t1, a0, 1311
 ; ILP32E-FPELIM-NEXT:    lui a0, 688509
 ; ILP32E-FPELIM-NEXT:    addi a5, a0, -2048
 ; ILP32E-FPELIM-NEXT:    li a0, 1
@@ -2423,7 +2420,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32E-FPELIM-NEXT:    addi a2, sp, 32
 ; ILP32E-FPELIM-NEXT:    li a3, 12
 ; ILP32E-FPELIM-NEXT:    li a4, 13
-; ILP32E-FPELIM-NEXT:    sw a6, 32(sp)
+; ILP32E-FPELIM-NEXT:    sw t1, 32(sp)
+; ILP32E-FPELIM-NEXT:    sw t0, 36(sp)
+; ILP32E-FPELIM-NEXT:    sw a7, 40(sp)
+; ILP32E-FPELIM-NEXT:    sw a6, 44(sp)
 ; ILP32E-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; ILP32E-FPELIM-NEXT:    addi sp, s0, -64
 ; ILP32E-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
@@ -2439,32 +2439,29 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 64
 ; ILP32E-WITHFP-NEXT:    andi sp, sp, -16
 ; ILP32E-WITHFP-NEXT:    li a0, 17
+; ILP32E-WITHFP-NEXT:    li a1, 16
+; ILP32E-WITHFP-NEXT:    li a2, 15
+; ILP32E-WITHFP-NEXT:    sw a2, 16(sp)
+; ILP32E-WITHFP-NEXT:    sw a1, 20(sp)
 ; ILP32E-WITHFP-NEXT:    sw a0, 24(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 16
-; ILP32E-WITHFP-NEXT:    sw a0, 20(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 15
-; ILP32E-WITHFP-NEXT:    sw a0, 16(sp)
 ; ILP32E-WITHFP-NEXT:    lui a0, 262236
 ; ILP32E-WITHFP-NEXT:    addi a0, a0, 655
+; ILP32E-WITHFP-NEXT:    lui a1, 377487
+; ILP32E-WITHFP-NEXT:    addi a1, a1, 1475
+; ILP32E-WITHFP-NEXT:    li a2, 14
+; ILP32E-WITHFP-NEXT:    li a3, 4
+; ILP32E-WITHFP-NEXT:    sw a3, 0(sp)
+; ILP32E-WITHFP-NEXT:    sw a2, 4(sp)
+; ILP32E-WITHFP-NEXT:    sw a1, 8(sp)
 ; ILP32E-WITHFP-NEXT:    sw a0, 12(sp)
-; ILP32E-WITHFP-NEXT:    lui a0, 377487
-; ILP32E-WITHFP-NEXT:    addi a0, a0, 1475
-; ILP32E-WITHFP-NEXT:    sw a0, 8(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 14
-; ILP32E-WITHFP-NEXT:    sw a0, 4(sp)
-; ILP32E-WITHFP-NEXT:    li a0, 4
-; ILP32E-WITHFP-NEXT:    sw a0, 0(sp)
 ; ILP32E-WITHFP-NEXT:    lui a0, 262153
-; ILP32E-WITHFP-NEXT:    addi a0, a0, 491
-; ILP32E-WITHFP-NEXT:    sw a0, 44(sp)
+; ILP32E-WITHFP-NEXT:    addi a6, a0, 491
 ; ILP32E-WITHFP-NEXT:    lui a0, 545260
-; ILP32E-WITHFP-NEXT:    addi a0, a0, -1967
-; ILP32E-WITHFP-NEXT:    sw a0, 40(sp)
+; ILP32E-WITHFP-NEXT:    addi a7, a0, -1967
 ; ILP32E-WITHFP-NEXT:    lui a0, 964690
-; ILP32E-WITHFP-NEXT:    addi a0, a0, -328
-; ILP32E-WITHFP-NEXT:    sw a0, 36(sp)
+; ILP32E-WITHFP-NEXT:    addi t0, a0, -328
 ; ILP32E-WITHFP-NEXT:    lui a0, 335544
-; ILP32E-WITHFP-NEXT:    addi a6, a0, 1311
+; ILP32E-WITHFP-NEXT:    addi t1, a0, 1311
 ; ILP32E-WITHFP-NEXT:    lui a0, 688509
 ; ILP32E-WITHFP-NEXT:    addi a5, a0, -2048
 ; ILP32E-WITHFP-NEXT:    li a0, 1
@@ -2472,7 +2469,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32E-WITHFP-NEXT:    addi a2, sp, 32
 ; ILP32E-WITHFP-NEXT:    li a3, 12
 ; ILP32E-WITHFP-NEXT:    li a4, 13
-; ILP32E-WITHFP-NEXT:    sw a6, 32(sp)
+; ILP32E-WITHFP-NEXT:    sw t1, 32(sp)
+; ILP32E-WITHFP-NEXT:    sw t0, 36(sp)
+; ILP32E-WITHFP-NEXT:    sw a7, 40(sp)
+; ILP32E-WITHFP-NEXT:    sw a6, 44(sp)
 ; ILP32E-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; ILP32E-WITHFP-NEXT:    addi sp, s0, -64
 ; ILP32E-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
@@ -2484,14 +2484,11 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -48
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
-; LP64-LP64F-LP64D-FPELIM-NEXT:    li a0, 17
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    li a0, 16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 16(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    li a0, 15
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    li t0, 17
+; LP64-LP64F-LP64D-FPELIM-NEXT:    li t1, 16
+; LP64-LP64F-LP64D-FPELIM-NEXT:    li t2, 15
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, %hi(.LCPI11_0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld t0, %lo(.LCPI11_0)(a0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld t3, %lo(.LCPI11_0)(a0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, %hi(.LCPI11_1)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a2, %lo(.LCPI11_1)(a0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, %hi(.LCPI11_2)
@@ -2504,7 +2501,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    li a4, 12
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    li a5, 13
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    li a7, 14
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t0, 0(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t3, 0(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t2, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t1, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t0, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 48
@@ -2516,14 +2516,11 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 48
-; LP64-LP64F-LP64D-WITHFP-NEXT:    li a0, 17
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    li a0, 16
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    li a0, 15
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 8(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    li t0, 17
+; LP64-LP64F-LP64D-WITHFP-NEXT:    li t1, 16
+; LP64-LP64F-LP64D-WITHFP-NEXT:    li t2, 15
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, %hi(.LCPI11_0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld t0, %lo(.LCPI11_0)(a0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld t3, %lo(.LCPI11_0)(a0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, %hi(.LCPI11_1)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a2, %lo(.LCPI11_1)(a0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, %hi(.LCPI11_2)
@@ -2536,7 +2533,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    li a4, 12
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    li a5, 13
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    li a7, 14
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t0, 0(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t3, 0(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t2, 8(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t1, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t0, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
@@ -2548,19 +2548,16 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -56
 ; LP64E-FPELIM-NEXT:    sd ra, 48(sp) # 8-byte Folded Spill
 ; LP64E-FPELIM-NEXT:    li a0, 17
+; LP64E-FPELIM-NEXT:    li a1, 16
+; LP64E-FPELIM-NEXT:    sd a1, 32(sp)
 ; LP64E-FPELIM-NEXT:    sd a0, 40(sp)
-; LP64E-FPELIM-NEXT:    li a0, 16
-; LP64E-FPELIM-NEXT:    lui a1, %hi(.LCPI11_0)
-; LP64E-FPELIM-NEXT:    ld a1, %lo(.LCPI11_0)(a1)
-; LP64E-FPELIM-NEXT:    sd a0, 32(sp)
-; LP64E-FPELIM-NEXT:    li a0, 15
-; LP64E-FPELIM-NEXT:    sd a0, 24(sp)
-; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
-; LP64E-FPELIM-NEXT:    li a0, 14
-; LP64E-FPELIM-NEXT:    sd a0, 8(sp)
+; LP64E-FPELIM-NEXT:    li a6, 15
+; LP64E-FPELIM-NEXT:    lui a0, %hi(.LCPI11_0)
+; LP64E-FPELIM-NEXT:    ld a7, %lo(.LCPI11_0)(a0)
+; LP64E-FPELIM-NEXT:    li t0, 14
 ; LP64E-FPELIM-NEXT:    lui a0, 2384
 ; LP64E-FPELIM-NEXT:    addiw a0, a0, 761
-; LP64E-FPELIM-NEXT:    slli a6, a0, 11
+; LP64E-FPELIM-NEXT:    slli t1, a0, 11
 ; LP64E-FPELIM-NEXT:    lui a0, %hi(.LCPI11_1)
 ; LP64E-FPELIM-NEXT:    ld a2, %lo(.LCPI11_1)(a0)
 ; LP64E-FPELIM-NEXT:    lui a0, %hi(.LCPI11_2)
@@ -2569,7 +2566,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64E-FPELIM-NEXT:    li a1, 11
 ; LP64E-FPELIM-NEXT:    li a4, 12
 ; LP64E-FPELIM-NEXT:    li a5, 13
-; LP64E-FPELIM-NEXT:    sd a6, 0(sp)
+; LP64E-FPELIM-NEXT:    sd t1, 0(sp)
+; LP64E-FPELIM-NEXT:    sd t0, 8(sp)
+; LP64E-FPELIM-NEXT:    sd a7, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a6, 24(sp)
 ; LP64E-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; LP64E-FPELIM-NEXT:    ld ra, 48(sp) # 8-byte Folded Reload
 ; LP64E-FPELIM-NEXT:    addi sp, sp, 56
@@ -2582,19 +2582,16 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64E-WITHFP-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 64
 ; LP64E-WITHFP-NEXT:    li a0, 17
+; LP64E-WITHFP-NEXT:    li a1, 16
+; LP64E-WITHFP-NEXT:    sd a1, 32(sp)
 ; LP64E-WITHFP-NEXT:    sd a0, 40(sp)
-; LP64E-WITHFP-NEXT:    li a0, 16
-; LP64E-WITHFP-NEXT:    lui a1, %hi(.LCPI11_0)
-; LP64E-WITHFP-NEXT:    ld a1, %lo(.LCPI11_0)(a1)
-; LP64E-WITHFP-NEXT:    sd a0, 32(sp)
-; LP64E-WITHFP-NEXT:    li a0, 15
-; LP64E-WITHFP-NEXT:    sd a0, 24(sp)
-; LP64E-WITHFP-NEXT:    sd a1, 16(sp)
-; LP64E-WITHFP-NEXT:    li a0, 14
-; LP64E-WITHFP-NEXT:    sd a0, 8(sp)
+; LP64E-WITHFP-NEXT:    li a6, 15
+; LP64E-WITHFP-NEXT:    lui a0, %hi(.LCPI11_0)
+; LP64E-WITHFP-NEXT:    ld a7, %lo(.LCPI11_0)(a0)
+; LP64E-WITHFP-NEXT:    li t0, 14
 ; LP64E-WITHFP-NEXT:    lui a0, 2384
 ; LP64E-WITHFP-NEXT:    addiw a0, a0, 761
-; LP64E-WITHFP-NEXT:    slli a6, a0, 11
+; LP64E-WITHFP-NEXT:    slli t1, a0, 11
 ; LP64E-WITHFP-NEXT:    lui a0, %hi(.LCPI11_1)
 ; LP64E-WITHFP-NEXT:    ld a2, %lo(.LCPI11_1)(a0)
 ; LP64E-WITHFP-NEXT:    lui a0, %hi(.LCPI11_2)
@@ -2603,7 +2600,10 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64E-WITHFP-NEXT:    li a1, 11
 ; LP64E-WITHFP-NEXT:    li a4, 12
 ; LP64E-WITHFP-NEXT:    li a5, 13
-; LP64E-WITHFP-NEXT:    sd a6, 0(sp)
+; LP64E-WITHFP-NEXT:    sd t1, 0(sp)
+; LP64E-WITHFP-NEXT:    sd t0, 8(sp)
+; LP64E-WITHFP-NEXT:    sd a7, 16(sp)
+; LP64E-WITHFP-NEXT:    sd a6, 24(sp)
 ; LP64E-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; LP64E-WITHFP-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; LP64E-WITHFP-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
@@ -2623,14 +2623,14 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va6_no_fixed_args:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 36(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 36(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 16(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 20
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
@@ -2642,14 +2642,14 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 0(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2660,14 +2660,14 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va6_no_fixed_args:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 36(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 16(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 20
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
@@ -2676,12 +2676,12 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32E-FPELIM-LABEL: va6_no_fixed_args:
 ; ILP32E-FPELIM:       # %bb.0:
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, -28
-; ILP32E-FPELIM-NEXT:    sw a5, 24(sp)
 ; ILP32E-FPELIM-NEXT:    sw a4, 20(sp)
-; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
-; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
-; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a5, 24(sp)
 ; ILP32E-FPELIM-NEXT:    sw a0, 4(sp)
+; ILP32E-FPELIM-NEXT:    sw a1, 8(sp)
+; ILP32E-FPELIM-NEXT:    sw a2, 12(sp)
+; ILP32E-FPELIM-NEXT:    sw a3, 16(sp)
 ; ILP32E-FPELIM-NEXT:    addi a1, sp, 8
 ; ILP32E-FPELIM-NEXT:    sw a1, 0(sp)
 ; ILP32E-FPELIM-NEXT:    addi sp, sp, 28
@@ -2693,12 +2693,12 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32E-WITHFP-NEXT:    sw ra, 8(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    sw s0, 4(sp) # 4-byte Folded Spill
 ; ILP32E-WITHFP-NEXT:    addi s0, sp, 12
-; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
 ; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
-; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
 ; ILP32E-WITHFP-NEXT:    sw a0, 0(s0)
+; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32E-WITHFP-NEXT:    addi a1, s0, 4
 ; ILP32E-WITHFP-NEXT:    sw a1, -12(s0)
 ; ILP32E-WITHFP-NEXT:    lw ra, 8(sp) # 4-byte Folded Reload
@@ -2709,14 +2709,14 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va6_no_fixed_args:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 24
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
@@ -2728,14 +2728,14 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -2746,12 +2746,12 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64E-FPELIM-LABEL: va6_no_fixed_args:
 ; LP64E-FPELIM:       # %bb.0:
 ; LP64E-FPELIM-NEXT:    addi sp, sp, -56
-; LP64E-FPELIM-NEXT:    sd a5, 48(sp)
 ; LP64E-FPELIM-NEXT:    sd a4, 40(sp)
-; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
-; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
-; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a5, 48(sp)
 ; LP64E-FPELIM-NEXT:    sd a0, 8(sp)
+; LP64E-FPELIM-NEXT:    sd a1, 16(sp)
+; LP64E-FPELIM-NEXT:    sd a2, 24(sp)
+; LP64E-FPELIM-NEXT:    sd a3, 32(sp)
 ; LP64E-FPELIM-NEXT:    addi a1, sp, 16
 ; LP64E-FPELIM-NEXT:    sd a1, 0(sp)
 ; LP64E-FPELIM-NEXT:    addi sp, sp, 56
@@ -2763,12 +2763,12 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64E-WITHFP-NEXT:    sd ra, 16(sp) # 8-byte Folded Spill
 ; LP64E-WITHFP-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
 ; LP64E-WITHFP-NEXT:    addi s0, sp, 24
-; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
 ; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
-; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
-; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
 ; LP64E-WITHFP-NEXT:    sd a0, 0(s0)
+; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64E-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64E-WITHFP-NEXT:    sd a1, -24(s0)
 ; LP64E-WITHFP-NEXT:    ld ra, 16(sp) # 8-byte Folded Reload
@@ -2794,25 +2794,25 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a0, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    lui t0, 24414
 ; ILP32-ILP32F-FPELIM-NEXT:    add t0, sp, t0
-; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 300(t0)
-; ILP32-ILP32F-FPELIM-NEXT:    lui a7, 24414
-; ILP32-ILP32F-FPELIM-NEXT:    add a7, sp, a7
-; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 296(a7)
-; ILP32-ILP32F-FPELIM-NEXT:    lui a6, 24414
-; ILP32-ILP32F-FPELIM-NEXT:    add a6, sp, a6
-; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 292(a6)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 292(t0)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    add a5, sp, a5
+; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 296(a5)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    add a5, sp, a5
+; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 300(a5)
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 24414
 ; ILP32-ILP32F-FPELIM-NEXT:    add a5, sp, a5
-; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 288(a5)
-; ILP32-ILP32F-FPELIM-NEXT:    lui a4, 24414
-; ILP32-ILP32F-FPELIM-NEXT:    add a4, sp, a4
-; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 284(a4)
-; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 24414
-; ILP32-ILP32F-FPELIM-NEXT:    add a3, sp, a3
-; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 280(a3)
-; ILP32-ILP32F-FPELIM-NEXT:    lui a2, 24414
-; ILP32-ILP32F-FPELIM-NEXT:    add a2, sp, a2
-; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 276(a2)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 276(a5)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    add a1, sp, a1
+; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 280(a1)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    add a1, sp, a1
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 284(a1)
+; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
+; ILP32-ILP32F-FPELIM-NEXT:    add a1, sp, a1
+; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 288(a1)
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, a1, 280
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, sp, a1
@@ -2836,13 +2836,13 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, -1728
 ; ILP32-ILP32F-WITHFP-NEXT:    sub sp, sp, a0
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a0, a1
-; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a2, 24414
 ; ILP32-ILP32F-WITHFP-NEXT:    sub a2, s0, a2
@@ -2864,25 +2864,25 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a0, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui t0, 24414
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add t0, sp, t0
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 300(t0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a7, 24414
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a7, sp, a7
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 296(a7)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a6, 24414
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a6, sp, a6
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 292(a6)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 292(t0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a5, sp, a5
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 296(a5)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a5, sp, a5
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 300(a5)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 24414
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a5, sp, a5
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 288(a5)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a4, 24414
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a4, sp, a4
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 284(a4)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 24414
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a3, sp, a3
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 280(a3)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a2, 24414
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, sp, a2
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 276(a2)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 276(a5)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, sp, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 280(a1)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, sp, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 284(a1)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, sp, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 288(a1)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, a1, 280
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, sp, a1
@@ -2904,16 +2904,16 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; ILP32E-FPELIM-NEXT:    sw a5, 284(a6)
 ; ILP32E-FPELIM-NEXT:    lui a5, 24414
 ; ILP32E-FPELIM-NEXT:    add a5, sp, a5
-; ILP32E-FPELIM-NEXT:    sw a4, 280(a5)
-; ILP32E-FPELIM-NEXT:    lui a4, 24414
-; ILP32E-FPELIM-NEXT:    add a4, sp, a4
-; ILP32E-FPELIM-NEXT:    sw a3, 276(a4)
-; ILP32E-FPELIM-NEXT:    lui a3, 24414
-; ILP32E-FPELIM-NEXT:    add a3, sp, a3
-; ILP32E-FPELIM-NEXT:    sw a2, 272(a3)
-; ILP32E-FPELIM-NEXT:    lui a2, 24414
-; ILP32E-FPELIM-NEXT:    add a2, sp, a2
-; ILP32E-FPELIM-NEXT:    sw a1, 268(a2)
+; ILP32E-FPELIM-NEXT:    sw a1, 268(a5)
+; ILP32E-FPELIM-NEXT:    lui a1, 24414
+; ILP32E-FPELIM-NEXT:    add a1, sp, a1
+; ILP32E-FPELIM-NEXT:    sw a2, 272(a1)
+; ILP32E-FPELIM-NEXT:    lui a1, 24414
+; ILP32E-FPELIM-NEXT:    add a1, sp, a1
+; ILP32E-FPELIM-NEXT:    sw a3, 276(a1)
+; ILP32E-FPELIM-NEXT:    lui a1, 24414
+; ILP32E-FPELIM-NEXT:    add a1, sp, a1
+; ILP32E-FPELIM-NEXT:    sw a4, 280(a1)
 ; ILP32E-FPELIM-NEXT:    lui a1, 24414
 ; ILP32E-FPELIM-NEXT:    addi a1, a1, 272
 ; ILP32E-FPELIM-NEXT:    add a1, sp, a1
@@ -2938,10 +2938,10 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; ILP32E-WITHFP-NEXT:    sub sp, sp, a0
 ; ILP32E-WITHFP-NEXT:    mv a0, a1
 ; ILP32E-WITHFP-NEXT:    sw a5, 20(s0)
-; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
-; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
-; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32E-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32E-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32E-WITHFP-NEXT:    sw a3, 12(s0)
+; ILP32E-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32E-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32E-WITHFP-NEXT:    lui a2, 24414
 ; ILP32E-WITHFP-NEXT:    sub a2, s0, a2
@@ -2964,12 +2964,6 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 280(a0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
-; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 328(a0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
-; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 320(a0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 284
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
@@ -2981,14 +2975,20 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 312(a1)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, sp, a1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 304(a1)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 320(a1)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, sp, a1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 296(a1)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 328(a1)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, sp, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 288(a1)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, sp, a1
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 296(a1)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
+; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, sp, a1
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 304(a1)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a1, a1, 336
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add sp, sp, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
@@ -3007,17 +3007,17 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, -1680
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sub sp, sp, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 12
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 24414
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sub a1, s0, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -288(a1)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 24414
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a1, a1, -1680
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add sp, sp, a1
@@ -3033,27 +3033,27 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64E-FPELIM-NEXT:    sub sp, sp, a0
 ; LP64E-FPELIM-NEXT:    .cfi_def_cfa_offset 100000064
 ; LP64E-FPELIM-NEXT:    lui a0, 24414
-; LP64E-FPELIM-NEXT:    add a0, sp, a0
-; LP64E-FPELIM-NEXT:    sd a1, 280(a0)
-; LP64E-FPELIM-NEXT:    lui a0, 24414
 ; LP64E-FPELIM-NEXT:    addiw a0, a0, 284
 ; LP64E-FPELIM-NEXT:    add a0, sp, a0
 ; LP64E-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64E-FPELIM-NEXT:    lui a0, 24414
 ; LP64E-FPELIM-NEXT:    add a0, sp, a0
+; LP64E-FPELIM-NEXT:    sd a1, 280(a0)
+; LP64E-FPELIM-NEXT:    lui a0, 24414
+; LP64E-FPELIM-NEXT:    add a0, sp, a0
 ; LP64E-FPELIM-NEXT:    lw a0, 280(a0)
 ; LP64E-FPELIM-NEXT:    lui a1, 24414
 ; LP64E-FPELIM-NEXT:    add a1, sp, a1
 ; LP64E-FPELIM-NEXT:    sd a5, 312(a1)
 ; LP64E-FPELIM-NEXT:    lui a1, 24414
 ; LP64E-FPELIM-NEXT:    add a1, sp, a1
-; LP64E-FPELIM-NEXT:    sd a4, 304(a1)
+; LP64E-FPELIM-NEXT:    sd a2, 288(a1)
 ; LP64E-FPELIM-NEXT:    lui a1, 24414
 ; LP64E-FPELIM-NEXT:    add a1, sp, a1
 ; LP64E-FPELIM-NEXT:    sd a3, 296(a1)
 ; LP64E-FPELIM-NEXT:    lui a1, 24414
 ; LP64E-FPELIM-NEXT:    add a1, sp, a1
-; LP64E-FPELIM-NEXT:    sd a2, 288(a1)
+; LP64E-FPELIM-NEXT:    sd a4, 304(a1)
 ; LP64E-FPELIM-NEXT:    lui a1, 24414
 ; LP64E-FPELIM-NEXT:    addiw a1, a1, 320
 ; LP64E-FPELIM-NEXT:    add sp, sp, a1
@@ -3072,16 +3072,16 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; LP64E-WITHFP-NEXT:    lui a0, 24414
 ; LP64E-WITHFP-NEXT:    addiw a0, a0, -1704
 ; LP64E-WITHFP-NEXT:    sub sp, sp, a0
-; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64E-WITHFP-NEXT:    addi a0, s0, 12
-; LP64E-WITHFP-NEXT:    lui a1, 24414
-; LP64E-WITHFP-NEXT:    sub a1, s0, a1
-; LP64E-WITHFP-NEXT:    sd a0, -288(a1)
+; LP64E-WITHFP-NEXT:    lui a6, 24414
+; LP64E-WITHFP-NEXT:    sub a6, s0, a6
+; LP64E-WITHFP-NEXT:    sd a0, -288(a6)
+; LP64E-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64E-WITHFP-NEXT:    lw a0, 8(s0)
 ; LP64E-WITHFP-NEXT:    sd a5, 40(s0)
-; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
-; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64E-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64E-WITHFP-NEXT:    sd a3, 24(s0)
+; LP64E-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64E-WITHFP-NEXT:    lui a1, 24414
 ; LP64E-WITHFP-NEXT:    addiw a1, a1, -1704
 ; LP64E-WITHFP-NEXT:    add sp, sp, a1
diff --git a/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll
index 3e14317a0047458577fc190c4f100a9389900b33..cae59c79aaaa8adca7db41b4e85c850bf80e96e7 100644
--- a/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/llvm/test/CodeGen/RISCV/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -18,13 +18,13 @@ define void @lshr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    srlw a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_4bytes:
@@ -51,13 +51,13 @@ define void @lshr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a4
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    srl a0, a0, a1
-; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i32, ptr %src.ptr, align 1
   %byteOff = load i32, ptr %byteOff.ptr, align 1
@@ -82,13 +82,13 @@ define void @shl_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    sllw a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_4bytes:
@@ -115,13 +115,13 @@ define void @shl_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a4
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    sll a0, a0, a1
-; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i32, ptr %src.ptr, align 1
   %byteOff = load i32, ptr %byteOff.ptr, align 1
@@ -146,13 +146,13 @@ define void @ashr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    sraw a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_4bytes:
@@ -179,13 +179,13 @@ define void @ashr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a4
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    sra a0, a0, a1
-; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i32, ptr %src.ptr, align 1
   %byteOff = load i32, ptr %byteOff.ptr, align 1
@@ -244,21 +244,21 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli a3, a3, 35
 ; RV64I-NEXT:    or a1, a3, a1
 ; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 48
+; RV64I-NEXT:    srli a3, a0, 56
+; RV64I-NEXT:    srli a4, a0, 32
+; RV64I-NEXT:    srli a5, a0, 40
+; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
 ; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_8bytes:
@@ -309,20 +309,20 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:  .LBB3_3:
 ; RV32I-NEXT:    srai a4, a4, 31
 ; RV32I-NEXT:    and a1, a4, a1
-; RV32I-NEXT:    sb a1, 4(a2)
 ; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    srli a4, a1, 24
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a5, 5(a2)
 ; RV32I-NEXT:    sb a3, 6(a2)
-; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 7(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 5(a2)
-; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 7(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i64, ptr %src.ptr, align 1
   %byteOff = load i64, ptr %byteOff.ptr, align 1
@@ -380,21 +380,21 @@ define void @shl_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli a3, a3, 35
 ; RV64I-NEXT:    or a1, a3, a1
 ; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 48
+; RV64I-NEXT:    srli a3, a0, 56
+; RV64I-NEXT:    srli a4, a0, 32
+; RV64I-NEXT:    srli a5, a0, 40
+; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
 ; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_8bytes:
@@ -445,20 +445,20 @@ define void @shl_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:  .LBB4_3:
 ; RV32I-NEXT:    srai a4, a4, 31
 ; RV32I-NEXT:    and a1, a4, a1
-; RV32I-NEXT:    sb a1, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
 ; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    srli a4, a1, 24
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
 ; RV32I-NEXT:    sb a3, 2(a2)
-; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 3(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    sb a4, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
 ; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a3, 7(a2)
 ; RV32I-NEXT:    ret
   %src = load i64, ptr %src.ptr, align 1
   %byteOff = load i64, ptr %byteOff.ptr, align 1
@@ -516,21 +516,21 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli a3, a3, 35
 ; RV64I-NEXT:    or a1, a3, a1
 ; RV64I-NEXT:    sra a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 48
+; RV64I-NEXT:    srli a3, a0, 56
+; RV64I-NEXT:    srli a4, a0, 32
+; RV64I-NEXT:    srli a5, a0, 40
+; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
 ; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_8bytes:
@@ -581,20 +581,20 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll a3, a3, a4
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:  .LBB5_3:
-; RV32I-NEXT:    sb a1, 4(a2)
 ; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    srli a4, a1, 24
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a5, 5(a2)
 ; RV32I-NEXT:    sb a3, 6(a2)
-; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 7(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 5(a2)
-; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 7(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i64, ptr %src.ptr, align 1
   %byteOff = load i64, ptr %byteOff.ptr, align 1
@@ -689,36 +689,36 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:  .LBB6_3:
 ; RV64I-NEXT:    srai a4, a4, 63
 ; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    sb a1, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
 ; RV64I-NEXT:    sb a3, 15(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 14(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 13(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 12(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a5, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a3, 6(a2)
 ; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
 ; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_16bytes:
@@ -774,14 +774,14 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, a7
 ; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a0, 12(sp)
 ; RV32I-NEXT:    andi a0, a1, 12
 ; RV32I-NEXT:    mv a3, sp
 ; RV32I-NEXT:    add a0, a3, a0
@@ -805,34 +805,34 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll a7, t1, a7
 ; RV32I-NEXT:    or a7, a5, a7
 ; RV32I-NEXT:    srl a0, a0, a1
-; RV32I-NEXT:    sb a5, 8(a2)
-; RV32I-NEXT:    sb a0, 12(a2)
-; RV32I-NEXT:    sb a3, 0(a2)
-; RV32I-NEXT:    sb a6, 4(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli t1, a0, 24
+; RV32I-NEXT:    srli t2, a0, 8
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb a1, 14(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 15(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t1, 15(a2)
 ; RV32I-NEXT:    srli a0, a7, 16
+; RV32I-NEXT:    srli a1, a7, 24
+; RV32I-NEXT:    srli a7, a7, 8
+; RV32I-NEXT:    sb a5, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
-; RV32I-NEXT:    srli a0, a7, 24
-; RV32I-NEXT:    sb a0, 11(a2)
-; RV32I-NEXT:    srli a0, a7, 8
-; RV32I-NEXT:    sb a0, 9(a2)
+; RV32I-NEXT:    sb a1, 11(a2)
 ; RV32I-NEXT:    srli a0, a4, 16
-; RV32I-NEXT:    sb a0, 2(a2)
-; RV32I-NEXT:    srli a0, a4, 24
-; RV32I-NEXT:    sb a0, 3(a2)
+; RV32I-NEXT:    srli a1, a4, 24
 ; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 0(a2)
 ; RV32I-NEXT:    sb a4, 1(a2)
+; RV32I-NEXT:    sb a0, 2(a2)
+; RV32I-NEXT:    sb a1, 3(a2)
 ; RV32I-NEXT:    srli a0, t0, 16
+; RV32I-NEXT:    srli a1, t0, 24
+; RV32I-NEXT:    srli a3, t0, 8
+; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a3, 5(a2)
 ; RV32I-NEXT:    sb a0, 6(a2)
-; RV32I-NEXT:    srli a0, t0, 24
-; RV32I-NEXT:    sb a0, 7(a2)
-; RV32I-NEXT:    srli a0, t0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a1, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -928,36 +928,36 @@ define void @lshr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:  .LBB7_3:
 ; RV64I-NEXT:    srai a4, a4, 63
 ; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    sb a1, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
 ; RV64I-NEXT:    sb a3, 15(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 14(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 13(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 12(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a5, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a3, 6(a2)
 ; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
 ; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_16bytes_wordOff:
@@ -1004,50 +1004,50 @@ define void @lshr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV32I-NEXT:    or a0, a0, a6
 ; RV32I-NEXT:    or a0, a0, a7
 ; RV32I-NEXT:    lbu a1, 0(a1)
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a0, 12(sp)
 ; RV32I-NEXT:    slli a1, a1, 2
 ; RV32I-NEXT:    andi a1, a1, 12
 ; RV32I-NEXT:    mv a0, sp
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    lw a3, 12(a0)
+; RV32I-NEXT:    lw a3, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sb a1, 8(a2)
-; RV32I-NEXT:    sb a3, 12(a2)
-; RV32I-NEXT:    sb a4, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    lw a0, 12(a0)
 ; RV32I-NEXT:    srli a5, a1, 16
+; RV32I-NEXT:    srli a6, a1, 24
+; RV32I-NEXT:    srli a7, a1, 8
+; RV32I-NEXT:    sb a1, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
 ; RV32I-NEXT:    sb a5, 10(a2)
-; RV32I-NEXT:    srli a5, a1, 24
-; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 9(a2)
-; RV32I-NEXT:    srli a1, a3, 16
+; RV32I-NEXT:    sb a6, 11(a2)
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a5, a0, 24
+; RV32I-NEXT:    srli a6, a0, 8
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
 ; RV32I-NEXT:    sb a1, 14(a2)
-; RV32I-NEXT:    srli a1, a3, 24
-; RV32I-NEXT:    sb a1, 15(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
-; RV32I-NEXT:    srli a1, a4, 16
-; RV32I-NEXT:    sb a1, 2(a2)
+; RV32I-NEXT:    sb a5, 15(a2)
+; RV32I-NEXT:    srli a0, a4, 16
 ; RV32I-NEXT:    srli a1, a4, 24
+; RV32I-NEXT:    srli a5, a4, 8
+; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a0, 2(a2)
 ; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 1(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a1, a3, 24
+; RV32I-NEXT:    srli a4, a3, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a0, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -1143,36 +1143,36 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:  .LBB8_3:
 ; RV64I-NEXT:    srai a4, a4, 63
 ; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    sb a1, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
 ; RV64I-NEXT:    sb a3, 7(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 6(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 5(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 4(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
 ; RV64I-NEXT:    sb a3, 3(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 2(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 1(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_16bytes:
@@ -1228,14 +1228,14 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, a7
 ; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw a5, 24(sp)
-; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw a5, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    andi a0, a1, 12
 ; RV32I-NEXT:    addi a3, sp, 16
 ; RV32I-NEXT:    sub a3, a3, a0
@@ -1259,34 +1259,34 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srl a4, a4, a7
 ; RV32I-NEXT:    or a4, a5, a4
 ; RV32I-NEXT:    sll a0, a0, a1
-; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    srli a5, a5, 24
+; RV32I-NEXT:    srli a1, a4, 16
+; RV32I-NEXT:    srli a7, a4, 8
+; RV32I-NEXT:    sb a4, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
+; RV32I-NEXT:    sb a1, 10(a2)
 ; RV32I-NEXT:    sb a5, 11(a2)
 ; RV32I-NEXT:    srli a3, a3, 24
+; RV32I-NEXT:    srli a1, t1, 16
+; RV32I-NEXT:    srli a4, t1, 8
+; RV32I-NEXT:    sb t1, 12(a2)
+; RV32I-NEXT:    sb a4, 13(a2)
+; RV32I-NEXT:    sb a1, 14(a2)
 ; RV32I-NEXT:    sb a3, 15(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    srli a0, a6, 24
-; RV32I-NEXT:    sb a0, 7(a2)
-; RV32I-NEXT:    sb a4, 8(a2)
-; RV32I-NEXT:    sb t1, 12(a2)
+; RV32I-NEXT:    srli a1, t0, 16
+; RV32I-NEXT:    srli a3, t0, 8
 ; RV32I-NEXT:    sb t0, 4(a2)
-; RV32I-NEXT:    srli a0, a4, 16
-; RV32I-NEXT:    sb a0, 10(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 9(a2)
-; RV32I-NEXT:    srli a0, t1, 16
-; RV32I-NEXT:    sb a0, 14(a2)
-; RV32I-NEXT:    srli a0, t1, 8
-; RV32I-NEXT:    sb a0, 13(a2)
-; RV32I-NEXT:    srli a0, t0, 16
-; RV32I-NEXT:    sb a0, 6(a2)
-; RV32I-NEXT:    srli a0, t0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a3, 5(a2)
+; RV32I-NEXT:    sb a1, 6(a2)
+; RV32I-NEXT:    sb a0, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -1382,36 +1382,36 @@ define void @shl_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV64I-NEXT:  .LBB9_3:
 ; RV64I-NEXT:    srai a4, a4, 63
 ; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    sb a1, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
 ; RV64I-NEXT:    sb a3, 7(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 6(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 5(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 4(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
 ; RV64I-NEXT:    sb a3, 3(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 2(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 1(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_16bytes_wordOff:
@@ -1458,50 +1458,50 @@ define void @shl_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV32I-NEXT:    or a0, a0, a6
 ; RV32I-NEXT:    or a0, a0, a7
 ; RV32I-NEXT:    lbu a1, 0(a1)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw a5, 24(sp)
-; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw a5, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    slli a1, a1, 2
 ; RV32I-NEXT:    andi a1, a1, 12
 ; RV32I-NEXT:    addi a0, sp, 16
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    lw a3, 12(a0)
+; RV32I-NEXT:    lw a3, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sb a1, 8(a2)
-; RV32I-NEXT:    sb a3, 12(a2)
-; RV32I-NEXT:    sb a4, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    lw a0, 12(a0)
 ; RV32I-NEXT:    srli a5, a1, 16
+; RV32I-NEXT:    srli a6, a1, 24
+; RV32I-NEXT:    srli a7, a1, 8
+; RV32I-NEXT:    sb a1, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
 ; RV32I-NEXT:    sb a5, 10(a2)
-; RV32I-NEXT:    srli a5, a1, 24
-; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 9(a2)
-; RV32I-NEXT:    srli a1, a3, 16
+; RV32I-NEXT:    sb a6, 11(a2)
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a5, a0, 24
+; RV32I-NEXT:    srli a6, a0, 8
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
 ; RV32I-NEXT:    sb a1, 14(a2)
-; RV32I-NEXT:    srli a1, a3, 24
-; RV32I-NEXT:    sb a1, 15(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
-; RV32I-NEXT:    srli a1, a4, 16
-; RV32I-NEXT:    sb a1, 2(a2)
+; RV32I-NEXT:    sb a5, 15(a2)
+; RV32I-NEXT:    srli a0, a4, 16
 ; RV32I-NEXT:    srli a1, a4, 24
+; RV32I-NEXT:    srli a5, a4, 8
+; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a0, 2(a2)
 ; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 1(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a1, a3, 24
+; RV32I-NEXT:    srli a4, a3, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a0, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -1598,36 +1598,36 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sll a3, a3, a4
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:  .LBB10_3:
-; RV64I-NEXT:    sb a1, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
 ; RV64I-NEXT:    sb a3, 15(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 14(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 13(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 12(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a5, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a3, 6(a2)
 ; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
 ; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_16bytes:
@@ -1684,14 +1684,14 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a7
 ; RV32I-NEXT:    or a1, a1, t0
 ; RV32I-NEXT:    srai a0, a0, 31
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 20(sp)
 ; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a0, 20(sp)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    andi a0, a1, 12
 ; RV32I-NEXT:    mv a3, sp
 ; RV32I-NEXT:    add a0, a3, a0
@@ -1715,34 +1715,34 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll a7, t1, a7
 ; RV32I-NEXT:    or a7, a5, a7
 ; RV32I-NEXT:    sra a0, a0, a1
-; RV32I-NEXT:    sb a5, 8(a2)
-; RV32I-NEXT:    sb a0, 12(a2)
-; RV32I-NEXT:    sb a3, 0(a2)
-; RV32I-NEXT:    sb a6, 4(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli t1, a0, 24
+; RV32I-NEXT:    srli t2, a0, 8
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb a1, 14(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 15(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 13(a2)
+; RV32I-NEXT:    sb t1, 15(a2)
 ; RV32I-NEXT:    srli a0, a7, 16
+; RV32I-NEXT:    srli a1, a7, 24
+; RV32I-NEXT:    srli a7, a7, 8
+; RV32I-NEXT:    sb a5, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
-; RV32I-NEXT:    srli a0, a7, 24
-; RV32I-NEXT:    sb a0, 11(a2)
-; RV32I-NEXT:    srli a0, a7, 8
-; RV32I-NEXT:    sb a0, 9(a2)
+; RV32I-NEXT:    sb a1, 11(a2)
 ; RV32I-NEXT:    srli a0, a4, 16
-; RV32I-NEXT:    sb a0, 2(a2)
-; RV32I-NEXT:    srli a0, a4, 24
-; RV32I-NEXT:    sb a0, 3(a2)
+; RV32I-NEXT:    srli a1, a4, 24
 ; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 0(a2)
 ; RV32I-NEXT:    sb a4, 1(a2)
+; RV32I-NEXT:    sb a0, 2(a2)
+; RV32I-NEXT:    sb a1, 3(a2)
 ; RV32I-NEXT:    srli a0, t0, 16
+; RV32I-NEXT:    srli a1, t0, 24
+; RV32I-NEXT:    srli a3, t0, 8
+; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a3, 5(a2)
 ; RV32I-NEXT:    sb a0, 6(a2)
-; RV32I-NEXT:    srli a0, t0, 24
-; RV32I-NEXT:    sb a0, 7(a2)
-; RV32I-NEXT:    srli a0, t0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a1, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -1838,36 +1838,36 @@ define void @ashr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    sll a3, a3, a4
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:  .LBB11_3:
-; RV64I-NEXT:    sb a1, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
 ; RV64I-NEXT:    sb a3, 15(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 14(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 13(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 12(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a5, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a3, 6(a2)
 ; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
 ; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_16bytes_wordOff:
@@ -1915,50 +1915,50 @@ define void @ashr_16bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV32I-NEXT:    or a6, a6, a7
 ; RV32I-NEXT:    lbu a1, 0(a1)
 ; RV32I-NEXT:    srai a0, a0, 31
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 20(sp)
 ; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a0, 20(sp)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    slli a1, a1, 2
 ; RV32I-NEXT:    andi a1, a1, 12
 ; RV32I-NEXT:    mv a0, sp
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    lw a3, 12(a0)
+; RV32I-NEXT:    lw a3, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sb a1, 8(a2)
-; RV32I-NEXT:    sb a3, 12(a2)
-; RV32I-NEXT:    sb a4, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    lw a0, 12(a0)
 ; RV32I-NEXT:    srli a5, a1, 16
+; RV32I-NEXT:    srli a6, a1, 24
+; RV32I-NEXT:    srli a7, a1, 8
+; RV32I-NEXT:    sb a1, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
 ; RV32I-NEXT:    sb a5, 10(a2)
-; RV32I-NEXT:    srli a5, a1, 24
-; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 9(a2)
-; RV32I-NEXT:    srli a1, a3, 16
+; RV32I-NEXT:    sb a6, 11(a2)
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a5, a0, 24
+; RV32I-NEXT:    srli a6, a0, 8
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
 ; RV32I-NEXT:    sb a1, 14(a2)
-; RV32I-NEXT:    srli a1, a3, 24
-; RV32I-NEXT:    sb a1, 15(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
-; RV32I-NEXT:    srli a1, a4, 16
-; RV32I-NEXT:    sb a1, 2(a2)
+; RV32I-NEXT:    sb a5, 15(a2)
+; RV32I-NEXT:    srli a0, a4, 16
 ; RV32I-NEXT:    srli a1, a4, 24
+; RV32I-NEXT:    srli a5, a4, 8
+; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a0, 2(a2)
 ; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 1(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a1, a3, 24
+; RV32I-NEXT:    srli a4, a3, 8
+; RV32I-NEXT:    sb a3, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a0, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -2083,97 +2083,97 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a6
-; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    sd zero, 48(sp)
-; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 32(sp)
-; RV64I-NEXT:    sd a0, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    sd zero, 48(sp)
+; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a0, 24(sp)
 ; RV64I-NEXT:    andi a0, a1, 24
 ; RV64I-NEXT:    mv a3, sp
 ; RV64I-NEXT:    add a0, a3, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    slli a1, a1, 3
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
-; RV64I-NEXT:    andi a0, a1, 56
-; RV64I-NEXT:    xori t0, a0, 63
-; RV64I-NEXT:    slli a0, a5, 1
-; RV64I-NEXT:    sll a0, a0, t0
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    srl a3, a3, a1
+; RV64I-NEXT:    slli a5, a1, 3
+; RV64I-NEXT:    ld a6, 16(a0)
+; RV64I-NEXT:    ld a7, 24(a0)
+; RV64I-NEXT:    srl a0, a4, a5
+; RV64I-NEXT:    andi a1, a5, 56
+; RV64I-NEXT:    xori t0, a1, 63
+; RV64I-NEXT:    slli a1, a6, 1
+; RV64I-NEXT:    sll a1, a1, t0
+; RV64I-NEXT:    or a1, a0, a1
+; RV64I-NEXT:    srl a3, a3, a5
 ; RV64I-NEXT:    slli a4, a4, 1
 ; RV64I-NEXT:    sll a4, a4, t0
 ; RV64I-NEXT:    or a4, a3, a4
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a6, 1
+; RV64I-NEXT:    srl a6, a6, a5
+; RV64I-NEXT:    slli t1, a7, 1
 ; RV64I-NEXT:    sll t0, t1, t0
-; RV64I-NEXT:    or t0, a5, t0
-; RV64I-NEXT:    srl a1, a6, a1
-; RV64I-NEXT:    sb a5, 16(a2)
-; RV64I-NEXT:    sb a1, 24(a2)
+; RV64I-NEXT:    or t0, a6, t0
+; RV64I-NEXT:    srl a5, a7, a5
+; RV64I-NEXT:    srli a7, a5, 56
+; RV64I-NEXT:    srli t1, a5, 48
+; RV64I-NEXT:    srli t2, a5, 40
+; RV64I-NEXT:    srli t3, a5, 32
+; RV64I-NEXT:    sb t3, 28(a2)
+; RV64I-NEXT:    sb t2, 29(a2)
+; RV64I-NEXT:    sb t1, 30(a2)
+; RV64I-NEXT:    sb a7, 31(a2)
+; RV64I-NEXT:    srli a7, a5, 24
+; RV64I-NEXT:    srli t1, a5, 16
+; RV64I-NEXT:    srli t2, a5, 8
+; RV64I-NEXT:    sb a5, 24(a2)
+; RV64I-NEXT:    sb t2, 25(a2)
+; RV64I-NEXT:    sb t1, 26(a2)
+; RV64I-NEXT:    sb a7, 27(a2)
+; RV64I-NEXT:    srli a5, t0, 56
+; RV64I-NEXT:    srli a7, t0, 48
+; RV64I-NEXT:    srli t1, t0, 40
+; RV64I-NEXT:    srli t2, t0, 32
+; RV64I-NEXT:    sb t2, 20(a2)
+; RV64I-NEXT:    sb t1, 21(a2)
+; RV64I-NEXT:    sb a7, 22(a2)
+; RV64I-NEXT:    sb a5, 23(a2)
+; RV64I-NEXT:    srli a5, t0, 24
+; RV64I-NEXT:    srli a7, t0, 16
+; RV64I-NEXT:    srli t0, t0, 8
+; RV64I-NEXT:    sb a6, 16(a2)
+; RV64I-NEXT:    sb t0, 17(a2)
+; RV64I-NEXT:    sb a7, 18(a2)
+; RV64I-NEXT:    sb a5, 19(a2)
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 4(a2)
+; RV64I-NEXT:    sb a7, 5(a2)
+; RV64I-NEXT:    sb a6, 6(a2)
+; RV64I-NEXT:    sb a5, 7(a2)
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a4, a4, 8
 ; RV64I-NEXT:    sb a3, 0(a2)
-; RV64I-NEXT:    sb a7, 8(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb a5, 3(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
-; RV64I-NEXT:    sb a3, 31(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 30(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 29(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 28(a2)
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
+; RV64I-NEXT:    sb a3, 15(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
-; RV64I-NEXT:    sb a3, 27(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 26(a2)
+; RV64I-NEXT:    srli a4, a1, 16
 ; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
-; RV64I-NEXT:    srli a1, t0, 56
-; RV64I-NEXT:    sb a1, 23(a2)
-; RV64I-NEXT:    srli a1, t0, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, t0, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, t0, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, t0, 24
-; RV64I-NEXT:    sb a1, 19(a2)
-; RV64I-NEXT:    srli a1, t0, 16
-; RV64I-NEXT:    sb a1, 18(a2)
-; RV64I-NEXT:    srli a1, t0, 8
-; RV64I-NEXT:    sb a1, 17(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 1(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
+; RV64I-NEXT:    sb a3, 11(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -2183,6 +2183,7 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw s0, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 72(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s2, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 64(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lbu a3, 1(a0)
 ; RV32I-NEXT:    lbu a4, 0(a0)
 ; RV32I-NEXT:    lbu a5, 2(a0)
@@ -2273,124 +2274,125 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    or a1, a1, t2
-; RV32I-NEXT:    sw zero, 60(sp)
-; RV32I-NEXT:    sw zero, 56(sp)
-; RV32I-NEXT:    sw zero, 52(sp)
 ; RV32I-NEXT:    sw zero, 48(sp)
-; RV32I-NEXT:    sw zero, 44(sp)
-; RV32I-NEXT:    sw zero, 40(sp)
-; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 52(sp)
+; RV32I-NEXT:    sw zero, 56(sp)
+; RV32I-NEXT:    sw zero, 60(sp)
 ; RV32I-NEXT:    sw zero, 32(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw t1, 24(sp)
-; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 40(sp)
+; RV32I-NEXT:    sw zero, 44(sp)
 ; RV32I-NEXT:    sw a7, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw t1, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    andi a0, a1, 28
 ; RV32I-NEXT:    mv a3, sp
-; RV32I-NEXT:    add a3, a3, a0
-; RV32I-NEXT:    lw a6, 0(a3)
-; RV32I-NEXT:    lw a7, 4(a3)
-; RV32I-NEXT:    slli a5, a1, 3
-; RV32I-NEXT:    lw t0, 8(a3)
-; RV32I-NEXT:    lw t1, 12(a3)
-; RV32I-NEXT:    srl a4, a7, a5
-; RV32I-NEXT:    andi a0, a5, 24
-; RV32I-NEXT:    xori t2, a0, 31
-; RV32I-NEXT:    slli a0, t0, 1
-; RV32I-NEXT:    sll a0, a0, t2
-; RV32I-NEXT:    or a0, a4, a0
-; RV32I-NEXT:    srl a6, a6, a5
-; RV32I-NEXT:    slli a7, a7, 1
-; RV32I-NEXT:    sll a1, a7, t2
-; RV32I-NEXT:    or a1, a6, a1
-; RV32I-NEXT:    srl a7, t1, a5
-; RV32I-NEXT:    lw t3, 16(a3)
-; RV32I-NEXT:    lw t4, 20(a3)
-; RV32I-NEXT:    lw t5, 24(a3)
-; RV32I-NEXT:    lw t6, 28(a3)
-; RV32I-NEXT:    slli a3, t3, 1
-; RV32I-NEXT:    sll a3, a3, t2
-; RV32I-NEXT:    or a3, a7, a3
-; RV32I-NEXT:    srl t0, t0, a5
-; RV32I-NEXT:    slli t1, t1, 1
-; RV32I-NEXT:    sll t1, t1, t2
-; RV32I-NEXT:    or t1, t0, t1
-; RV32I-NEXT:    srl s0, t4, a5
+; RV32I-NEXT:    add a6, a3, a0
+; RV32I-NEXT:    lw a3, 0(a6)
+; RV32I-NEXT:    lw a4, 4(a6)
+; RV32I-NEXT:    slli t1, a1, 3
+; RV32I-NEXT:    lw a7, 8(a6)
+; RV32I-NEXT:    lw t0, 12(a6)
+; RV32I-NEXT:    srl a0, a4, t1
+; RV32I-NEXT:    andi a1, t1, 24
+; RV32I-NEXT:    xori t2, a1, 31
+; RV32I-NEXT:    slli a1, a7, 1
+; RV32I-NEXT:    sll a1, a1, t2
+; RV32I-NEXT:    or a1, a0, a1
+; RV32I-NEXT:    srl a3, a3, t1
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    sll a4, a4, t2
+; RV32I-NEXT:    or a4, a3, a4
+; RV32I-NEXT:    srl a5, t0, t1
+; RV32I-NEXT:    lw t3, 16(a6)
+; RV32I-NEXT:    lw t4, 20(a6)
+; RV32I-NEXT:    lw t5, 24(a6)
+; RV32I-NEXT:    lw t6, 28(a6)
+; RV32I-NEXT:    slli a6, t3, 1
+; RV32I-NEXT:    sll a6, a6, t2
+; RV32I-NEXT:    or a6, a5, a6
+; RV32I-NEXT:    srl a7, a7, t1
+; RV32I-NEXT:    slli t0, t0, 1
+; RV32I-NEXT:    sll t0, t0, t2
+; RV32I-NEXT:    or t0, a7, t0
+; RV32I-NEXT:    srl s0, t4, t1
 ; RV32I-NEXT:    slli s1, t5, 1
 ; RV32I-NEXT:    sll s1, s1, t2
 ; RV32I-NEXT:    or s1, s0, s1
-; RV32I-NEXT:    srl t3, t3, a5
+; RV32I-NEXT:    srl t3, t3, t1
 ; RV32I-NEXT:    slli t4, t4, 1
 ; RV32I-NEXT:    sll t4, t4, t2
 ; RV32I-NEXT:    or t4, t3, t4
-; RV32I-NEXT:    srl t5, t5, a5
+; RV32I-NEXT:    srl t5, t5, t1
 ; RV32I-NEXT:    slli s2, t6, 1
 ; RV32I-NEXT:    sll t2, s2, t2
 ; RV32I-NEXT:    or t2, t5, t2
-; RV32I-NEXT:    srl a5, t6, a5
+; RV32I-NEXT:    srl t1, t6, t1
+; RV32I-NEXT:    srli t6, t1, 24
+; RV32I-NEXT:    srli s2, t1, 16
+; RV32I-NEXT:    srli s3, t1, 8
+; RV32I-NEXT:    sb t1, 28(a2)
+; RV32I-NEXT:    sb s3, 29(a2)
+; RV32I-NEXT:    sb s2, 30(a2)
+; RV32I-NEXT:    sb t6, 31(a2)
+; RV32I-NEXT:    srli t1, t2, 24
+; RV32I-NEXT:    srli t6, t2, 16
+; RV32I-NEXT:    srli t2, t2, 8
 ; RV32I-NEXT:    sb t5, 24(a2)
-; RV32I-NEXT:    sb a5, 28(a2)
+; RV32I-NEXT:    sb t2, 25(a2)
+; RV32I-NEXT:    sb t6, 26(a2)
+; RV32I-NEXT:    sb t1, 27(a2)
+; RV32I-NEXT:    srli t1, t4, 24
+; RV32I-NEXT:    srli t2, t4, 16
+; RV32I-NEXT:    srli t4, t4, 8
 ; RV32I-NEXT:    sb t3, 16(a2)
-; RV32I-NEXT:    sb s0, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a7, 12(a2)
-; RV32I-NEXT:    sb a6, 0(a2)
-; RV32I-NEXT:    sb a4, 4(a2)
-; RV32I-NEXT:    srli a4, a5, 24
-; RV32I-NEXT:    sb a4, 31(a2)
-; RV32I-NEXT:    srli a4, a5, 16
-; RV32I-NEXT:    sb a4, 30(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a4, t2, 24
-; RV32I-NEXT:    sb a4, 27(a2)
-; RV32I-NEXT:    srli a4, t2, 16
-; RV32I-NEXT:    sb a4, 26(a2)
-; RV32I-NEXT:    srli a4, t2, 8
-; RV32I-NEXT:    sb a4, 25(a2)
-; RV32I-NEXT:    srli a4, t4, 24
-; RV32I-NEXT:    sb a4, 19(a2)
-; RV32I-NEXT:    srli a4, t4, 16
-; RV32I-NEXT:    sb a4, 18(a2)
-; RV32I-NEXT:    srli a4, t4, 8
-; RV32I-NEXT:    sb a4, 17(a2)
-; RV32I-NEXT:    srli a4, s1, 24
-; RV32I-NEXT:    sb a4, 23(a2)
-; RV32I-NEXT:    srli a4, s1, 16
-; RV32I-NEXT:    sb a4, 22(a2)
+; RV32I-NEXT:    sb t4, 17(a2)
+; RV32I-NEXT:    sb t2, 18(a2)
+; RV32I-NEXT:    sb t1, 19(a2)
+; RV32I-NEXT:    srli t1, s1, 24
+; RV32I-NEXT:    srli t2, s1, 16
 ; RV32I-NEXT:    srli s1, s1, 8
+; RV32I-NEXT:    sb s0, 20(a2)
 ; RV32I-NEXT:    sb s1, 21(a2)
-; RV32I-NEXT:    srli a4, t1, 24
-; RV32I-NEXT:    sb a4, 11(a2)
-; RV32I-NEXT:    srli a4, t1, 16
-; RV32I-NEXT:    sb a4, 10(a2)
-; RV32I-NEXT:    srli a4, t1, 8
-; RV32I-NEXT:    sb a4, 9(a2)
-; RV32I-NEXT:    srli a4, a3, 24
-; RV32I-NEXT:    sb a4, 15(a2)
-; RV32I-NEXT:    srli a4, a3, 16
-; RV32I-NEXT:    sb a4, 14(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
+; RV32I-NEXT:    sb t2, 22(a2)
+; RV32I-NEXT:    sb t1, 23(a2)
+; RV32I-NEXT:    srli t1, t0, 24
+; RV32I-NEXT:    srli t2, t0, 16
+; RV32I-NEXT:    srli t0, t0, 8
+; RV32I-NEXT:    sb a7, 8(a2)
+; RV32I-NEXT:    sb t0, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t1, 11(a2)
+; RV32I-NEXT:    srli a7, a6, 24
+; RV32I-NEXT:    srli t0, a6, 16
+; RV32I-NEXT:    srli a6, a6, 8
+; RV32I-NEXT:    sb a5, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb t0, 14(a2)
+; RV32I-NEXT:    sb a7, 15(a2)
+; RV32I-NEXT:    srli a5, a4, 24
+; RV32I-NEXT:    srli a6, a4, 16
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
+; RV32I-NEXT:    sb a6, 2(a2)
+; RV32I-NEXT:    sb a5, 3(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 3(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 2(a2)
+; RV32I-NEXT:    srli a4, a1, 16
 ; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a1, 5(a2)
+; RV32I-NEXT:    sb a4, 6(a2)
+; RV32I-NEXT:    sb a3, 7(a2)
 ; RV32I-NEXT:    lw s0, 76(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 72(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s2, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 64(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 80
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -2515,98 +2517,98 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a6
-; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    sd zero, 48(sp)
-; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 32(sp)
-; RV64I-NEXT:    sd a0, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    sd zero, 48(sp)
+; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a0, 24(sp)
 ; RV64I-NEXT:    slli a0, a1, 2
 ; RV64I-NEXT:    andi a0, a0, 24
 ; RV64I-NEXT:    mv a3, sp
 ; RV64I-NEXT:    add a0, a3, a0
-; RV64I-NEXT:    ld a3, 0(a0)
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    slli a1, a1, 5
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
-; RV64I-NEXT:    andi a0, a1, 32
-; RV64I-NEXT:    xori t0, a0, 63
-; RV64I-NEXT:    slli a0, a5, 1
-; RV64I-NEXT:    sll a0, a0, t0
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    srl a3, a3, a1
-; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    sll a4, a4, t0
-; RV64I-NEXT:    or a4, a3, a4
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a6, 1
-; RV64I-NEXT:    sll t0, t1, t0
-; RV64I-NEXT:    or t0, a5, t0
-; RV64I-NEXT:    srl a1, a6, a1
+; RV64I-NEXT:    ld a4, 0(a0)
+; RV64I-NEXT:    ld a5, 8(a0)
+; RV64I-NEXT:    slli a6, a1, 5
+; RV64I-NEXT:    ld a7, 16(a0)
+; RV64I-NEXT:    ld t0, 24(a0)
+; RV64I-NEXT:    srl a3, a5, a6
+; RV64I-NEXT:    andi a0, a6, 32
+; RV64I-NEXT:    xori t1, a0, 63
+; RV64I-NEXT:    slli a0, a7, 1
+; RV64I-NEXT:    sll a0, a0, t1
+; RV64I-NEXT:    or a0, a3, a0
+; RV64I-NEXT:    srl t2, a4, a6
+; RV64I-NEXT:    slli a5, a5, 1
+; RV64I-NEXT:    sll a1, a5, t1
+; RV64I-NEXT:    or a1, t2, a1
+; RV64I-NEXT:    srl a5, a7, a6
+; RV64I-NEXT:    slli a4, t0, 1
+; RV64I-NEXT:    sll a4, a4, t1
+; RV64I-NEXT:    or a4, a5, a4
+; RV64I-NEXT:    srl a6, t0, a6
+; RV64I-NEXT:    srli a7, a5, 24
+; RV64I-NEXT:    srli t0, a5, 16
+; RV64I-NEXT:    srli t1, a5, 8
 ; RV64I-NEXT:    sb a5, 16(a2)
-; RV64I-NEXT:    sb a1, 24(a2)
-; RV64I-NEXT:    sb a3, 0(a2)
-; RV64I-NEXT:    sb a7, 8(a2)
-; RV64I-NEXT:    srli a6, a5, 24
-; RV64I-NEXT:    sb a6, 19(a2)
-; RV64I-NEXT:    srli a6, a5, 16
-; RV64I-NEXT:    sb a6, 18(a2)
-; RV64I-NEXT:    srli a5, a5, 8
-; RV64I-NEXT:    sb a5, 17(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    sb t1, 17(a2)
+; RV64I-NEXT:    sb t0, 18(a2)
+; RV64I-NEXT:    sb a7, 19(a2)
+; RV64I-NEXT:    srli a5, a6, 56
+; RV64I-NEXT:    srli a7, a6, 48
+; RV64I-NEXT:    srli t0, a6, 40
+; RV64I-NEXT:    srli t1, a6, 32
+; RV64I-NEXT:    sb t1, 28(a2)
+; RV64I-NEXT:    sb t0, 29(a2)
+; RV64I-NEXT:    sb a7, 30(a2)
 ; RV64I-NEXT:    sb a5, 31(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 30(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 29(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 28(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a6, 24
+; RV64I-NEXT:    srli a7, a6, 16
+; RV64I-NEXT:    srli t0, a6, 8
+; RV64I-NEXT:    sb a6, 24(a2)
+; RV64I-NEXT:    sb t0, 25(a2)
+; RV64I-NEXT:    sb a7, 26(a2)
 ; RV64I-NEXT:    sb a5, 27(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 26(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 1(a2)
-; RV64I-NEXT:    srli a1, a7, 24
-; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a7, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a1, a7, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    srli a1, t0, 56
-; RV64I-NEXT:    sb a1, 23(a2)
-; RV64I-NEXT:    srli a1, t0, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, t0, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, t0, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    srli a5, t2, 24
+; RV64I-NEXT:    srli a6, t2, 16
+; RV64I-NEXT:    srli a7, t2, 8
+; RV64I-NEXT:    sb t2, 0(a2)
+; RV64I-NEXT:    sb a7, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb a5, 3(a2)
+; RV64I-NEXT:    srli a5, a3, 24
+; RV64I-NEXT:    srli a6, a3, 16
+; RV64I-NEXT:    srli a7, a3, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a7, 9(a2)
+; RV64I-NEXT:    sb a6, 10(a2)
+; RV64I-NEXT:    sb a5, 11(a2)
+; RV64I-NEXT:    srli a3, a4, 56
+; RV64I-NEXT:    srli a5, a4, 48
+; RV64I-NEXT:    srli a6, a4, 40
 ; RV64I-NEXT:    srli a4, a4, 32
-; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a4, 20(a2)
+; RV64I-NEXT:    sb a6, 21(a2)
+; RV64I-NEXT:    sb a5, 22(a2)
+; RV64I-NEXT:    sb a3, 23(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    sb a1, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    sb a0, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
+; RV64I-NEXT:    sb a1, 15(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -2694,90 +2696,90 @@ define void @lshr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV32I-NEXT:    or a0, a0, t2
 ; RV32I-NEXT:    or a0, a0, t3
 ; RV32I-NEXT:    lbu a1, 0(a1)
-; RV32I-NEXT:    sw zero, 60(sp)
-; RV32I-NEXT:    sw zero, 56(sp)
-; RV32I-NEXT:    sw zero, 52(sp)
 ; RV32I-NEXT:    sw zero, 48(sp)
-; RV32I-NEXT:    sw zero, 44(sp)
-; RV32I-NEXT:    sw zero, 40(sp)
-; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 52(sp)
+; RV32I-NEXT:    sw zero, 56(sp)
+; RV32I-NEXT:    sw zero, 60(sp)
 ; RV32I-NEXT:    sw zero, 32(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw t1, 24(sp)
-; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 40(sp)
+; RV32I-NEXT:    sw zero, 44(sp)
 ; RV32I-NEXT:    sw a7, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw t1, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    slli a1, a1, 2
 ; RV32I-NEXT:    andi a1, a1, 28
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    add a1, a0, a1
-; RV32I-NEXT:    lw a3, 16(a1)
-; RV32I-NEXT:    lw a4, 20(a1)
-; RV32I-NEXT:    lw a5, 24(a1)
-; RV32I-NEXT:    lw a6, 28(a1)
-; RV32I-NEXT:    lw a7, 0(a1)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    lw t0, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sb a5, 24(a2)
-; RV32I-NEXT:    sb a6, 28(a2)
-; RV32I-NEXT:    sb a3, 16(a2)
-; RV32I-NEXT:    sb a4, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a7, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
-; RV32I-NEXT:    srli t1, a5, 24
+; RV32I-NEXT:    add a4, a0, a1
+; RV32I-NEXT:    lw a5, 16(a4)
+; RV32I-NEXT:    lw a6, 20(a4)
+; RV32I-NEXT:    lw a7, 24(a4)
+; RV32I-NEXT:    lw a1, 0(a4)
+; RV32I-NEXT:    lw a0, 4(a4)
+; RV32I-NEXT:    lw t0, 8(a4)
+; RV32I-NEXT:    lw a3, 12(a4)
+; RV32I-NEXT:    lw a4, 28(a4)
+; RV32I-NEXT:    srli t1, a7, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 8
+; RV32I-NEXT:    sb a7, 24(a2)
+; RV32I-NEXT:    sb t3, 25(a2)
+; RV32I-NEXT:    sb t2, 26(a2)
 ; RV32I-NEXT:    sb t1, 27(a2)
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    sb t1, 26(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 25(a2)
-; RV32I-NEXT:    srli a5, a6, 24
-; RV32I-NEXT:    sb a5, 31(a2)
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 8
+; RV32I-NEXT:    sb a4, 28(a2)
+; RV32I-NEXT:    sb t2, 29(a2)
+; RV32I-NEXT:    sb t1, 30(a2)
+; RV32I-NEXT:    sb a7, 31(a2)
+; RV32I-NEXT:    srli a4, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t1, a5, 8
+; RV32I-NEXT:    sb a5, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb a7, 18(a2)
+; RV32I-NEXT:    sb a4, 19(a2)
+; RV32I-NEXT:    srli a4, a6, 24
 ; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 30(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    sb a5, 19(a2)
+; RV32I-NEXT:    srli a7, a6, 8
+; RV32I-NEXT:    sb a6, 20(a2)
+; RV32I-NEXT:    sb a7, 21(a2)
+; RV32I-NEXT:    sb a5, 22(a2)
+; RV32I-NEXT:    sb a4, 23(a2)
+; RV32I-NEXT:    srli a4, t0, 24
+; RV32I-NEXT:    srli a5, t0, 16
+; RV32I-NEXT:    srli a6, t0, 8
+; RV32I-NEXT:    sb t0, 8(a2)
+; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a4, a3, 24
 ; RV32I-NEXT:    srli a5, a3, 16
-; RV32I-NEXT:    sb a5, 18(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 17(a2)
-; RV32I-NEXT:    srli a3, a4, 24
-; RV32I-NEXT:    sb a3, 23(a2)
-; RV32I-NEXT:    srli a3, a4, 16
-; RV32I-NEXT:    sb a3, 22(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 21(a2)
-; RV32I-NEXT:    srli a3, t0, 24
-; RV32I-NEXT:    sb a3, 11(a2)
-; RV32I-NEXT:    srli a3, t0, 16
-; RV32I-NEXT:    sb a3, 10(a2)
-; RV32I-NEXT:    srli a3, t0, 8
-; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
+; RV32I-NEXT:    sb a4, 15(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 15(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
-; RV32I-NEXT:    srli a1, a7, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a1, a7, 16
-; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a7, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -2881,82 +2883,82 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    lbu a1, 0(a1)
-; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    sd zero, 48(sp)
-; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 32(sp)
-; RV64I-NEXT:    sd a0, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    sd zero, 48(sp)
+; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a0, 24(sp)
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    andi a1, a1, 24
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 16(a0)
-; RV64I-NEXT:    ld a3, 24(a0)
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a0, 8(a0)
-; RV64I-NEXT:    sb a1, 16(a2)
-; RV64I-NEXT:    sb a3, 24(a2)
-; RV64I-NEXT:    sb a4, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    add a3, a0, a1
+; RV64I-NEXT:    ld a4, 16(a3)
+; RV64I-NEXT:    ld a0, 8(a3)
+; RV64I-NEXT:    ld a1, 0(a3)
+; RV64I-NEXT:    ld a3, 24(a3)
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 20(a2)
+; RV64I-NEXT:    sb a7, 21(a2)
+; RV64I-NEXT:    sb a6, 22(a2)
 ; RV64I-NEXT:    sb a5, 23(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 22(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 21(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 20(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a7, a4, 8
+; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    sb a7, 17(a2)
+; RV64I-NEXT:    sb a6, 18(a2)
 ; RV64I-NEXT:    sb a5, 19(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 18(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 17(a2)
-; RV64I-NEXT:    srli a1, a3, 56
-; RV64I-NEXT:    sb a1, 31(a2)
-; RV64I-NEXT:    srli a1, a3, 48
-; RV64I-NEXT:    sb a1, 30(a2)
-; RV64I-NEXT:    srli a1, a3, 40
-; RV64I-NEXT:    sb a1, 29(a2)
-; RV64I-NEXT:    srli a1, a3, 32
-; RV64I-NEXT:    sb a1, 28(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 27(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 26(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 25(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    srli a4, a3, 56
+; RV64I-NEXT:    srli a5, a3, 48
+; RV64I-NEXT:    srli a6, a3, 40
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    sb a7, 28(a2)
+; RV64I-NEXT:    sb a6, 29(a2)
+; RV64I-NEXT:    sb a5, 30(a2)
+; RV64I-NEXT:    sb a4, 31(a2)
+; RV64I-NEXT:    srli a4, a3, 24
+; RV64I-NEXT:    srli a5, a3, 16
+; RV64I-NEXT:    srli a6, a3, 8
+; RV64I-NEXT:    sb a3, 24(a2)
+; RV64I-NEXT:    sb a6, 25(a2)
+; RV64I-NEXT:    sb a5, 26(a2)
+; RV64I-NEXT:    sb a4, 27(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
+; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -3044,90 +3046,90 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
 ; RV32I-NEXT:    or a0, a0, t2
 ; RV32I-NEXT:    or a0, a0, t3
 ; RV32I-NEXT:    lbu a1, 0(a1)
-; RV32I-NEXT:    sw zero, 60(sp)
-; RV32I-NEXT:    sw zero, 56(sp)
-; RV32I-NEXT:    sw zero, 52(sp)
 ; RV32I-NEXT:    sw zero, 48(sp)
-; RV32I-NEXT:    sw zero, 44(sp)
-; RV32I-NEXT:    sw zero, 40(sp)
-; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 52(sp)
+; RV32I-NEXT:    sw zero, 56(sp)
+; RV32I-NEXT:    sw zero, 60(sp)
 ; RV32I-NEXT:    sw zero, 32(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw t1, 24(sp)
-; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 40(sp)
+; RV32I-NEXT:    sw zero, 44(sp)
 ; RV32I-NEXT:    sw a7, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw t1, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    andi a1, a1, 24
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    add a1, a0, a1
-; RV32I-NEXT:    lw a3, 16(a1)
-; RV32I-NEXT:    lw a4, 20(a1)
-; RV32I-NEXT:    lw a5, 24(a1)
-; RV32I-NEXT:    lw a6, 28(a1)
-; RV32I-NEXT:    lw a7, 0(a1)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    lw t0, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sb a5, 24(a2)
-; RV32I-NEXT:    sb a6, 28(a2)
-; RV32I-NEXT:    sb a3, 16(a2)
-; RV32I-NEXT:    sb a4, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a7, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
-; RV32I-NEXT:    srli t1, a5, 24
+; RV32I-NEXT:    add a4, a0, a1
+; RV32I-NEXT:    lw a5, 16(a4)
+; RV32I-NEXT:    lw a6, 20(a4)
+; RV32I-NEXT:    lw a7, 24(a4)
+; RV32I-NEXT:    lw a1, 0(a4)
+; RV32I-NEXT:    lw a0, 4(a4)
+; RV32I-NEXT:    lw t0, 8(a4)
+; RV32I-NEXT:    lw a3, 12(a4)
+; RV32I-NEXT:    lw a4, 28(a4)
+; RV32I-NEXT:    srli t1, a7, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 8
+; RV32I-NEXT:    sb a7, 24(a2)
+; RV32I-NEXT:    sb t3, 25(a2)
+; RV32I-NEXT:    sb t2, 26(a2)
 ; RV32I-NEXT:    sb t1, 27(a2)
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    sb t1, 26(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 25(a2)
-; RV32I-NEXT:    srli a5, a6, 24
-; RV32I-NEXT:    sb a5, 31(a2)
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 8
+; RV32I-NEXT:    sb a4, 28(a2)
+; RV32I-NEXT:    sb t2, 29(a2)
+; RV32I-NEXT:    sb t1, 30(a2)
+; RV32I-NEXT:    sb a7, 31(a2)
+; RV32I-NEXT:    srli a4, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t1, a5, 8
+; RV32I-NEXT:    sb a5, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb a7, 18(a2)
+; RV32I-NEXT:    sb a4, 19(a2)
+; RV32I-NEXT:    srli a4, a6, 24
 ; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 30(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    sb a5, 19(a2)
+; RV32I-NEXT:    srli a7, a6, 8
+; RV32I-NEXT:    sb a6, 20(a2)
+; RV32I-NEXT:    sb a7, 21(a2)
+; RV32I-NEXT:    sb a5, 22(a2)
+; RV32I-NEXT:    sb a4, 23(a2)
+; RV32I-NEXT:    srli a4, t0, 24
+; RV32I-NEXT:    srli a5, t0, 16
+; RV32I-NEXT:    srli a6, t0, 8
+; RV32I-NEXT:    sb t0, 8(a2)
+; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a4, a3, 24
 ; RV32I-NEXT:    srli a5, a3, 16
-; RV32I-NEXT:    sb a5, 18(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 17(a2)
-; RV32I-NEXT:    srli a3, a4, 24
-; RV32I-NEXT:    sb a3, 23(a2)
-; RV32I-NEXT:    srli a3, a4, 16
-; RV32I-NEXT:    sb a3, 22(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 21(a2)
-; RV32I-NEXT:    srli a3, t0, 24
-; RV32I-NEXT:    sb a3, 11(a2)
-; RV32I-NEXT:    srli a3, t0, 16
-; RV32I-NEXT:    sb a3, 10(a2)
-; RV32I-NEXT:    srli a3, t0, 8
-; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
+; RV32I-NEXT:    sb a4, 15(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 15(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
-; RV32I-NEXT:    srli a1, a7, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a1, a7, 16
-; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a7, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -3252,97 +3254,97 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a6
-; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    sd zero, 0(sp)
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a5, 48(sp)
-; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    sd a3, 32(sp)
+; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd a5, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
 ; RV64I-NEXT:    andi a0, a1, 24
 ; RV64I-NEXT:    addi a3, sp, 32
 ; RV64I-NEXT:    sub a3, a3, a0
-; RV64I-NEXT:    ld a4, 0(a3)
-; RV64I-NEXT:    ld a5, 8(a3)
-; RV64I-NEXT:    slli a1, a1, 3
-; RV64I-NEXT:    ld a6, 16(a3)
-; RV64I-NEXT:    ld a3, 24(a3)
-; RV64I-NEXT:    sll a7, a5, a1
-; RV64I-NEXT:    andi a0, a1, 56
-; RV64I-NEXT:    xori t0, a0, 63
-; RV64I-NEXT:    srli a0, a4, 1
-; RV64I-NEXT:    srl a0, a0, t0
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    sll a3, a3, a1
-; RV64I-NEXT:    srli t1, a6, 1
-; RV64I-NEXT:    srl t1, t1, t0
-; RV64I-NEXT:    or t1, a3, t1
-; RV64I-NEXT:    sll a6, a6, a1
-; RV64I-NEXT:    srli a5, a5, 1
-; RV64I-NEXT:    srl a5, a5, t0
-; RV64I-NEXT:    or a5, a6, a5
-; RV64I-NEXT:    sll a1, a4, a1
-; RV64I-NEXT:    sb a1, 0(a2)
-; RV64I-NEXT:    srli a4, a6, 56
-; RV64I-NEXT:    sb a4, 23(a2)
-; RV64I-NEXT:    srli a3, a3, 56
-; RV64I-NEXT:    sb a3, 31(a2)
-; RV64I-NEXT:    srli a3, a1, 56
-; RV64I-NEXT:    sb a3, 7(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 6(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 5(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 4(a2)
+; RV64I-NEXT:    ld a5, 0(a3)
+; RV64I-NEXT:    ld a6, 8(a3)
+; RV64I-NEXT:    slli a7, a1, 3
+; RV64I-NEXT:    ld t0, 16(a3)
+; RV64I-NEXT:    ld a1, 24(a3)
+; RV64I-NEXT:    sll a4, a6, a7
+; RV64I-NEXT:    andi a0, a7, 56
+; RV64I-NEXT:    xori a3, a0, 63
+; RV64I-NEXT:    srli a0, a5, 1
+; RV64I-NEXT:    srl a0, a0, a3
+; RV64I-NEXT:    or a0, a4, a0
+; RV64I-NEXT:    sll t1, a1, a7
+; RV64I-NEXT:    srli a1, t0, 1
+; RV64I-NEXT:    srl a1, a1, a3
+; RV64I-NEXT:    or a1, t1, a1
+; RV64I-NEXT:    sll t0, t0, a7
+; RV64I-NEXT:    srli a6, a6, 1
+; RV64I-NEXT:    srl a3, a6, a3
+; RV64I-NEXT:    or a3, t0, a3
+; RV64I-NEXT:    sll a5, a5, a7
+; RV64I-NEXT:    srli a6, t0, 56
+; RV64I-NEXT:    srli a7, a3, 48
+; RV64I-NEXT:    srli t0, a3, 40
+; RV64I-NEXT:    srli t2, a3, 32
+; RV64I-NEXT:    sb t2, 20(a2)
+; RV64I-NEXT:    sb t0, 21(a2)
+; RV64I-NEXT:    sb a7, 22(a2)
+; RV64I-NEXT:    sb a6, 23(a2)
+; RV64I-NEXT:    srli a6, t1, 56
+; RV64I-NEXT:    srli a7, a1, 48
+; RV64I-NEXT:    srli t0, a1, 40
+; RV64I-NEXT:    srli t1, a1, 32
+; RV64I-NEXT:    sb t1, 28(a2)
+; RV64I-NEXT:    sb t0, 29(a2)
+; RV64I-NEXT:    sb a7, 30(a2)
+; RV64I-NEXT:    sb a6, 31(a2)
+; RV64I-NEXT:    srli a6, a5, 56
+; RV64I-NEXT:    srli a7, a5, 48
+; RV64I-NEXT:    srli t0, a5, 40
+; RV64I-NEXT:    srli t1, a5, 32
+; RV64I-NEXT:    sb t1, 4(a2)
+; RV64I-NEXT:    sb t0, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
+; RV64I-NEXT:    sb a6, 7(a2)
+; RV64I-NEXT:    srli a6, a5, 24
+; RV64I-NEXT:    srli a7, a5, 16
+; RV64I-NEXT:    srli t0, a5, 8
+; RV64I-NEXT:    sb a5, 0(a2)
+; RV64I-NEXT:    sb t0, 1(a2)
+; RV64I-NEXT:    sb a7, 2(a2)
+; RV64I-NEXT:    sb a6, 3(a2)
+; RV64I-NEXT:    srli a4, a4, 56
+; RV64I-NEXT:    srli a5, a0, 48
+; RV64I-NEXT:    srli a6, a0, 40
+; RV64I-NEXT:    srli a7, a0, 32
+; RV64I-NEXT:    sb a7, 12(a2)
+; RV64I-NEXT:    sb a6, 13(a2)
+; RV64I-NEXT:    sb a5, 14(a2)
+; RV64I-NEXT:    sb a4, 15(a2)
+; RV64I-NEXT:    srli a4, a3, 24
+; RV64I-NEXT:    srli a5, a3, 16
+; RV64I-NEXT:    srli a6, a3, 8
+; RV64I-NEXT:    sb a3, 16(a2)
+; RV64I-NEXT:    sb a6, 17(a2)
+; RV64I-NEXT:    sb a5, 18(a2)
+; RV64I-NEXT:    sb a4, 19(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
-; RV64I-NEXT:    sb a3, 3(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 2(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 1(a2)
-; RV64I-NEXT:    srli a1, a7, 56
-; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    sb a5, 16(a2)
-; RV64I-NEXT:    sb t1, 24(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a1, a5, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, a5, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, a5, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, a5, 24
-; RV64I-NEXT:    sb a1, 19(a2)
-; RV64I-NEXT:    srli a1, a5, 16
-; RV64I-NEXT:    sb a1, 18(a2)
-; RV64I-NEXT:    srli a5, a5, 8
-; RV64I-NEXT:    sb a5, 17(a2)
-; RV64I-NEXT:    srli a1, t1, 48
-; RV64I-NEXT:    sb a1, 30(a2)
-; RV64I-NEXT:    srli a1, t1, 40
-; RV64I-NEXT:    sb a1, 29(a2)
-; RV64I-NEXT:    srli a1, t1, 32
-; RV64I-NEXT:    sb a1, 28(a2)
-; RV64I-NEXT:    srli a1, t1, 24
-; RV64I-NEXT:    sb a1, 27(a2)
-; RV64I-NEXT:    srli a1, t1, 16
-; RV64I-NEXT:    sb a1, 26(a2)
-; RV64I-NEXT:    srli a1, t1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 24(a2)
+; RV64I-NEXT:    sb a5, 25(a2)
+; RV64I-NEXT:    sb a4, 26(a2)
+; RV64I-NEXT:    sb a3, 27(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -3352,6 +3354,7 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw s0, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 72(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s2, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 64(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lbu a3, 1(a0)
 ; RV32I-NEXT:    lbu a4, 0(a0)
 ; RV32I-NEXT:    lbu a5, 2(a0)
@@ -3442,124 +3445,125 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    or a1, a1, t2
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw t1, 56(sp)
-; RV32I-NEXT:    sw t0, 52(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw a7, 48(sp)
-; RV32I-NEXT:    sw a6, 44(sp)
-; RV32I-NEXT:    sw a5, 40(sp)
-; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw t0, 52(sp)
+; RV32I-NEXT:    sw t1, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a3, 32(sp)
+; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw a5, 40(sp)
+; RV32I-NEXT:    sw a6, 44(sp)
 ; RV32I-NEXT:    andi a0, a1, 28
 ; RV32I-NEXT:    addi a3, sp, 32
-; RV32I-NEXT:    sub a5, a3, a0
-; RV32I-NEXT:    lw a6, 0(a5)
-; RV32I-NEXT:    lw a3, 4(a5)
-; RV32I-NEXT:    slli a7, a1, 3
-; RV32I-NEXT:    lw t0, 8(a5)
-; RV32I-NEXT:    lw t1, 12(a5)
-; RV32I-NEXT:    sll a4, a3, a7
-; RV32I-NEXT:    andi a0, a7, 24
-; RV32I-NEXT:    xori t2, a0, 31
-; RV32I-NEXT:    srli a0, a6, 1
-; RV32I-NEXT:    srl a0, a0, t2
-; RV32I-NEXT:    or a0, a4, a0
-; RV32I-NEXT:    sll t3, t1, a7
-; RV32I-NEXT:    srli a1, t0, 1
-; RV32I-NEXT:    srl a1, a1, t2
-; RV32I-NEXT:    or a1, t3, a1
-; RV32I-NEXT:    sll t0, t0, a7
-; RV32I-NEXT:    srli a3, a3, 1
-; RV32I-NEXT:    srl a3, a3, t2
-; RV32I-NEXT:    lw t4, 16(a5)
-; RV32I-NEXT:    lw t5, 20(a5)
-; RV32I-NEXT:    or a3, t0, a3
-; RV32I-NEXT:    lw t6, 24(a5)
-; RV32I-NEXT:    lw a5, 28(a5)
-; RV32I-NEXT:    sll s0, t5, a7
-; RV32I-NEXT:    srli s1, t4, 1
-; RV32I-NEXT:    srl s1, s1, t2
-; RV32I-NEXT:    or s1, s0, s1
-; RV32I-NEXT:    sll t4, t4, a7
-; RV32I-NEXT:    srli t1, t1, 1
-; RV32I-NEXT:    srl t1, t1, t2
-; RV32I-NEXT:    or t1, t4, t1
-; RV32I-NEXT:    sll a5, a5, a7
-; RV32I-NEXT:    srli s2, t6, 1
-; RV32I-NEXT:    srl s2, s2, t2
-; RV32I-NEXT:    or s2, a5, s2
-; RV32I-NEXT:    sll t6, t6, a7
-; RV32I-NEXT:    srli t5, t5, 1
-; RV32I-NEXT:    srl t2, t5, t2
+; RV32I-NEXT:    sub a7, a3, a0
+; RV32I-NEXT:    lw t3, 0(a7)
+; RV32I-NEXT:    lw a6, 4(a7)
+; RV32I-NEXT:    slli t4, a1, 3
+; RV32I-NEXT:    lw a5, 8(a7)
+; RV32I-NEXT:    lw t2, 12(a7)
+; RV32I-NEXT:    sll a0, a6, t4
+; RV32I-NEXT:    andi a1, t4, 24
+; RV32I-NEXT:    xori t5, a1, 31
+; RV32I-NEXT:    srli a1, t3, 1
+; RV32I-NEXT:    srl a1, a1, t5
+; RV32I-NEXT:    or a1, a0, a1
+; RV32I-NEXT:    sll a4, t2, t4
+; RV32I-NEXT:    srli a3, a5, 1
+; RV32I-NEXT:    srl a3, a3, t5
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    sll a5, a5, t4
+; RV32I-NEXT:    srli a6, a6, 1
+; RV32I-NEXT:    srl a6, a6, t5
+; RV32I-NEXT:    lw t6, 16(a7)
+; RV32I-NEXT:    lw s0, 20(a7)
+; RV32I-NEXT:    or a6, a5, a6
+; RV32I-NEXT:    lw s1, 24(a7)
+; RV32I-NEXT:    lw a7, 28(a7)
+; RV32I-NEXT:    sll t1, s0, t4
+; RV32I-NEXT:    srli t0, t6, 1
+; RV32I-NEXT:    srl t0, t0, t5
+; RV32I-NEXT:    or t0, t1, t0
+; RV32I-NEXT:    sll t6, t6, t4
+; RV32I-NEXT:    srli t2, t2, 1
+; RV32I-NEXT:    srl t2, t2, t5
 ; RV32I-NEXT:    or t2, t6, t2
-; RV32I-NEXT:    sll a6, a6, a7
-; RV32I-NEXT:    sb a6, 0(a2)
-; RV32I-NEXT:    srli a7, t6, 24
-; RV32I-NEXT:    sb a7, 27(a2)
+; RV32I-NEXT:    sll s2, a7, t4
+; RV32I-NEXT:    srli a7, s1, 1
+; RV32I-NEXT:    srl a7, a7, t5
+; RV32I-NEXT:    or s3, s2, a7
+; RV32I-NEXT:    sll s1, s1, t4
+; RV32I-NEXT:    srli s0, s0, 1
+; RV32I-NEXT:    srl a7, s0, t5
+; RV32I-NEXT:    or t5, s1, a7
+; RV32I-NEXT:    sll a7, t3, t4
+; RV32I-NEXT:    srli s1, s1, 24
+; RV32I-NEXT:    srli t3, t5, 16
+; RV32I-NEXT:    srli t4, t5, 8
+; RV32I-NEXT:    sb t5, 24(a2)
+; RV32I-NEXT:    sb t4, 25(a2)
+; RV32I-NEXT:    sb t3, 26(a2)
+; RV32I-NEXT:    sb s1, 27(a2)
+; RV32I-NEXT:    srli t3, s2, 24
+; RV32I-NEXT:    srli t4, s3, 16
+; RV32I-NEXT:    srli t5, s3, 8
+; RV32I-NEXT:    sb s3, 28(a2)
+; RV32I-NEXT:    sb t5, 29(a2)
+; RV32I-NEXT:    sb t4, 30(a2)
+; RV32I-NEXT:    sb t3, 31(a2)
+; RV32I-NEXT:    srli t3, t6, 24
+; RV32I-NEXT:    srli t4, t2, 16
+; RV32I-NEXT:    srli t5, t2, 8
+; RV32I-NEXT:    sb t2, 16(a2)
+; RV32I-NEXT:    sb t5, 17(a2)
+; RV32I-NEXT:    sb t4, 18(a2)
+; RV32I-NEXT:    sb t3, 19(a2)
+; RV32I-NEXT:    srli t1, t1, 24
+; RV32I-NEXT:    srli t2, t0, 16
+; RV32I-NEXT:    srli t3, t0, 8
+; RV32I-NEXT:    sb t0, 20(a2)
+; RV32I-NEXT:    sb t3, 21(a2)
+; RV32I-NEXT:    sb t2, 22(a2)
+; RV32I-NEXT:    sb t1, 23(a2)
 ; RV32I-NEXT:    srli a5, a5, 24
-; RV32I-NEXT:    sb a5, 31(a2)
-; RV32I-NEXT:    srli a5, t4, 24
-; RV32I-NEXT:    sb a5, 19(a2)
-; RV32I-NEXT:    srli s0, s0, 24
-; RV32I-NEXT:    sb s0, 23(a2)
-; RV32I-NEXT:    srli a5, t0, 24
+; RV32I-NEXT:    srli t0, a6, 16
+; RV32I-NEXT:    srli t1, a6, 8
+; RV32I-NEXT:    sb a6, 8(a2)
+; RV32I-NEXT:    sb t1, 9(a2)
+; RV32I-NEXT:    sb t0, 10(a2)
 ; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a5, t3, 24
-; RV32I-NEXT:    sb a5, 15(a2)
-; RV32I-NEXT:    srli a5, a6, 24
-; RV32I-NEXT:    sb a5, 3(a2)
-; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 2(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 1(a2)
 ; RV32I-NEXT:    srli a4, a4, 24
-; RV32I-NEXT:    sb a4, 7(a2)
-; RV32I-NEXT:    sb t2, 24(a2)
-; RV32I-NEXT:    sb s2, 28(a2)
-; RV32I-NEXT:    sb t1, 16(a2)
-; RV32I-NEXT:    sb s1, 20(a2)
-; RV32I-NEXT:    sb a3, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
-; RV32I-NEXT:    srli a4, t2, 16
-; RV32I-NEXT:    sb a4, 26(a2)
-; RV32I-NEXT:    srli a4, t2, 8
-; RV32I-NEXT:    sb a4, 25(a2)
-; RV32I-NEXT:    srli a4, s2, 16
-; RV32I-NEXT:    sb a4, 30(a2)
-; RV32I-NEXT:    srli a4, s2, 8
-; RV32I-NEXT:    sb a4, 29(a2)
-; RV32I-NEXT:    srli a4, t1, 16
-; RV32I-NEXT:    sb a4, 18(a2)
-; RV32I-NEXT:    srli a4, t1, 8
-; RV32I-NEXT:    sb a4, 17(a2)
-; RV32I-NEXT:    srli a4, s1, 16
-; RV32I-NEXT:    sb a4, 22(a2)
-; RV32I-NEXT:    srli s1, s1, 8
-; RV32I-NEXT:    sb s1, 21(a2)
-; RV32I-NEXT:    srli a4, a3, 16
-; RV32I-NEXT:    sb a4, 10(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    srli a5, a3, 16
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
+; RV32I-NEXT:    sb a4, 15(a2)
+; RV32I-NEXT:    srli a3, a7, 24
+; RV32I-NEXT:    srli a4, a7, 16
+; RV32I-NEXT:    srli a5, a7, 8
+; RV32I-NEXT:    sb a7, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
+; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    srli a4, a1, 8
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
+; RV32I-NEXT:    sb a0, 7(a2)
 ; RV32I-NEXT:    lw s0, 76(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 72(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s2, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 64(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 80
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -3684,27 +3688,27 @@ define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a6
-; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    sd zero, 0(sp)
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a5, 48(sp)
-; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    sd a3, 32(sp)
+; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd a5, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
 ; RV64I-NEXT:    slli a0, a1, 2
 ; RV64I-NEXT:    andi a0, a0, 24
 ; RV64I-NEXT:    addi a3, sp, 32
 ; RV64I-NEXT:    sub a3, a3, a0
-; RV64I-NEXT:    ld a4, 0(a3)
-; RV64I-NEXT:    ld a5, 8(a3)
+; RV64I-NEXT:    ld a5, 0(a3)
+; RV64I-NEXT:    ld a4, 8(a3)
 ; RV64I-NEXT:    slli a6, a1, 5
 ; RV64I-NEXT:    ld a7, 16(a3)
 ; RV64I-NEXT:    ld a1, 24(a3)
-; RV64I-NEXT:    sll a3, a5, a6
+; RV64I-NEXT:    sll a3, a4, a6
 ; RV64I-NEXT:    andi a0, a6, 32
 ; RV64I-NEXT:    xori t0, a0, 63
-; RV64I-NEXT:    srli a0, a4, 1
+; RV64I-NEXT:    srli a0, a5, 1
 ; RV64I-NEXT:    srl a0, a0, t0
 ; RV64I-NEXT:    or a0, a3, a0
 ; RV64I-NEXT:    sll t1, a1, a6
@@ -3712,70 +3716,70 @@ define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV64I-NEXT:    srl a1, a1, t0
 ; RV64I-NEXT:    or a1, t1, a1
 ; RV64I-NEXT:    sll a7, a7, a6
-; RV64I-NEXT:    srli a5, a5, 1
-; RV64I-NEXT:    srl a5, a5, t0
-; RV64I-NEXT:    or a5, a7, a5
-; RV64I-NEXT:    sll a4, a4, a6
-; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    srli a4, a4, 1
+; RV64I-NEXT:    srl a4, a4, t0
+; RV64I-NEXT:    or a4, a7, a4
+; RV64I-NEXT:    sll a5, a5, a6
 ; RV64I-NEXT:    srli a6, a7, 56
+; RV64I-NEXT:    srli t0, a7, 48
+; RV64I-NEXT:    srli t2, a7, 40
+; RV64I-NEXT:    srli a7, a7, 32
+; RV64I-NEXT:    sb a7, 20(a2)
+; RV64I-NEXT:    sb t2, 21(a2)
+; RV64I-NEXT:    sb t0, 22(a2)
 ; RV64I-NEXT:    sb a6, 23(a2)
-; RV64I-NEXT:    srli a6, a7, 48
-; RV64I-NEXT:    sb a6, 22(a2)
-; RV64I-NEXT:    srli a6, a7, 40
-; RV64I-NEXT:    sb a6, 21(a2)
-; RV64I-NEXT:    srli a6, a7, 32
-; RV64I-NEXT:    sb a6, 20(a2)
 ; RV64I-NEXT:    srli a6, t1, 56
+; RV64I-NEXT:    srli a7, t1, 48
+; RV64I-NEXT:    srli t0, t1, 40
+; RV64I-NEXT:    srli t1, t1, 32
+; RV64I-NEXT:    sb t1, 28(a2)
+; RV64I-NEXT:    sb t0, 29(a2)
+; RV64I-NEXT:    sb a7, 30(a2)
 ; RV64I-NEXT:    sb a6, 31(a2)
-; RV64I-NEXT:    srli a6, t1, 48
-; RV64I-NEXT:    sb a6, 30(a2)
-; RV64I-NEXT:    srli a6, t1, 40
-; RV64I-NEXT:    sb a6, 29(a2)
-; RV64I-NEXT:    srli a6, t1, 32
-; RV64I-NEXT:    sb a6, 28(a2)
-; RV64I-NEXT:    srli a6, a4, 56
+; RV64I-NEXT:    srli a6, a5, 56
+; RV64I-NEXT:    srli a7, a5, 48
+; RV64I-NEXT:    srli t0, a5, 40
+; RV64I-NEXT:    srli t1, a5, 32
+; RV64I-NEXT:    sb t1, 4(a2)
+; RV64I-NEXT:    sb t0, 5(a2)
+; RV64I-NEXT:    sb a7, 6(a2)
 ; RV64I-NEXT:    sb a6, 7(a2)
-; RV64I-NEXT:    srli a6, a4, 48
-; RV64I-NEXT:    sb a6, 6(a2)
-; RV64I-NEXT:    srli a6, a4, 40
-; RV64I-NEXT:    sb a6, 5(a2)
-; RV64I-NEXT:    srli a6, a4, 32
-; RV64I-NEXT:    sb a6, 4(a2)
-; RV64I-NEXT:    srli a6, a4, 24
+; RV64I-NEXT:    srli a6, a5, 24
+; RV64I-NEXT:    srli a7, a5, 16
+; RV64I-NEXT:    srli t0, a5, 8
+; RV64I-NEXT:    sb a5, 0(a2)
+; RV64I-NEXT:    sb t0, 1(a2)
+; RV64I-NEXT:    sb a7, 2(a2)
 ; RV64I-NEXT:    sb a6, 3(a2)
-; RV64I-NEXT:    srli a6, a4, 16
-; RV64I-NEXT:    sb a6, 2(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 1(a2)
-; RV64I-NEXT:    srli a4, a3, 56
-; RV64I-NEXT:    sb a4, 15(a2)
-; RV64I-NEXT:    srli a4, a3, 48
-; RV64I-NEXT:    sb a4, 14(a2)
-; RV64I-NEXT:    srli a4, a3, 40
-; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    srli a5, a3, 56
+; RV64I-NEXT:    srli a6, a3, 48
+; RV64I-NEXT:    srli a7, a3, 40
 ; RV64I-NEXT:    srli a3, a3, 32
 ; RV64I-NEXT:    sb a3, 12(a2)
-; RV64I-NEXT:    sb a5, 16(a2)
-; RV64I-NEXT:    sb a1, 24(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a3, a5, 24
+; RV64I-NEXT:    sb a7, 13(a2)
+; RV64I-NEXT:    sb a6, 14(a2)
+; RV64I-NEXT:    sb a5, 15(a2)
+; RV64I-NEXT:    srli a3, a4, 24
+; RV64I-NEXT:    srli a5, a4, 16
+; RV64I-NEXT:    srli a6, a4, 8
+; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    sb a6, 17(a2)
+; RV64I-NEXT:    sb a5, 18(a2)
 ; RV64I-NEXT:    sb a3, 19(a2)
-; RV64I-NEXT:    srli a3, a5, 16
-; RV64I-NEXT:    sb a3, 18(a2)
-; RV64I-NEXT:    srli a5, a5, 8
-; RV64I-NEXT:    sb a5, 17(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 24(a2)
+; RV64I-NEXT:    sb a5, 25(a2)
+; RV64I-NEXT:    sb a4, 26(a2)
 ; RV64I-NEXT:    sb a3, 27(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 26(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -3863,90 +3867,90 @@ define void @shl_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) nounw
 ; RV32I-NEXT:    or a0, a0, t2
 ; RV32I-NEXT:    or a0, a0, t3
 ; RV32I-NEXT:    lbu a1, 0(a1)
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw t1, 56(sp)
-; RV32I-NEXT:    sw t0, 52(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw a7, 48(sp)
-; RV32I-NEXT:    sw a6, 44(sp)
-; RV32I-NEXT:    sw a5, 40(sp)
-; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw t0, 52(sp)
+; RV32I-NEXT:    sw t1, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a3, 32(sp)
+; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw a5, 40(sp)
+; RV32I-NEXT:    sw a6, 44(sp)
 ; RV32I-NEXT:    slli a1, a1, 2
 ; RV32I-NEXT:    andi a1, a1, 28
 ; RV32I-NEXT:    addi a0, sp, 32
-; RV32I-NEXT:    sub a1, a0, a1
-; RV32I-NEXT:    lw a3, 16(a1)
-; RV32I-NEXT:    lw a4, 20(a1)
-; RV32I-NEXT:    lw a5, 24(a1)
-; RV32I-NEXT:    lw a6, 28(a1)
-; RV32I-NEXT:    lw a7, 0(a1)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    lw t0, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sb a5, 24(a2)
-; RV32I-NEXT:    sb a6, 28(a2)
-; RV32I-NEXT:    sb a3, 16(a2)
-; RV32I-NEXT:    sb a4, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a7, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
-; RV32I-NEXT:    srli t1, a5, 24
+; RV32I-NEXT:    sub a4, a0, a1
+; RV32I-NEXT:    lw a5, 16(a4)
+; RV32I-NEXT:    lw a6, 20(a4)
+; RV32I-NEXT:    lw a7, 24(a4)
+; RV32I-NEXT:    lw a1, 0(a4)
+; RV32I-NEXT:    lw a0, 4(a4)
+; RV32I-NEXT:    lw t0, 8(a4)
+; RV32I-NEXT:    lw a3, 12(a4)
+; RV32I-NEXT:    lw a4, 28(a4)
+; RV32I-NEXT:    srli t1, a7, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 8
+; RV32I-NEXT:    sb a7, 24(a2)
+; RV32I-NEXT:    sb t3, 25(a2)
+; RV32I-NEXT:    sb t2, 26(a2)
 ; RV32I-NEXT:    sb t1, 27(a2)
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    sb t1, 26(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 25(a2)
-; RV32I-NEXT:    srli a5, a6, 24
-; RV32I-NEXT:    sb a5, 31(a2)
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 8
+; RV32I-NEXT:    sb a4, 28(a2)
+; RV32I-NEXT:    sb t2, 29(a2)
+; RV32I-NEXT:    sb t1, 30(a2)
+; RV32I-NEXT:    sb a7, 31(a2)
+; RV32I-NEXT:    srli a4, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t1, a5, 8
+; RV32I-NEXT:    sb a5, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb a7, 18(a2)
+; RV32I-NEXT:    sb a4, 19(a2)
+; RV32I-NEXT:    srli a4, a6, 24
 ; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 30(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    sb a5, 19(a2)
+; RV32I-NEXT:    srli a7, a6, 8
+; RV32I-NEXT:    sb a6, 20(a2)
+; RV32I-NEXT:    sb a7, 21(a2)
+; RV32I-NEXT:    sb a5, 22(a2)
+; RV32I-NEXT:    sb a4, 23(a2)
+; RV32I-NEXT:    srli a4, t0, 24
+; RV32I-NEXT:    srli a5, t0, 16
+; RV32I-NEXT:    srli a6, t0, 8
+; RV32I-NEXT:    sb t0, 8(a2)
+; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a4, a3, 24
 ; RV32I-NEXT:    srli a5, a3, 16
-; RV32I-NEXT:    sb a5, 18(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 17(a2)
-; RV32I-NEXT:    srli a3, a4, 24
-; RV32I-NEXT:    sb a3, 23(a2)
-; RV32I-NEXT:    srli a3, a4, 16
-; RV32I-NEXT:    sb a3, 22(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 21(a2)
-; RV32I-NEXT:    srli a3, t0, 24
-; RV32I-NEXT:    sb a3, 11(a2)
-; RV32I-NEXT:    srli a3, t0, 16
-; RV32I-NEXT:    sb a3, 10(a2)
-; RV32I-NEXT:    srli a3, t0, 8
-; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
+; RV32I-NEXT:    sb a4, 15(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 15(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
-; RV32I-NEXT:    srli a1, a7, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a1, a7, 16
-; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a7, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -4050,82 +4054,82 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    or a0, a0, a6
 ; RV64I-NEXT:    lbu a1, 0(a1)
-; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    sd zero, 0(sp)
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a5, 48(sp)
-; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    sd a3, 32(sp)
+; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd a5, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    andi a1, a1, 24
 ; RV64I-NEXT:    addi a0, sp, 32
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    ld a1, 16(a0)
-; RV64I-NEXT:    ld a3, 24(a0)
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a0, 8(a0)
-; RV64I-NEXT:    sb a1, 16(a2)
-; RV64I-NEXT:    sb a3, 24(a2)
-; RV64I-NEXT:    sb a4, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    sub a3, a0, a1
+; RV64I-NEXT:    ld a4, 16(a3)
+; RV64I-NEXT:    ld a0, 8(a3)
+; RV64I-NEXT:    ld a1, 0(a3)
+; RV64I-NEXT:    ld a3, 24(a3)
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 20(a2)
+; RV64I-NEXT:    sb a7, 21(a2)
+; RV64I-NEXT:    sb a6, 22(a2)
 ; RV64I-NEXT:    sb a5, 23(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 22(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 21(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 20(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a7, a4, 8
+; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    sb a7, 17(a2)
+; RV64I-NEXT:    sb a6, 18(a2)
 ; RV64I-NEXT:    sb a5, 19(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 18(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 17(a2)
-; RV64I-NEXT:    srli a1, a3, 56
-; RV64I-NEXT:    sb a1, 31(a2)
-; RV64I-NEXT:    srli a1, a3, 48
-; RV64I-NEXT:    sb a1, 30(a2)
-; RV64I-NEXT:    srli a1, a3, 40
-; RV64I-NEXT:    sb a1, 29(a2)
-; RV64I-NEXT:    srli a1, a3, 32
-; RV64I-NEXT:    sb a1, 28(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 27(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 26(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 25(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    srli a4, a3, 56
+; RV64I-NEXT:    srli a5, a3, 48
+; RV64I-NEXT:    srli a6, a3, 40
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    sb a7, 28(a2)
+; RV64I-NEXT:    sb a6, 29(a2)
+; RV64I-NEXT:    sb a5, 30(a2)
+; RV64I-NEXT:    sb a4, 31(a2)
+; RV64I-NEXT:    srli a4, a3, 24
+; RV64I-NEXT:    srli a5, a3, 16
+; RV64I-NEXT:    srli a6, a3, 8
+; RV64I-NEXT:    sb a3, 24(a2)
+; RV64I-NEXT:    sb a6, 25(a2)
+; RV64I-NEXT:    sb a5, 26(a2)
+; RV64I-NEXT:    sb a4, 27(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
+; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -4213,90 +4217,90 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
 ; RV32I-NEXT:    or a0, a0, t2
 ; RV32I-NEXT:    or a0, a0, t3
 ; RV32I-NEXT:    lbu a1, 0(a1)
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw t1, 56(sp)
-; RV32I-NEXT:    sw t0, 52(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw a7, 48(sp)
-; RV32I-NEXT:    sw a6, 44(sp)
-; RV32I-NEXT:    sw a5, 40(sp)
-; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw t0, 52(sp)
+; RV32I-NEXT:    sw t1, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a3, 32(sp)
+; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw a5, 40(sp)
+; RV32I-NEXT:    sw a6, 44(sp)
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    andi a1, a1, 24
 ; RV32I-NEXT:    addi a0, sp, 32
-; RV32I-NEXT:    sub a1, a0, a1
-; RV32I-NEXT:    lw a3, 16(a1)
-; RV32I-NEXT:    lw a4, 20(a1)
-; RV32I-NEXT:    lw a5, 24(a1)
-; RV32I-NEXT:    lw a6, 28(a1)
-; RV32I-NEXT:    lw a7, 0(a1)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    lw t0, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sb a5, 24(a2)
-; RV32I-NEXT:    sb a6, 28(a2)
-; RV32I-NEXT:    sb a3, 16(a2)
-; RV32I-NEXT:    sb a4, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a7, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
-; RV32I-NEXT:    srli t1, a5, 24
+; RV32I-NEXT:    sub a4, a0, a1
+; RV32I-NEXT:    lw a5, 16(a4)
+; RV32I-NEXT:    lw a6, 20(a4)
+; RV32I-NEXT:    lw a7, 24(a4)
+; RV32I-NEXT:    lw a1, 0(a4)
+; RV32I-NEXT:    lw a0, 4(a4)
+; RV32I-NEXT:    lw t0, 8(a4)
+; RV32I-NEXT:    lw a3, 12(a4)
+; RV32I-NEXT:    lw a4, 28(a4)
+; RV32I-NEXT:    srli t1, a7, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 8
+; RV32I-NEXT:    sb a7, 24(a2)
+; RV32I-NEXT:    sb t3, 25(a2)
+; RV32I-NEXT:    sb t2, 26(a2)
 ; RV32I-NEXT:    sb t1, 27(a2)
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    sb t1, 26(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 25(a2)
-; RV32I-NEXT:    srli a5, a6, 24
-; RV32I-NEXT:    sb a5, 31(a2)
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 8
+; RV32I-NEXT:    sb a4, 28(a2)
+; RV32I-NEXT:    sb t2, 29(a2)
+; RV32I-NEXT:    sb t1, 30(a2)
+; RV32I-NEXT:    sb a7, 31(a2)
+; RV32I-NEXT:    srli a4, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t1, a5, 8
+; RV32I-NEXT:    sb a5, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb a7, 18(a2)
+; RV32I-NEXT:    sb a4, 19(a2)
+; RV32I-NEXT:    srli a4, a6, 24
 ; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 30(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    sb a5, 19(a2)
+; RV32I-NEXT:    srli a7, a6, 8
+; RV32I-NEXT:    sb a6, 20(a2)
+; RV32I-NEXT:    sb a7, 21(a2)
+; RV32I-NEXT:    sb a5, 22(a2)
+; RV32I-NEXT:    sb a4, 23(a2)
+; RV32I-NEXT:    srli a4, t0, 24
+; RV32I-NEXT:    srli a5, t0, 16
+; RV32I-NEXT:    srli a6, t0, 8
+; RV32I-NEXT:    sb t0, 8(a2)
+; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a4, a3, 24
 ; RV32I-NEXT:    srli a5, a3, 16
-; RV32I-NEXT:    sb a5, 18(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 17(a2)
-; RV32I-NEXT:    srli a3, a4, 24
-; RV32I-NEXT:    sb a3, 23(a2)
-; RV32I-NEXT:    srli a3, a4, 16
-; RV32I-NEXT:    sb a3, 22(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 21(a2)
-; RV32I-NEXT:    srli a3, t0, 24
-; RV32I-NEXT:    sb a3, 11(a2)
-; RV32I-NEXT:    srli a3, t0, 16
-; RV32I-NEXT:    sb a3, 10(a2)
-; RV32I-NEXT:    srli a3, t0, 8
-; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
+; RV32I-NEXT:    sb a4, 15(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 15(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
-; RV32I-NEXT:    srli a1, a7, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a1, a7, 16
-; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a7, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -4422,97 +4426,97 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a7
 ; RV64I-NEXT:    sraiw a0, a0, 31
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    sd a0, 32(sp)
-; RV64I-NEXT:    sd a6, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a6, 24(sp)
 ; RV64I-NEXT:    andi a0, a1, 24
 ; RV64I-NEXT:    mv a3, sp
 ; RV64I-NEXT:    add a0, a3, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    slli a1, a1, 3
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
-; RV64I-NEXT:    andi a0, a1, 56
-; RV64I-NEXT:    xori t0, a0, 63
-; RV64I-NEXT:    slli a0, a5, 1
-; RV64I-NEXT:    sll a0, a0, t0
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    srl a3, a3, a1
+; RV64I-NEXT:    slli a5, a1, 3
+; RV64I-NEXT:    ld a6, 16(a0)
+; RV64I-NEXT:    ld a7, 24(a0)
+; RV64I-NEXT:    srl a0, a4, a5
+; RV64I-NEXT:    andi a1, a5, 56
+; RV64I-NEXT:    xori t0, a1, 63
+; RV64I-NEXT:    slli a1, a6, 1
+; RV64I-NEXT:    sll a1, a1, t0
+; RV64I-NEXT:    or a1, a0, a1
+; RV64I-NEXT:    srl a3, a3, a5
 ; RV64I-NEXT:    slli a4, a4, 1
 ; RV64I-NEXT:    sll a4, a4, t0
 ; RV64I-NEXT:    or a4, a3, a4
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a6, 1
+; RV64I-NEXT:    srl a6, a6, a5
+; RV64I-NEXT:    slli t1, a7, 1
 ; RV64I-NEXT:    sll t0, t1, t0
-; RV64I-NEXT:    or t0, a5, t0
-; RV64I-NEXT:    sra a1, a6, a1
-; RV64I-NEXT:    sb a5, 16(a2)
-; RV64I-NEXT:    sb a1, 24(a2)
+; RV64I-NEXT:    or t0, a6, t0
+; RV64I-NEXT:    sra a5, a7, a5
+; RV64I-NEXT:    srli a7, a5, 56
+; RV64I-NEXT:    srli t1, a5, 48
+; RV64I-NEXT:    srli t2, a5, 40
+; RV64I-NEXT:    srli t3, a5, 32
+; RV64I-NEXT:    sb t3, 28(a2)
+; RV64I-NEXT:    sb t2, 29(a2)
+; RV64I-NEXT:    sb t1, 30(a2)
+; RV64I-NEXT:    sb a7, 31(a2)
+; RV64I-NEXT:    srli a7, a5, 24
+; RV64I-NEXT:    srli t1, a5, 16
+; RV64I-NEXT:    srli t2, a5, 8
+; RV64I-NEXT:    sb a5, 24(a2)
+; RV64I-NEXT:    sb t2, 25(a2)
+; RV64I-NEXT:    sb t1, 26(a2)
+; RV64I-NEXT:    sb a7, 27(a2)
+; RV64I-NEXT:    srli a5, t0, 56
+; RV64I-NEXT:    srli a7, t0, 48
+; RV64I-NEXT:    srli t1, t0, 40
+; RV64I-NEXT:    srli t2, t0, 32
+; RV64I-NEXT:    sb t2, 20(a2)
+; RV64I-NEXT:    sb t1, 21(a2)
+; RV64I-NEXT:    sb a7, 22(a2)
+; RV64I-NEXT:    sb a5, 23(a2)
+; RV64I-NEXT:    srli a5, t0, 24
+; RV64I-NEXT:    srli a7, t0, 16
+; RV64I-NEXT:    srli t0, t0, 8
+; RV64I-NEXT:    sb a6, 16(a2)
+; RV64I-NEXT:    sb t0, 17(a2)
+; RV64I-NEXT:    sb a7, 18(a2)
+; RV64I-NEXT:    sb a5, 19(a2)
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 4(a2)
+; RV64I-NEXT:    sb a7, 5(a2)
+; RV64I-NEXT:    sb a6, 6(a2)
+; RV64I-NEXT:    sb a5, 7(a2)
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a4, a4, 8
 ; RV64I-NEXT:    sb a3, 0(a2)
-; RV64I-NEXT:    sb a7, 8(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb a5, 3(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
-; RV64I-NEXT:    sb a3, 31(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 30(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 29(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 28(a2)
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
+; RV64I-NEXT:    sb a3, 15(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
-; RV64I-NEXT:    sb a3, 27(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 26(a2)
+; RV64I-NEXT:    srli a4, a1, 16
 ; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
-; RV64I-NEXT:    srli a1, t0, 56
-; RV64I-NEXT:    sb a1, 23(a2)
-; RV64I-NEXT:    srli a1, t0, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, t0, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, t0, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, t0, 24
-; RV64I-NEXT:    sb a1, 19(a2)
-; RV64I-NEXT:    srli a1, t0, 16
-; RV64I-NEXT:    sb a1, 18(a2)
-; RV64I-NEXT:    srli a1, t0, 8
-; RV64I-NEXT:    sb a1, 17(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 1(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
+; RV64I-NEXT:    sb a3, 11(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -4522,6 +4526,7 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw s0, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s1, 72(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s2, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 64(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lbu a3, 1(a0)
 ; RV32I-NEXT:    lbu a4, 0(a0)
 ; RV32I-NEXT:    lbu a5, 2(a0)
@@ -4613,124 +4618,125 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    or a1, a1, t4
 ; RV32I-NEXT:    srai a0, a0, 31
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw a0, 56(sp)
-; RV32I-NEXT:    sw a0, 52(sp)
 ; RV32I-NEXT:    sw a0, 48(sp)
-; RV32I-NEXT:    sw a0, 44(sp)
-; RV32I-NEXT:    sw a0, 40(sp)
-; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 52(sp)
+; RV32I-NEXT:    sw a0, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a0, 32(sp)
-; RV32I-NEXT:    sw t2, 28(sp)
-; RV32I-NEXT:    sw t1, 24(sp)
-; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    sw a0, 44(sp)
 ; RV32I-NEXT:    sw a7, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw t1, 24(sp)
+; RV32I-NEXT:    sw t2, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    andi a0, a1, 28
 ; RV32I-NEXT:    mv a3, sp
-; RV32I-NEXT:    add a3, a3, a0
-; RV32I-NEXT:    lw a6, 0(a3)
-; RV32I-NEXT:    lw a7, 4(a3)
-; RV32I-NEXT:    slli a5, a1, 3
-; RV32I-NEXT:    lw t0, 8(a3)
-; RV32I-NEXT:    lw t1, 12(a3)
-; RV32I-NEXT:    srl a4, a7, a5
-; RV32I-NEXT:    andi a0, a5, 24
-; RV32I-NEXT:    xori t2, a0, 31
-; RV32I-NEXT:    slli a0, t0, 1
-; RV32I-NEXT:    sll a0, a0, t2
-; RV32I-NEXT:    or a0, a4, a0
-; RV32I-NEXT:    srl a6, a6, a5
-; RV32I-NEXT:    slli a7, a7, 1
-; RV32I-NEXT:    sll a1, a7, t2
-; RV32I-NEXT:    or a1, a6, a1
-; RV32I-NEXT:    srl a7, t1, a5
-; RV32I-NEXT:    lw t3, 16(a3)
-; RV32I-NEXT:    lw t4, 20(a3)
-; RV32I-NEXT:    lw t5, 24(a3)
-; RV32I-NEXT:    lw t6, 28(a3)
-; RV32I-NEXT:    slli a3, t3, 1
-; RV32I-NEXT:    sll a3, a3, t2
-; RV32I-NEXT:    or a3, a7, a3
-; RV32I-NEXT:    srl t0, t0, a5
-; RV32I-NEXT:    slli t1, t1, 1
-; RV32I-NEXT:    sll t1, t1, t2
-; RV32I-NEXT:    or t1, t0, t1
-; RV32I-NEXT:    srl s0, t4, a5
+; RV32I-NEXT:    add a6, a3, a0
+; RV32I-NEXT:    lw a3, 0(a6)
+; RV32I-NEXT:    lw a4, 4(a6)
+; RV32I-NEXT:    slli t1, a1, 3
+; RV32I-NEXT:    lw a7, 8(a6)
+; RV32I-NEXT:    lw t0, 12(a6)
+; RV32I-NEXT:    srl a0, a4, t1
+; RV32I-NEXT:    andi a1, t1, 24
+; RV32I-NEXT:    xori t2, a1, 31
+; RV32I-NEXT:    slli a1, a7, 1
+; RV32I-NEXT:    sll a1, a1, t2
+; RV32I-NEXT:    or a1, a0, a1
+; RV32I-NEXT:    srl a3, a3, t1
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    sll a4, a4, t2
+; RV32I-NEXT:    or a4, a3, a4
+; RV32I-NEXT:    srl a5, t0, t1
+; RV32I-NEXT:    lw t3, 16(a6)
+; RV32I-NEXT:    lw t4, 20(a6)
+; RV32I-NEXT:    lw t5, 24(a6)
+; RV32I-NEXT:    lw t6, 28(a6)
+; RV32I-NEXT:    slli a6, t3, 1
+; RV32I-NEXT:    sll a6, a6, t2
+; RV32I-NEXT:    or a6, a5, a6
+; RV32I-NEXT:    srl a7, a7, t1
+; RV32I-NEXT:    slli t0, t0, 1
+; RV32I-NEXT:    sll t0, t0, t2
+; RV32I-NEXT:    or t0, a7, t0
+; RV32I-NEXT:    srl s0, t4, t1
 ; RV32I-NEXT:    slli s1, t5, 1
 ; RV32I-NEXT:    sll s1, s1, t2
 ; RV32I-NEXT:    or s1, s0, s1
-; RV32I-NEXT:    srl t3, t3, a5
+; RV32I-NEXT:    srl t3, t3, t1
 ; RV32I-NEXT:    slli t4, t4, 1
 ; RV32I-NEXT:    sll t4, t4, t2
 ; RV32I-NEXT:    or t4, t3, t4
-; RV32I-NEXT:    srl t5, t5, a5
+; RV32I-NEXT:    srl t5, t5, t1
 ; RV32I-NEXT:    slli s2, t6, 1
 ; RV32I-NEXT:    sll t2, s2, t2
 ; RV32I-NEXT:    or t2, t5, t2
-; RV32I-NEXT:    sra a5, t6, a5
+; RV32I-NEXT:    sra t1, t6, t1
+; RV32I-NEXT:    srli t6, t1, 24
+; RV32I-NEXT:    srli s2, t1, 16
+; RV32I-NEXT:    srli s3, t1, 8
+; RV32I-NEXT:    sb t1, 28(a2)
+; RV32I-NEXT:    sb s3, 29(a2)
+; RV32I-NEXT:    sb s2, 30(a2)
+; RV32I-NEXT:    sb t6, 31(a2)
+; RV32I-NEXT:    srli t1, t2, 24
+; RV32I-NEXT:    srli t6, t2, 16
+; RV32I-NEXT:    srli t2, t2, 8
 ; RV32I-NEXT:    sb t5, 24(a2)
-; RV32I-NEXT:    sb a5, 28(a2)
+; RV32I-NEXT:    sb t2, 25(a2)
+; RV32I-NEXT:    sb t6, 26(a2)
+; RV32I-NEXT:    sb t1, 27(a2)
+; RV32I-NEXT:    srli t1, t4, 24
+; RV32I-NEXT:    srli t2, t4, 16
+; RV32I-NEXT:    srli t4, t4, 8
 ; RV32I-NEXT:    sb t3, 16(a2)
-; RV32I-NEXT:    sb s0, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a7, 12(a2)
-; RV32I-NEXT:    sb a6, 0(a2)
-; RV32I-NEXT:    sb a4, 4(a2)
-; RV32I-NEXT:    srli a4, a5, 24
-; RV32I-NEXT:    sb a4, 31(a2)
-; RV32I-NEXT:    srli a4, a5, 16
-; RV32I-NEXT:    sb a4, 30(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a4, t2, 24
-; RV32I-NEXT:    sb a4, 27(a2)
-; RV32I-NEXT:    srli a4, t2, 16
-; RV32I-NEXT:    sb a4, 26(a2)
-; RV32I-NEXT:    srli a4, t2, 8
-; RV32I-NEXT:    sb a4, 25(a2)
-; RV32I-NEXT:    srli a4, t4, 24
-; RV32I-NEXT:    sb a4, 19(a2)
-; RV32I-NEXT:    srli a4, t4, 16
-; RV32I-NEXT:    sb a4, 18(a2)
-; RV32I-NEXT:    srli a4, t4, 8
-; RV32I-NEXT:    sb a4, 17(a2)
-; RV32I-NEXT:    srli a4, s1, 24
-; RV32I-NEXT:    sb a4, 23(a2)
-; RV32I-NEXT:    srli a4, s1, 16
-; RV32I-NEXT:    sb a4, 22(a2)
+; RV32I-NEXT:    sb t4, 17(a2)
+; RV32I-NEXT:    sb t2, 18(a2)
+; RV32I-NEXT:    sb t1, 19(a2)
+; RV32I-NEXT:    srli t1, s1, 24
+; RV32I-NEXT:    srli t2, s1, 16
 ; RV32I-NEXT:    srli s1, s1, 8
+; RV32I-NEXT:    sb s0, 20(a2)
 ; RV32I-NEXT:    sb s1, 21(a2)
-; RV32I-NEXT:    srli a4, t1, 24
-; RV32I-NEXT:    sb a4, 11(a2)
-; RV32I-NEXT:    srli a4, t1, 16
-; RV32I-NEXT:    sb a4, 10(a2)
-; RV32I-NEXT:    srli a4, t1, 8
-; RV32I-NEXT:    sb a4, 9(a2)
-; RV32I-NEXT:    srli a4, a3, 24
-; RV32I-NEXT:    sb a4, 15(a2)
-; RV32I-NEXT:    srli a4, a3, 16
-; RV32I-NEXT:    sb a4, 14(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
+; RV32I-NEXT:    sb t2, 22(a2)
+; RV32I-NEXT:    sb t1, 23(a2)
+; RV32I-NEXT:    srli t1, t0, 24
+; RV32I-NEXT:    srli t2, t0, 16
+; RV32I-NEXT:    srli t0, t0, 8
+; RV32I-NEXT:    sb a7, 8(a2)
+; RV32I-NEXT:    sb t0, 9(a2)
+; RV32I-NEXT:    sb t2, 10(a2)
+; RV32I-NEXT:    sb t1, 11(a2)
+; RV32I-NEXT:    srli a7, a6, 24
+; RV32I-NEXT:    srli t0, a6, 16
+; RV32I-NEXT:    srli a6, a6, 8
+; RV32I-NEXT:    sb a5, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb t0, 14(a2)
+; RV32I-NEXT:    sb a7, 15(a2)
+; RV32I-NEXT:    srli a5, a4, 24
+; RV32I-NEXT:    srli a6, a4, 16
+; RV32I-NEXT:    srli a4, a4, 8
+; RV32I-NEXT:    sb a3, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
+; RV32I-NEXT:    sb a6, 2(a2)
+; RV32I-NEXT:    sb a5, 3(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 3(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 2(a2)
+; RV32I-NEXT:    srli a4, a1, 16
 ; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a1, 5(a2)
+; RV32I-NEXT:    sb a4, 6(a2)
+; RV32I-NEXT:    sb a3, 7(a2)
 ; RV32I-NEXT:    lw s0, 76(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 72(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s2, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 64(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 80
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -4856,98 +4862,98 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a7
 ; RV64I-NEXT:    sraiw a0, a0, 31
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    sd a0, 32(sp)
-; RV64I-NEXT:    sd a6, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a6, 24(sp)
 ; RV64I-NEXT:    slli a0, a1, 2
 ; RV64I-NEXT:    andi a0, a0, 24
 ; RV64I-NEXT:    mv a3, sp
 ; RV64I-NEXT:    add a0, a3, a0
-; RV64I-NEXT:    ld a3, 0(a0)
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    slli a1, a1, 5
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
-; RV64I-NEXT:    andi a0, a1, 32
-; RV64I-NEXT:    xori t0, a0, 63
-; RV64I-NEXT:    slli a0, a5, 1
-; RV64I-NEXT:    sll a0, a0, t0
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    srl a3, a3, a1
-; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    sll a4, a4, t0
-; RV64I-NEXT:    or a4, a3, a4
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a6, 1
-; RV64I-NEXT:    sll t0, t1, t0
-; RV64I-NEXT:    or t0, a5, t0
-; RV64I-NEXT:    sra a1, a6, a1
+; RV64I-NEXT:    ld a4, 0(a0)
+; RV64I-NEXT:    ld a5, 8(a0)
+; RV64I-NEXT:    slli a6, a1, 5
+; RV64I-NEXT:    ld a7, 16(a0)
+; RV64I-NEXT:    ld t0, 24(a0)
+; RV64I-NEXT:    srl a3, a5, a6
+; RV64I-NEXT:    andi a0, a6, 32
+; RV64I-NEXT:    xori t1, a0, 63
+; RV64I-NEXT:    slli a0, a7, 1
+; RV64I-NEXT:    sll a0, a0, t1
+; RV64I-NEXT:    or a0, a3, a0
+; RV64I-NEXT:    srl t2, a4, a6
+; RV64I-NEXT:    slli a5, a5, 1
+; RV64I-NEXT:    sll a1, a5, t1
+; RV64I-NEXT:    or a1, t2, a1
+; RV64I-NEXT:    srl a5, a7, a6
+; RV64I-NEXT:    slli a4, t0, 1
+; RV64I-NEXT:    sll a4, a4, t1
+; RV64I-NEXT:    or a4, a5, a4
+; RV64I-NEXT:    sra a6, t0, a6
+; RV64I-NEXT:    srli a7, a5, 24
+; RV64I-NEXT:    srli t0, a5, 16
+; RV64I-NEXT:    srli t1, a5, 8
 ; RV64I-NEXT:    sb a5, 16(a2)
-; RV64I-NEXT:    sb a1, 24(a2)
-; RV64I-NEXT:    sb a3, 0(a2)
-; RV64I-NEXT:    sb a7, 8(a2)
-; RV64I-NEXT:    srli a6, a5, 24
-; RV64I-NEXT:    sb a6, 19(a2)
-; RV64I-NEXT:    srli a6, a5, 16
-; RV64I-NEXT:    sb a6, 18(a2)
-; RV64I-NEXT:    srli a5, a5, 8
-; RV64I-NEXT:    sb a5, 17(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    sb t1, 17(a2)
+; RV64I-NEXT:    sb t0, 18(a2)
+; RV64I-NEXT:    sb a7, 19(a2)
+; RV64I-NEXT:    srli a5, a6, 56
+; RV64I-NEXT:    srli a7, a6, 48
+; RV64I-NEXT:    srli t0, a6, 40
+; RV64I-NEXT:    srli t1, a6, 32
+; RV64I-NEXT:    sb t1, 28(a2)
+; RV64I-NEXT:    sb t0, 29(a2)
+; RV64I-NEXT:    sb a7, 30(a2)
 ; RV64I-NEXT:    sb a5, 31(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 30(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 29(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 28(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a6, 24
+; RV64I-NEXT:    srli a7, a6, 16
+; RV64I-NEXT:    srli t0, a6, 8
+; RV64I-NEXT:    sb a6, 24(a2)
+; RV64I-NEXT:    sb t0, 25(a2)
+; RV64I-NEXT:    sb a7, 26(a2)
 ; RV64I-NEXT:    sb a5, 27(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 26(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 1(a2)
-; RV64I-NEXT:    srli a1, a7, 24
-; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a7, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a1, a7, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    srli a1, t0, 56
-; RV64I-NEXT:    sb a1, 23(a2)
-; RV64I-NEXT:    srli a1, t0, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, t0, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, t0, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    srli a5, t2, 24
+; RV64I-NEXT:    srli a6, t2, 16
+; RV64I-NEXT:    srli a7, t2, 8
+; RV64I-NEXT:    sb t2, 0(a2)
+; RV64I-NEXT:    sb a7, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
+; RV64I-NEXT:    sb a5, 3(a2)
+; RV64I-NEXT:    srli a5, a3, 24
+; RV64I-NEXT:    srli a6, a3, 16
+; RV64I-NEXT:    srli a7, a3, 8
+; RV64I-NEXT:    sb a3, 8(a2)
+; RV64I-NEXT:    sb a7, 9(a2)
+; RV64I-NEXT:    sb a6, 10(a2)
+; RV64I-NEXT:    sb a5, 11(a2)
+; RV64I-NEXT:    srli a3, a4, 56
+; RV64I-NEXT:    srli a5, a4, 48
+; RV64I-NEXT:    srli a6, a4, 40
 ; RV64I-NEXT:    srli a4, a4, 32
-; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a4, 20(a2)
+; RV64I-NEXT:    sb a6, 21(a2)
+; RV64I-NEXT:    sb a5, 22(a2)
+; RV64I-NEXT:    sb a3, 23(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    sb a1, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    sb a0, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
+; RV64I-NEXT:    sb a1, 15(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -5036,90 +5042,90 @@ define void @ashr_32bytes_wordOff(ptr %src.ptr, ptr %wordOff.ptr, ptr %dst) noun
 ; RV32I-NEXT:    or t2, t2, t3
 ; RV32I-NEXT:    lbu a1, 0(a1)
 ; RV32I-NEXT:    srai a0, a0, 31
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw a0, 56(sp)
-; RV32I-NEXT:    sw a0, 52(sp)
 ; RV32I-NEXT:    sw a0, 48(sp)
-; RV32I-NEXT:    sw a0, 44(sp)
-; RV32I-NEXT:    sw a0, 40(sp)
-; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 52(sp)
+; RV32I-NEXT:    sw a0, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a0, 32(sp)
-; RV32I-NEXT:    sw t2, 28(sp)
-; RV32I-NEXT:    sw t1, 24(sp)
-; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    sw a0, 44(sp)
 ; RV32I-NEXT:    sw a7, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw t1, 24(sp)
+; RV32I-NEXT:    sw t2, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    slli a1, a1, 2
 ; RV32I-NEXT:    andi a1, a1, 28
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    add a1, a0, a1
-; RV32I-NEXT:    lw a3, 16(a1)
-; RV32I-NEXT:    lw a4, 20(a1)
-; RV32I-NEXT:    lw a5, 24(a1)
-; RV32I-NEXT:    lw a6, 28(a1)
-; RV32I-NEXT:    lw a7, 0(a1)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    lw t0, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sb a5, 24(a2)
-; RV32I-NEXT:    sb a6, 28(a2)
-; RV32I-NEXT:    sb a3, 16(a2)
-; RV32I-NEXT:    sb a4, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a7, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
-; RV32I-NEXT:    srli t1, a5, 24
+; RV32I-NEXT:    add a4, a0, a1
+; RV32I-NEXT:    lw a5, 16(a4)
+; RV32I-NEXT:    lw a6, 20(a4)
+; RV32I-NEXT:    lw a7, 24(a4)
+; RV32I-NEXT:    lw a1, 0(a4)
+; RV32I-NEXT:    lw a0, 4(a4)
+; RV32I-NEXT:    lw t0, 8(a4)
+; RV32I-NEXT:    lw a3, 12(a4)
+; RV32I-NEXT:    lw a4, 28(a4)
+; RV32I-NEXT:    srli t1, a7, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 8
+; RV32I-NEXT:    sb a7, 24(a2)
+; RV32I-NEXT:    sb t3, 25(a2)
+; RV32I-NEXT:    sb t2, 26(a2)
 ; RV32I-NEXT:    sb t1, 27(a2)
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    sb t1, 26(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 25(a2)
-; RV32I-NEXT:    srli a5, a6, 24
-; RV32I-NEXT:    sb a5, 31(a2)
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 8
+; RV32I-NEXT:    sb a4, 28(a2)
+; RV32I-NEXT:    sb t2, 29(a2)
+; RV32I-NEXT:    sb t1, 30(a2)
+; RV32I-NEXT:    sb a7, 31(a2)
+; RV32I-NEXT:    srli a4, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t1, a5, 8
+; RV32I-NEXT:    sb a5, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb a7, 18(a2)
+; RV32I-NEXT:    sb a4, 19(a2)
+; RV32I-NEXT:    srli a4, a6, 24
 ; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 30(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    sb a5, 19(a2)
+; RV32I-NEXT:    srli a7, a6, 8
+; RV32I-NEXT:    sb a6, 20(a2)
+; RV32I-NEXT:    sb a7, 21(a2)
+; RV32I-NEXT:    sb a5, 22(a2)
+; RV32I-NEXT:    sb a4, 23(a2)
+; RV32I-NEXT:    srli a4, t0, 24
+; RV32I-NEXT:    srli a5, t0, 16
+; RV32I-NEXT:    srli a6, t0, 8
+; RV32I-NEXT:    sb t0, 8(a2)
+; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a4, a3, 24
 ; RV32I-NEXT:    srli a5, a3, 16
-; RV32I-NEXT:    sb a5, 18(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 17(a2)
-; RV32I-NEXT:    srli a3, a4, 24
-; RV32I-NEXT:    sb a3, 23(a2)
-; RV32I-NEXT:    srli a3, a4, 16
-; RV32I-NEXT:    sb a3, 22(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 21(a2)
-; RV32I-NEXT:    srli a3, t0, 24
-; RV32I-NEXT:    sb a3, 11(a2)
-; RV32I-NEXT:    srli a3, t0, 16
-; RV32I-NEXT:    sb a3, 10(a2)
-; RV32I-NEXT:    srli a3, t0, 8
-; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
+; RV32I-NEXT:    sb a4, 15(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 15(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
-; RV32I-NEXT:    srli a1, a7, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a1, a7, 16
-; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a7, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -5224,82 +5230,82 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
 ; RV64I-NEXT:    or a6, a7, a6
 ; RV64I-NEXT:    lbu a1, 0(a1)
 ; RV64I-NEXT:    sraiw a0, a0, 31
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    sd a0, 32(sp)
-; RV64I-NEXT:    sd a6, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a6, 24(sp)
 ; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    andi a1, a1, 24
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 16(a0)
-; RV64I-NEXT:    ld a3, 24(a0)
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a0, 8(a0)
-; RV64I-NEXT:    sb a1, 16(a2)
-; RV64I-NEXT:    sb a3, 24(a2)
-; RV64I-NEXT:    sb a4, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    add a3, a0, a1
+; RV64I-NEXT:    ld a4, 16(a3)
+; RV64I-NEXT:    ld a0, 8(a3)
+; RV64I-NEXT:    ld a1, 0(a3)
+; RV64I-NEXT:    ld a3, 24(a3)
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 20(a2)
+; RV64I-NEXT:    sb a7, 21(a2)
+; RV64I-NEXT:    sb a6, 22(a2)
 ; RV64I-NEXT:    sb a5, 23(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 22(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 21(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 20(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a7, a4, 8
+; RV64I-NEXT:    sb a4, 16(a2)
+; RV64I-NEXT:    sb a7, 17(a2)
+; RV64I-NEXT:    sb a6, 18(a2)
 ; RV64I-NEXT:    sb a5, 19(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 18(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 17(a2)
-; RV64I-NEXT:    srli a1, a3, 56
-; RV64I-NEXT:    sb a1, 31(a2)
-; RV64I-NEXT:    srli a1, a3, 48
-; RV64I-NEXT:    sb a1, 30(a2)
-; RV64I-NEXT:    srli a1, a3, 40
-; RV64I-NEXT:    sb a1, 29(a2)
-; RV64I-NEXT:    srli a1, a3, 32
-; RV64I-NEXT:    sb a1, 28(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 27(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 26(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 25(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    srli a4, a3, 56
+; RV64I-NEXT:    srli a5, a3, 48
+; RV64I-NEXT:    srli a6, a3, 40
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    sb a7, 28(a2)
+; RV64I-NEXT:    sb a6, 29(a2)
+; RV64I-NEXT:    sb a5, 30(a2)
+; RV64I-NEXT:    sb a4, 31(a2)
+; RV64I-NEXT:    srli a4, a3, 24
+; RV64I-NEXT:    srli a5, a3, 16
+; RV64I-NEXT:    srli a6, a3, 8
+; RV64I-NEXT:    sb a3, 24(a2)
+; RV64I-NEXT:    sb a6, 25(a2)
+; RV64I-NEXT:    sb a5, 26(a2)
+; RV64I-NEXT:    sb a4, 27(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
+; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -5388,90 +5394,90 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
 ; RV32I-NEXT:    or t2, t2, t3
 ; RV32I-NEXT:    lbu a1, 0(a1)
 ; RV32I-NEXT:    srai a0, a0, 31
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw a0, 56(sp)
-; RV32I-NEXT:    sw a0, 52(sp)
 ; RV32I-NEXT:    sw a0, 48(sp)
-; RV32I-NEXT:    sw a0, 44(sp)
-; RV32I-NEXT:    sw a0, 40(sp)
-; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 52(sp)
+; RV32I-NEXT:    sw a0, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a0, 32(sp)
-; RV32I-NEXT:    sw t2, 28(sp)
-; RV32I-NEXT:    sw t1, 24(sp)
-; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    sw a0, 44(sp)
 ; RV32I-NEXT:    sw a7, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t0, 20(sp)
+; RV32I-NEXT:    sw t1, 24(sp)
+; RV32I-NEXT:    sw t2, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    slli a1, a1, 3
 ; RV32I-NEXT:    andi a1, a1, 24
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    add a1, a0, a1
-; RV32I-NEXT:    lw a3, 16(a1)
-; RV32I-NEXT:    lw a4, 20(a1)
-; RV32I-NEXT:    lw a5, 24(a1)
-; RV32I-NEXT:    lw a6, 28(a1)
-; RV32I-NEXT:    lw a7, 0(a1)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    lw t0, 8(a1)
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    sb a5, 24(a2)
-; RV32I-NEXT:    sb a6, 28(a2)
-; RV32I-NEXT:    sb a3, 16(a2)
-; RV32I-NEXT:    sb a4, 20(a2)
-; RV32I-NEXT:    sb t0, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a7, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
-; RV32I-NEXT:    srli t1, a5, 24
+; RV32I-NEXT:    add a4, a0, a1
+; RV32I-NEXT:    lw a5, 16(a4)
+; RV32I-NEXT:    lw a6, 20(a4)
+; RV32I-NEXT:    lw a7, 24(a4)
+; RV32I-NEXT:    lw a1, 0(a4)
+; RV32I-NEXT:    lw a0, 4(a4)
+; RV32I-NEXT:    lw t0, 8(a4)
+; RV32I-NEXT:    lw a3, 12(a4)
+; RV32I-NEXT:    lw a4, 28(a4)
+; RV32I-NEXT:    srli t1, a7, 24
+; RV32I-NEXT:    srli t2, a7, 16
+; RV32I-NEXT:    srli t3, a7, 8
+; RV32I-NEXT:    sb a7, 24(a2)
+; RV32I-NEXT:    sb t3, 25(a2)
+; RV32I-NEXT:    sb t2, 26(a2)
 ; RV32I-NEXT:    sb t1, 27(a2)
-; RV32I-NEXT:    srli t1, a5, 16
-; RV32I-NEXT:    sb t1, 26(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 25(a2)
-; RV32I-NEXT:    srli a5, a6, 24
-; RV32I-NEXT:    sb a5, 31(a2)
+; RV32I-NEXT:    srli a7, a4, 24
+; RV32I-NEXT:    srli t1, a4, 16
+; RV32I-NEXT:    srli t2, a4, 8
+; RV32I-NEXT:    sb a4, 28(a2)
+; RV32I-NEXT:    sb t2, 29(a2)
+; RV32I-NEXT:    sb t1, 30(a2)
+; RV32I-NEXT:    sb a7, 31(a2)
+; RV32I-NEXT:    srli a4, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t1, a5, 8
+; RV32I-NEXT:    sb a5, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb a7, 18(a2)
+; RV32I-NEXT:    sb a4, 19(a2)
+; RV32I-NEXT:    srli a4, a6, 24
 ; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 30(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 29(a2)
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    sb a5, 19(a2)
+; RV32I-NEXT:    srli a7, a6, 8
+; RV32I-NEXT:    sb a6, 20(a2)
+; RV32I-NEXT:    sb a7, 21(a2)
+; RV32I-NEXT:    sb a5, 22(a2)
+; RV32I-NEXT:    sb a4, 23(a2)
+; RV32I-NEXT:    srli a4, t0, 24
+; RV32I-NEXT:    srli a5, t0, 16
+; RV32I-NEXT:    srli a6, t0, 8
+; RV32I-NEXT:    sb t0, 8(a2)
+; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a4, a3, 24
 ; RV32I-NEXT:    srli a5, a3, 16
-; RV32I-NEXT:    sb a5, 18(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 17(a2)
-; RV32I-NEXT:    srli a3, a4, 24
-; RV32I-NEXT:    sb a3, 23(a2)
-; RV32I-NEXT:    srli a3, a4, 16
-; RV32I-NEXT:    sb a3, 22(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 21(a2)
-; RV32I-NEXT:    srli a3, t0, 24
-; RV32I-NEXT:    sb a3, 11(a2)
-; RV32I-NEXT:    srli a3, t0, 16
-; RV32I-NEXT:    sb a3, 10(a2)
-; RV32I-NEXT:    srli a3, t0, 8
-; RV32I-NEXT:    sb a3, 9(a2)
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
+; RV32I-NEXT:    sb a4, 15(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 15(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
-; RV32I-NEXT:    srli a1, a7, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a1, a7, 16
-; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a7, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
diff --git a/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll b/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
index 190d67a5d8c118821428f25b20842e11bc7e049e..d36c660b3b142143c7b7fc9355ad7c854110e9e0 100644
--- a/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
+++ b/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
@@ -17,13 +17,13 @@ define void @lshr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, a5
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    srlw a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_4bytes:
@@ -49,13 +49,13 @@ define void @lshr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    or a1, a1, a4
 ; RV32I-NEXT:    srl a0, a0, a1
-; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i32, ptr %src.ptr, align 1
   %bitOff = load i32, ptr %bitOff.ptr, align 1
@@ -78,13 +78,13 @@ define void @shl_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, a5
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    sllw a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_4bytes:
@@ -110,13 +110,13 @@ define void @shl_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    or a1, a1, a4
 ; RV32I-NEXT:    sll a0, a0, a1
-; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i32, ptr %src.ptr, align 1
   %bitOff = load i32, ptr %bitOff.ptr, align 1
@@ -139,13 +139,13 @@ define void @ashr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a0, a0, a5
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    sraw a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_4bytes:
@@ -171,13 +171,13 @@ define void @ashr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a3
 ; RV32I-NEXT:    or a1, a1, a4
 ; RV32I-NEXT:    sra a0, a0, a1
-; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i32, ptr %src.ptr, align 1
   %bitOff = load i32, ptr %bitOff.ptr, align 1
@@ -234,21 +234,21 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 48
+; RV64I-NEXT:    srli a3, a0, 56
+; RV64I-NEXT:    srli a4, a0, 32
+; RV64I-NEXT:    srli a5, a0, 40
+; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
 ; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_8bytes:
@@ -298,20 +298,20 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:  .LBB3_3:
 ; RV32I-NEXT:    srai a4, a4, 31
 ; RV32I-NEXT:    and a1, a4, a1
-; RV32I-NEXT:    sb a1, 4(a2)
 ; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    srli a4, a1, 24
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a5, 5(a2)
 ; RV32I-NEXT:    sb a3, 6(a2)
-; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 7(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 5(a2)
-; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 7(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i64, ptr %src.ptr, align 1
   %bitOff = load i64, ptr %bitOff.ptr, align 1
@@ -367,21 +367,21 @@ define void @shl_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 48
+; RV64I-NEXT:    srli a3, a0, 56
+; RV64I-NEXT:    srli a4, a0, 32
+; RV64I-NEXT:    srli a5, a0, 40
+; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
 ; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_8bytes:
@@ -431,20 +431,20 @@ define void @shl_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:  .LBB4_3:
 ; RV32I-NEXT:    srai a4, a4, 31
 ; RV32I-NEXT:    and a1, a4, a1
-; RV32I-NEXT:    sb a1, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
 ; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    srli a4, a1, 24
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
 ; RV32I-NEXT:    sb a3, 2(a2)
-; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 3(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a2)
+; RV32I-NEXT:    sb a4, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
 ; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a3, 7(a2)
 ; RV32I-NEXT:    ret
   %src = load i64, ptr %src.ptr, align 1
   %bitOff = load i64, ptr %bitOff.ptr, align 1
@@ -500,21 +500,21 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    sra a0, a0, a1
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 48
+; RV64I-NEXT:    srli a3, a0, 56
+; RV64I-NEXT:    srli a4, a0, 32
+; RV64I-NEXT:    srli a5, a0, 40
+; RV64I-NEXT:    sb a4, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
 ; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
 ; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
 ; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a1, a0, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_8bytes:
@@ -564,20 +564,20 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll a3, a3, a4
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:  .LBB5_3:
-; RV32I-NEXT:    sb a1, 4(a2)
 ; RV32I-NEXT:    srli a3, a1, 16
+; RV32I-NEXT:    srli a4, a1, 24
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a5, 5(a2)
 ; RV32I-NEXT:    sb a3, 6(a2)
-; RV32I-NEXT:    srli a3, a1, 24
-; RV32I-NEXT:    sb a3, 7(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 5(a2)
-; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 7(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a1, 2(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 3(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 1(a2)
+; RV32I-NEXT:    sb a3, 3(a2)
 ; RV32I-NEXT:    ret
   %src = load i64, ptr %src.ptr, align 1
   %bitOff = load i64, ptr %bitOff.ptr, align 1
@@ -670,36 +670,36 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:  .LBB6_3:
 ; RV64I-NEXT:    srai a4, a4, 63
 ; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    sb a1, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
 ; RV64I-NEXT:    sb a3, 15(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 14(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 13(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 12(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a5, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a3, 6(a2)
 ; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
 ; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_16bytes:
@@ -755,14 +755,14 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, a7
 ; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a0, 12(sp)
 ; RV32I-NEXT:    srli a0, a1, 3
 ; RV32I-NEXT:    andi a0, a0, 12
 ; RV32I-NEXT:    mv a3, sp
@@ -786,34 +786,34 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll a5, a5, a7
 ; RV32I-NEXT:    or a4, a4, a5
 ; RV32I-NEXT:    srl a0, a0, a1
-; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a5, a0, 24
+; RV32I-NEXT:    srli a7, a0, 8
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a7, 13(a2)
 ; RV32I-NEXT:    sb a1, 14(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 15(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 13(a2)
-; RV32I-NEXT:    sb a4, 8(a2)
-; RV32I-NEXT:    sb a3, 0(a2)
-; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a5, 15(a2)
 ; RV32I-NEXT:    srli a0, a4, 16
+; RV32I-NEXT:    srli a1, a4, 24
+; RV32I-NEXT:    srli a5, a4, 8
+; RV32I-NEXT:    sb a4, 8(a2)
+; RV32I-NEXT:    sb a5, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
-; RV32I-NEXT:    srli a0, a4, 24
-; RV32I-NEXT:    sb a0, 11(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a1, 11(a2)
 ; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a1, a3, 24
+; RV32I-NEXT:    srli a4, a3, 8
+; RV32I-NEXT:    sb a3, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a0, 2(a2)
-; RV32I-NEXT:    srli a0, a3, 24
-; RV32I-NEXT:    sb a0, 3(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 1(a2)
+; RV32I-NEXT:    sb a1, 3(a2)
 ; RV32I-NEXT:    srli a0, a6, 16
+; RV32I-NEXT:    srli a1, a6, 24
+; RV32I-NEXT:    srli a3, a6, 8
+; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a3, 5(a2)
 ; RV32I-NEXT:    sb a0, 6(a2)
-; RV32I-NEXT:    srli a0, a6, 24
-; RV32I-NEXT:    sb a0, 7(a2)
-; RV32I-NEXT:    srli a0, a6, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a1, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -906,36 +906,36 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:  .LBB7_3:
 ; RV64I-NEXT:    srai a4, a4, 63
 ; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    sb a1, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
 ; RV64I-NEXT:    sb a3, 7(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 6(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 5(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 4(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
 ; RV64I-NEXT:    sb a3, 3(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 2(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 1(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_16bytes:
@@ -991,14 +991,14 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, a7
 ; RV32I-NEXT:    or a1, a1, a6
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw a5, 24(sp)
-; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw a5, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    srli a0, a1, 3
 ; RV32I-NEXT:    andi a0, a0, 12
 ; RV32I-NEXT:    addi a3, sp, 16
@@ -1022,34 +1022,34 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srl a0, a0, a7
 ; RV32I-NEXT:    or a0, a5, a0
 ; RV32I-NEXT:    sll a1, a4, a1
-; RV32I-NEXT:    sb a1, 0(a2)
 ; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 24
+; RV32I-NEXT:    srli a7, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a7, 1(a2)
 ; RV32I-NEXT:    sb a4, 2(a2)
-; RV32I-NEXT:    srli a4, a1, 24
-; RV32I-NEXT:    sb a4, 3(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a2)
-; RV32I-NEXT:    sb a0, 8(a2)
-; RV32I-NEXT:    sb a3, 12(a2)
-; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a5, 3(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a4, a0, 24
+; RV32I-NEXT:    srli a5, a0, 8
+; RV32I-NEXT:    sb a0, 8(a2)
+; RV32I-NEXT:    sb a5, 9(a2)
 ; RV32I-NEXT:    sb a1, 10(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 11(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 9(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
 ; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a1, a3, 24
+; RV32I-NEXT:    srli a4, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a4, 13(a2)
 ; RV32I-NEXT:    sb a0, 14(a2)
-; RV32I-NEXT:    srli a0, a3, 24
-; RV32I-NEXT:    sb a0, 15(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
+; RV32I-NEXT:    sb a1, 15(a2)
 ; RV32I-NEXT:    srli a0, a6, 16
+; RV32I-NEXT:    srli a1, a6, 24
+; RV32I-NEXT:    srli a3, a6, 8
+; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a3, 5(a2)
 ; RV32I-NEXT:    sb a0, 6(a2)
-; RV32I-NEXT:    srli a0, a6, 24
-; RV32I-NEXT:    sb a0, 7(a2)
-; RV32I-NEXT:    srli a0, a6, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a1, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -1142,36 +1142,36 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sll a3, a3, a4
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:  .LBB8_3:
-; RV64I-NEXT:    sb a1, 8(a2)
 ; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 12(a2)
+; RV64I-NEXT:    sb a5, 13(a2)
+; RV64I-NEXT:    sb a4, 14(a2)
 ; RV64I-NEXT:    sb a3, 15(a2)
-; RV64I-NEXT:    srli a3, a1, 48
-; RV64I-NEXT:    sb a3, 14(a2)
-; RV64I-NEXT:    srli a3, a1, 40
-; RV64I-NEXT:    sb a3, 13(a2)
-; RV64I-NEXT:    srli a3, a1, 32
-; RV64I-NEXT:    sb a3, 12(a2)
 ; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 8(a2)
+; RV64I-NEXT:    sb a5, 9(a2)
+; RV64I-NEXT:    sb a4, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    srli a3, a1, 16
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 9(a2)
-; RV64I-NEXT:    sb a0, 0(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 4(a2)
+; RV64I-NEXT:    sb a4, 5(a2)
+; RV64I-NEXT:    sb a3, 6(a2)
 ; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 4(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 0(a2)
+; RV64I-NEXT:    sb a4, 1(a2)
+; RV64I-NEXT:    sb a3, 2(a2)
 ; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 1(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_16bytes:
@@ -1228,14 +1228,14 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a7
 ; RV32I-NEXT:    or a1, a1, t0
 ; RV32I-NEXT:    srai a0, a0, 31
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 20(sp)
 ; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    sw a6, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a0, 20(sp)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a6, 12(sp)
 ; RV32I-NEXT:    srli a0, a1, 3
 ; RV32I-NEXT:    andi a0, a0, 12
 ; RV32I-NEXT:    mv a3, sp
@@ -1259,34 +1259,34 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll a5, a5, a7
 ; RV32I-NEXT:    or a4, a4, a5
 ; RV32I-NEXT:    sra a0, a0, a1
-; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    srli a5, a0, 24
+; RV32I-NEXT:    srli a7, a0, 8
+; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a7, 13(a2)
 ; RV32I-NEXT:    sb a1, 14(a2)
-; RV32I-NEXT:    srli a1, a0, 24
-; RV32I-NEXT:    sb a1, 15(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 13(a2)
-; RV32I-NEXT:    sb a4, 8(a2)
-; RV32I-NEXT:    sb a3, 0(a2)
-; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a5, 15(a2)
 ; RV32I-NEXT:    srli a0, a4, 16
+; RV32I-NEXT:    srli a1, a4, 24
+; RV32I-NEXT:    srli a5, a4, 8
+; RV32I-NEXT:    sb a4, 8(a2)
+; RV32I-NEXT:    sb a5, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
-; RV32I-NEXT:    srli a0, a4, 24
-; RV32I-NEXT:    sb a0, 11(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 9(a2)
+; RV32I-NEXT:    sb a1, 11(a2)
 ; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a1, a3, 24
+; RV32I-NEXT:    srli a4, a3, 8
+; RV32I-NEXT:    sb a3, 0(a2)
+; RV32I-NEXT:    sb a4, 1(a2)
 ; RV32I-NEXT:    sb a0, 2(a2)
-; RV32I-NEXT:    srli a0, a3, 24
-; RV32I-NEXT:    sb a0, 3(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 1(a2)
+; RV32I-NEXT:    sb a1, 3(a2)
 ; RV32I-NEXT:    srli a0, a6, 16
+; RV32I-NEXT:    srli a1, a6, 24
+; RV32I-NEXT:    srli a3, a6, 8
+; RV32I-NEXT:    sb a6, 4(a2)
+; RV32I-NEXT:    sb a3, 5(a2)
 ; RV32I-NEXT:    sb a0, 6(a2)
-; RV32I-NEXT:    srli a0, a6, 24
-; RV32I-NEXT:    sb a0, 7(a2)
-; RV32I-NEXT:    srli a0, a6, 8
-; RV32I-NEXT:    sb a0, 5(a2)
+; RV32I-NEXT:    sb a1, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %src = load i128, ptr %src.ptr, align 1
@@ -1409,98 +1409,98 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a1, a1, a7
 ; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    or a1, a1, a6
-; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    sd zero, 48(sp)
-; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    or a6, a1, a6
 ; RV64I-NEXT:    sd zero, 32(sp)
-; RV64I-NEXT:    sd a0, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    sd zero, 48(sp)
+; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
-; RV64I-NEXT:    srli a0, a1, 3
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a0, 24(sp)
+; RV64I-NEXT:    srli a0, a6, 3
 ; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    mv a3, sp
-; RV64I-NEXT:    add a0, a3, a0
-; RV64I-NEXT:    ld a3, 8(a0)
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 24(a0)
-; RV64I-NEXT:    srl a0, a3, a1
-; RV64I-NEXT:    andi a7, a1, 63
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    ld a1, 8(a0)
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    ld a4, 16(a0)
+; RV64I-NEXT:    ld a5, 24(a0)
+; RV64I-NEXT:    srl a0, a1, a6
+; RV64I-NEXT:    andi a7, a6, 63
 ; RV64I-NEXT:    xori a7, a7, 63
-; RV64I-NEXT:    slli t0, a5, 1
+; RV64I-NEXT:    slli t0, a4, 1
 ; RV64I-NEXT:    sll t0, t0, a7
 ; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    srl a4, a4, a1
-; RV64I-NEXT:    slli a3, a3, 1
-; RV64I-NEXT:    sll a3, a3, a7
-; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    srl a4, a5, a1
-; RV64I-NEXT:    slli a5, a6, 1
-; RV64I-NEXT:    sll a5, a5, a7
-; RV64I-NEXT:    or a4, a4, a5
-; RV64I-NEXT:    srl a1, a6, a1
-; RV64I-NEXT:    sb a1, 24(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    srl a3, a3, a6
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    sll a1, a1, a7
+; RV64I-NEXT:    or a1, a3, a1
+; RV64I-NEXT:    srl a3, a4, a6
+; RV64I-NEXT:    slli a4, a5, 1
+; RV64I-NEXT:    sll a4, a4, a7
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    srl a4, a5, a6
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 28(a2)
+; RV64I-NEXT:    sb a7, 29(a2)
+; RV64I-NEXT:    sb a6, 30(a2)
 ; RV64I-NEXT:    sb a5, 31(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 30(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 29(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 28(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a7, a4, 8
+; RV64I-NEXT:    sb a4, 24(a2)
+; RV64I-NEXT:    sb a7, 25(a2)
+; RV64I-NEXT:    sb a6, 26(a2)
 ; RV64I-NEXT:    sb a5, 27(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 26(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
-; RV64I-NEXT:    sb a4, 16(a2)
-; RV64I-NEXT:    sb a3, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 23(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 19(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 18(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 17(a2)
-; RV64I-NEXT:    srli a1, a3, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a3, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a3, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a3, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 1(a2)
+; RV64I-NEXT:    srli a4, a3, 56
+; RV64I-NEXT:    srli a5, a3, 48
+; RV64I-NEXT:    srli a6, a3, 40
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    sb a7, 20(a2)
+; RV64I-NEXT:    sb a6, 21(a2)
+; RV64I-NEXT:    sb a5, 22(a2)
+; RV64I-NEXT:    sb a4, 23(a2)
+; RV64I-NEXT:    srli a4, a3, 24
+; RV64I-NEXT:    srli a5, a3, 16
+; RV64I-NEXT:    srli a6, a3, 8
+; RV64I-NEXT:    sb a3, 16(a2)
+; RV64I-NEXT:    sb a6, 17(a2)
+; RV64I-NEXT:    sb a5, 18(a2)
+; RV64I-NEXT:    sb a4, 19(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
+; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -1597,22 +1597,22 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    or a6, a1, a6
-; RV32I-NEXT:    sw zero, 60(sp)
-; RV32I-NEXT:    sw zero, 56(sp)
-; RV32I-NEXT:    sw zero, 52(sp)
 ; RV32I-NEXT:    sw zero, 48(sp)
-; RV32I-NEXT:    sw zero, 44(sp)
-; RV32I-NEXT:    sw zero, 40(sp)
-; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 52(sp)
+; RV32I-NEXT:    sw zero, 56(sp)
+; RV32I-NEXT:    sw zero, 60(sp)
 ; RV32I-NEXT:    sw zero, 32(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    sw t2, 24(sp)
-; RV32I-NEXT:    sw t1, 20(sp)
+; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 40(sp)
+; RV32I-NEXT:    sw zero, 44(sp)
 ; RV32I-NEXT:    sw t0, 16(sp)
-; RV32I-NEXT:    sw a7, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t1, 20(sp)
+; RV32I-NEXT:    sw t2, 24(sp)
+; RV32I-NEXT:    sw a0, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a7, 12(sp)
 ; RV32I-NEXT:    srli a0, a6, 3
 ; RV32I-NEXT:    andi a0, a0, 28
 ; RV32I-NEXT:    mv a1, sp
@@ -1656,62 +1656,62 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll t0, t2, t0
 ; RV32I-NEXT:    or t0, t1, t0
 ; RV32I-NEXT:    srl a6, t4, a6
-; RV32I-NEXT:    sb a6, 28(a2)
 ; RV32I-NEXT:    srli t1, a6, 24
+; RV32I-NEXT:    srli t2, a6, 16
+; RV32I-NEXT:    srli t3, a6, 8
+; RV32I-NEXT:    sb a6, 28(a2)
+; RV32I-NEXT:    sb t3, 29(a2)
+; RV32I-NEXT:    sb t2, 30(a2)
 ; RV32I-NEXT:    sb t1, 31(a2)
-; RV32I-NEXT:    srli t1, a6, 16
-; RV32I-NEXT:    sb t1, 30(a2)
-; RV32I-NEXT:    srli a6, a6, 8
-; RV32I-NEXT:    sb a6, 29(a2)
-; RV32I-NEXT:    sb t0, 24(a2)
-; RV32I-NEXT:    sb a7, 16(a2)
-; RV32I-NEXT:    sb a5, 20(a2)
-; RV32I-NEXT:    sb a4, 8(a2)
-; RV32I-NEXT:    sb a3, 12(a2)
-; RV32I-NEXT:    sb a1, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
 ; RV32I-NEXT:    srli a6, t0, 24
+; RV32I-NEXT:    srli t1, t0, 16
+; RV32I-NEXT:    srli t2, t0, 8
+; RV32I-NEXT:    sb t0, 24(a2)
+; RV32I-NEXT:    sb t2, 25(a2)
+; RV32I-NEXT:    sb t1, 26(a2)
 ; RV32I-NEXT:    sb a6, 27(a2)
-; RV32I-NEXT:    srli a6, t0, 16
-; RV32I-NEXT:    sb a6, 26(a2)
-; RV32I-NEXT:    srli a6, t0, 8
-; RV32I-NEXT:    sb a6, 25(a2)
 ; RV32I-NEXT:    srli a6, a7, 24
+; RV32I-NEXT:    srli t0, a7, 16
+; RV32I-NEXT:    srli t1, a7, 8
+; RV32I-NEXT:    sb a7, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb t0, 18(a2)
 ; RV32I-NEXT:    sb a6, 19(a2)
-; RV32I-NEXT:    srli a6, a7, 16
-; RV32I-NEXT:    sb a6, 18(a2)
-; RV32I-NEXT:    srli a6, a7, 8
-; RV32I-NEXT:    sb a6, 17(a2)
 ; RV32I-NEXT:    srli a6, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t0, a5, 8
+; RV32I-NEXT:    sb a5, 20(a2)
+; RV32I-NEXT:    sb t0, 21(a2)
+; RV32I-NEXT:    sb a7, 22(a2)
 ; RV32I-NEXT:    sb a6, 23(a2)
-; RV32I-NEXT:    srli a6, a5, 16
-; RV32I-NEXT:    sb a6, 22(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 21(a2)
 ; RV32I-NEXT:    srli a5, a4, 24
+; RV32I-NEXT:    srli a6, a4, 16
+; RV32I-NEXT:    srli a7, a4, 8
+; RV32I-NEXT:    sb a4, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
 ; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a5, a4, 16
-; RV32I-NEXT:    sb a5, 10(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 9(a2)
 ; RV32I-NEXT:    srli a4, a3, 24
+; RV32I-NEXT:    srli a5, a3, 16
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
 ; RV32I-NEXT:    sb a4, 15(a2)
-; RV32I-NEXT:    srli a4, a3, 16
-; RV32I-NEXT:    sb a4, 14(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
 ; RV32I-NEXT:    sb a3, 3(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 2(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -1833,98 +1833,98 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a1, a1, a7
 ; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    or a1, a1, a6
-; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    sd zero, 16(sp)
-; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    or a6, a1, a6
 ; RV64I-NEXT:    sd zero, 0(sp)
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a5, 48(sp)
-; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    sd a3, 32(sp)
-; RV64I-NEXT:    srli a0, a1, 3
+; RV64I-NEXT:    sd a4, 40(sp)
+; RV64I-NEXT:    sd a5, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
+; RV64I-NEXT:    srli a0, a6, 3
 ; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    addi a3, sp, 32
-; RV64I-NEXT:    sub a3, a3, a0
-; RV64I-NEXT:    ld a4, 8(a3)
-; RV64I-NEXT:    ld a5, 0(a3)
-; RV64I-NEXT:    ld a6, 16(a3)
-; RV64I-NEXT:    ld a3, 24(a3)
-; RV64I-NEXT:    sll a0, a4, a1
-; RV64I-NEXT:    andi a7, a1, 63
+; RV64I-NEXT:    addi a1, sp, 32
+; RV64I-NEXT:    sub a1, a1, a0
+; RV64I-NEXT:    ld a3, 8(a1)
+; RV64I-NEXT:    ld a4, 0(a1)
+; RV64I-NEXT:    ld a5, 16(a1)
+; RV64I-NEXT:    ld a1, 24(a1)
+; RV64I-NEXT:    sll a0, a3, a6
+; RV64I-NEXT:    andi a7, a6, 63
 ; RV64I-NEXT:    xori a7, a7, 63
-; RV64I-NEXT:    srli t0, a5, 1
+; RV64I-NEXT:    srli t0, a4, 1
 ; RV64I-NEXT:    srl t0, t0, a7
 ; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    sll a3, a3, a1
-; RV64I-NEXT:    srli t0, a6, 1
+; RV64I-NEXT:    sll a1, a1, a6
+; RV64I-NEXT:    srli t0, a5, 1
 ; RV64I-NEXT:    srl t0, t0, a7
-; RV64I-NEXT:    or a3, a3, t0
-; RV64I-NEXT:    sll a6, a6, a1
-; RV64I-NEXT:    srli a4, a4, 1
-; RV64I-NEXT:    srl a4, a4, a7
-; RV64I-NEXT:    or a4, a6, a4
-; RV64I-NEXT:    sll a1, a5, a1
-; RV64I-NEXT:    sb a1, 0(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    or a1, a1, t0
+; RV64I-NEXT:    sll a5, a5, a6
+; RV64I-NEXT:    srli a3, a3, 1
+; RV64I-NEXT:    srl a3, a3, a7
+; RV64I-NEXT:    or a3, a5, a3
+; RV64I-NEXT:    sll a4, a4, a6
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 4(a2)
+; RV64I-NEXT:    sb a7, 5(a2)
+; RV64I-NEXT:    sb a6, 6(a2)
 ; RV64I-NEXT:    sb a5, 7(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 6(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 5(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 4(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a7, a4, 8
+; RV64I-NEXT:    sb a4, 0(a2)
+; RV64I-NEXT:    sb a7, 1(a2)
+; RV64I-NEXT:    sb a6, 2(a2)
 ; RV64I-NEXT:    sb a5, 3(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 2(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 1(a2)
-; RV64I-NEXT:    sb a4, 16(a2)
-; RV64I-NEXT:    sb a3, 24(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 23(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 19(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 18(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 17(a2)
-; RV64I-NEXT:    srli a1, a3, 56
-; RV64I-NEXT:    sb a1, 31(a2)
-; RV64I-NEXT:    srli a1, a3, 48
-; RV64I-NEXT:    sb a1, 30(a2)
-; RV64I-NEXT:    srli a1, a3, 40
-; RV64I-NEXT:    sb a1, 29(a2)
-; RV64I-NEXT:    srli a1, a3, 32
-; RV64I-NEXT:    sb a1, 28(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 27(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 26(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 25(a2)
+; RV64I-NEXT:    srli a4, a3, 56
+; RV64I-NEXT:    srli a5, a3, 48
+; RV64I-NEXT:    srli a6, a3, 40
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    sb a7, 20(a2)
+; RV64I-NEXT:    sb a6, 21(a2)
+; RV64I-NEXT:    sb a5, 22(a2)
+; RV64I-NEXT:    sb a4, 23(a2)
+; RV64I-NEXT:    srli a4, a3, 24
+; RV64I-NEXT:    srli a5, a3, 16
+; RV64I-NEXT:    srli a6, a3, 8
+; RV64I-NEXT:    sb a3, 16(a2)
+; RV64I-NEXT:    sb a6, 17(a2)
+; RV64I-NEXT:    sb a5, 18(a2)
+; RV64I-NEXT:    sb a4, 19(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 28(a2)
+; RV64I-NEXT:    sb a5, 29(a2)
+; RV64I-NEXT:    sb a4, 30(a2)
+; RV64I-NEXT:    sb a3, 31(a2)
+; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 24(a2)
+; RV64I-NEXT:    sb a5, 25(a2)
+; RV64I-NEXT:    sb a4, 26(a2)
+; RV64I-NEXT:    sb a3, 27(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -2021,22 +2021,22 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    or a1, a1, t3
 ; RV32I-NEXT:    or a7, a1, a7
-; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    sw zero, 24(sp)
-; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
-; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    sw zero, 8(sp)
-; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw zero, 0(sp)
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw t2, 56(sp)
-; RV32I-NEXT:    sw t1, 52(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw t0, 48(sp)
-; RV32I-NEXT:    sw a6, 44(sp)
-; RV32I-NEXT:    sw a5, 40(sp)
-; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw t1, 52(sp)
+; RV32I-NEXT:    sw t2, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a3, 32(sp)
+; RV32I-NEXT:    sw a4, 36(sp)
+; RV32I-NEXT:    sw a5, 40(sp)
+; RV32I-NEXT:    sw a6, 44(sp)
 ; RV32I-NEXT:    srli a0, a7, 3
 ; RV32I-NEXT:    andi a0, a0, 28
 ; RV32I-NEXT:    addi a1, sp, 32
@@ -2062,80 +2062,80 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lw t3, 20(a4)
 ; RV32I-NEXT:    or a3, a6, a3
 ; RV32I-NEXT:    lw a6, 24(a4)
-; RV32I-NEXT:    lw a4, 28(a4)
-; RV32I-NEXT:    sll t4, t3, a7
+; RV32I-NEXT:    lw t4, 28(a4)
+; RV32I-NEXT:    sll a4, t3, a7
 ; RV32I-NEXT:    srli t5, t2, 1
 ; RV32I-NEXT:    srl t5, t5, t1
-; RV32I-NEXT:    or t4, t4, t5
+; RV32I-NEXT:    or a4, a4, t5
 ; RV32I-NEXT:    sll t2, t2, a7
 ; RV32I-NEXT:    srli t0, t0, 1
 ; RV32I-NEXT:    srl t0, t0, t1
 ; RV32I-NEXT:    or t0, t2, t0
-; RV32I-NEXT:    sll a4, a4, a7
-; RV32I-NEXT:    srli t2, a6, 1
-; RV32I-NEXT:    srl t2, t2, t1
-; RV32I-NEXT:    or a4, a4, t2
+; RV32I-NEXT:    sll t2, t4, a7
+; RV32I-NEXT:    srli t4, a6, 1
+; RV32I-NEXT:    srl t4, t4, t1
+; RV32I-NEXT:    or t2, t2, t4
 ; RV32I-NEXT:    sll a6, a6, a7
-; RV32I-NEXT:    srli t2, t3, 1
-; RV32I-NEXT:    srl t1, t2, t1
+; RV32I-NEXT:    srli t3, t3, 1
+; RV32I-NEXT:    srl t1, t3, t1
 ; RV32I-NEXT:    or a6, a6, t1
 ; RV32I-NEXT:    sll a5, a5, a7
-; RV32I-NEXT:    sb a5, 0(a2)
 ; RV32I-NEXT:    srli a7, a5, 24
+; RV32I-NEXT:    srli t1, a5, 16
+; RV32I-NEXT:    srli t3, a5, 8
+; RV32I-NEXT:    sb a5, 0(a2)
+; RV32I-NEXT:    sb t3, 1(a2)
+; RV32I-NEXT:    sb t1, 2(a2)
 ; RV32I-NEXT:    sb a7, 3(a2)
-; RV32I-NEXT:    srli a7, a5, 16
-; RV32I-NEXT:    sb a7, 2(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 1(a2)
-; RV32I-NEXT:    sb a6, 24(a2)
-; RV32I-NEXT:    sb a4, 28(a2)
-; RV32I-NEXT:    sb t0, 16(a2)
-; RV32I-NEXT:    sb t4, 20(a2)
-; RV32I-NEXT:    sb a3, 8(a2)
-; RV32I-NEXT:    sb a1, 12(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
 ; RV32I-NEXT:    srli a5, a6, 24
+; RV32I-NEXT:    srli a7, a6, 16
+; RV32I-NEXT:    srli t1, a6, 8
+; RV32I-NEXT:    sb a6, 24(a2)
+; RV32I-NEXT:    sb t1, 25(a2)
+; RV32I-NEXT:    sb a7, 26(a2)
 ; RV32I-NEXT:    sb a5, 27(a2)
-; RV32I-NEXT:    srli a5, a6, 16
-; RV32I-NEXT:    sb a5, 26(a2)
-; RV32I-NEXT:    srli a5, a6, 8
-; RV32I-NEXT:    sb a5, 25(a2)
-; RV32I-NEXT:    srli a5, a4, 24
+; RV32I-NEXT:    srli a5, t2, 24
+; RV32I-NEXT:    srli a6, t2, 16
+; RV32I-NEXT:    srli a7, t2, 8
+; RV32I-NEXT:    sb t2, 28(a2)
+; RV32I-NEXT:    sb a7, 29(a2)
+; RV32I-NEXT:    sb a6, 30(a2)
 ; RV32I-NEXT:    sb a5, 31(a2)
-; RV32I-NEXT:    srli a5, a4, 16
-; RV32I-NEXT:    sb a5, 30(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 29(a2)
-; RV32I-NEXT:    srli a4, t0, 24
-; RV32I-NEXT:    sb a4, 19(a2)
-; RV32I-NEXT:    srli a4, t0, 16
-; RV32I-NEXT:    sb a4, 18(a2)
-; RV32I-NEXT:    srli a4, t0, 8
-; RV32I-NEXT:    sb a4, 17(a2)
-; RV32I-NEXT:    srli a4, t4, 24
-; RV32I-NEXT:    sb a4, 23(a2)
-; RV32I-NEXT:    srli a4, t4, 16
-; RV32I-NEXT:    sb a4, 22(a2)
-; RV32I-NEXT:    srli a4, t4, 8
-; RV32I-NEXT:    sb a4, 21(a2)
+; RV32I-NEXT:    srli a5, t0, 24
+; RV32I-NEXT:    srli a6, t0, 16
+; RV32I-NEXT:    srli a7, t0, 8
+; RV32I-NEXT:    sb t0, 16(a2)
+; RV32I-NEXT:    sb a7, 17(a2)
+; RV32I-NEXT:    sb a6, 18(a2)
+; RV32I-NEXT:    sb a5, 19(a2)
+; RV32I-NEXT:    srli a5, a4, 24
+; RV32I-NEXT:    srli a6, a4, 16
+; RV32I-NEXT:    srli a7, a4, 8
+; RV32I-NEXT:    sb a4, 20(a2)
+; RV32I-NEXT:    sb a7, 21(a2)
+; RV32I-NEXT:    sb a6, 22(a2)
+; RV32I-NEXT:    sb a5, 23(a2)
 ; RV32I-NEXT:    srli a4, a3, 24
+; RV32I-NEXT:    srli a5, a3, 16
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 8(a2)
+; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 10(a2)
 ; RV32I-NEXT:    sb a4, 11(a2)
-; RV32I-NEXT:    srli a4, a3, 16
-; RV32I-NEXT:    sb a4, 10(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 9(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 12(a2)
+; RV32I-NEXT:    sb a5, 13(a2)
+; RV32I-NEXT:    sb a4, 14(a2)
 ; RV32I-NEXT:    sb a3, 15(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 14(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 13(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
@@ -2257,99 +2257,99 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    or a1, a1, t0
 ; RV64I-NEXT:    or a1, a1, t1
 ; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    or a1, a1, a7
+; RV64I-NEXT:    or a7, a1, a7
 ; RV64I-NEXT:    sraiw a0, a0, 31
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    sd a0, 32(sp)
-; RV64I-NEXT:    sd a6, 24(sp)
-; RV64I-NEXT:    sd a5, 16(sp)
-; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    sd a0, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
-; RV64I-NEXT:    srli a0, a1, 3
+; RV64I-NEXT:    sd a4, 8(sp)
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a6, 24(sp)
+; RV64I-NEXT:    srli a0, a7, 3
 ; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    mv a3, sp
-; RV64I-NEXT:    add a0, a3, a0
-; RV64I-NEXT:    ld a3, 8(a0)
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 24(a0)
-; RV64I-NEXT:    srl a0, a3, a1
-; RV64I-NEXT:    andi a7, a1, 63
-; RV64I-NEXT:    xori a7, a7, 63
-; RV64I-NEXT:    slli t0, a5, 1
-; RV64I-NEXT:    sll t0, t0, a7
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    ld a1, 8(a0)
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    ld a4, 16(a0)
+; RV64I-NEXT:    ld a5, 24(a0)
+; RV64I-NEXT:    srl a0, a1, a7
+; RV64I-NEXT:    andi a6, a7, 63
+; RV64I-NEXT:    xori a6, a6, 63
+; RV64I-NEXT:    slli t0, a4, 1
+; RV64I-NEXT:    sll t0, t0, a6
 ; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    srl a4, a4, a1
-; RV64I-NEXT:    slli a3, a3, 1
-; RV64I-NEXT:    sll a3, a3, a7
-; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    srl a4, a5, a1
-; RV64I-NEXT:    slli a5, a6, 1
-; RV64I-NEXT:    sll a5, a5, a7
-; RV64I-NEXT:    or a4, a4, a5
-; RV64I-NEXT:    sra a1, a6, a1
-; RV64I-NEXT:    sb a1, 24(a2)
-; RV64I-NEXT:    srli a5, a1, 56
+; RV64I-NEXT:    srl a3, a3, a7
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    sll a1, a1, a6
+; RV64I-NEXT:    or a1, a3, a1
+; RV64I-NEXT:    srl a3, a4, a7
+; RV64I-NEXT:    slli a4, a5, 1
+; RV64I-NEXT:    sll a4, a4, a6
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    sra a4, a5, a7
+; RV64I-NEXT:    srli a5, a4, 56
+; RV64I-NEXT:    srli a6, a4, 48
+; RV64I-NEXT:    srli a7, a4, 40
+; RV64I-NEXT:    srli t0, a4, 32
+; RV64I-NEXT:    sb t0, 28(a2)
+; RV64I-NEXT:    sb a7, 29(a2)
+; RV64I-NEXT:    sb a6, 30(a2)
 ; RV64I-NEXT:    sb a5, 31(a2)
-; RV64I-NEXT:    srli a5, a1, 48
-; RV64I-NEXT:    sb a5, 30(a2)
-; RV64I-NEXT:    srli a5, a1, 40
-; RV64I-NEXT:    sb a5, 29(a2)
-; RV64I-NEXT:    srli a5, a1, 32
-; RV64I-NEXT:    sb a5, 28(a2)
-; RV64I-NEXT:    srli a5, a1, 24
+; RV64I-NEXT:    srli a5, a4, 24
+; RV64I-NEXT:    srli a6, a4, 16
+; RV64I-NEXT:    srli a7, a4, 8
+; RV64I-NEXT:    sb a4, 24(a2)
+; RV64I-NEXT:    sb a7, 25(a2)
+; RV64I-NEXT:    sb a6, 26(a2)
 ; RV64I-NEXT:    sb a5, 27(a2)
-; RV64I-NEXT:    srli a5, a1, 16
-; RV64I-NEXT:    sb a5, 26(a2)
-; RV64I-NEXT:    srli a1, a1, 8
-; RV64I-NEXT:    sb a1, 25(a2)
-; RV64I-NEXT:    sb a4, 16(a2)
-; RV64I-NEXT:    sb a3, 0(a2)
-; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    srli a1, a4, 56
-; RV64I-NEXT:    sb a1, 23(a2)
-; RV64I-NEXT:    srli a1, a4, 48
-; RV64I-NEXT:    sb a1, 22(a2)
-; RV64I-NEXT:    srli a1, a4, 40
-; RV64I-NEXT:    sb a1, 21(a2)
-; RV64I-NEXT:    srli a1, a4, 32
-; RV64I-NEXT:    sb a1, 20(a2)
-; RV64I-NEXT:    srli a1, a4, 24
-; RV64I-NEXT:    sb a1, 19(a2)
-; RV64I-NEXT:    srli a1, a4, 16
-; RV64I-NEXT:    sb a1, 18(a2)
-; RV64I-NEXT:    srli a4, a4, 8
-; RV64I-NEXT:    sb a4, 17(a2)
-; RV64I-NEXT:    srli a1, a3, 56
-; RV64I-NEXT:    sb a1, 7(a2)
-; RV64I-NEXT:    srli a1, a3, 48
-; RV64I-NEXT:    sb a1, 6(a2)
-; RV64I-NEXT:    srli a1, a3, 40
-; RV64I-NEXT:    sb a1, 5(a2)
-; RV64I-NEXT:    srli a1, a3, 32
-; RV64I-NEXT:    sb a1, 4(a2)
-; RV64I-NEXT:    srli a1, a3, 24
-; RV64I-NEXT:    sb a1, 3(a2)
-; RV64I-NEXT:    srli a1, a3, 16
-; RV64I-NEXT:    sb a1, 2(a2)
-; RV64I-NEXT:    srli a3, a3, 8
-; RV64I-NEXT:    sb a3, 1(a2)
+; RV64I-NEXT:    srli a4, a3, 56
+; RV64I-NEXT:    srli a5, a3, 48
+; RV64I-NEXT:    srli a6, a3, 40
+; RV64I-NEXT:    srli a7, a3, 32
+; RV64I-NEXT:    sb a7, 20(a2)
+; RV64I-NEXT:    sb a6, 21(a2)
+; RV64I-NEXT:    sb a5, 22(a2)
+; RV64I-NEXT:    sb a4, 23(a2)
+; RV64I-NEXT:    srli a4, a3, 24
+; RV64I-NEXT:    srli a5, a3, 16
+; RV64I-NEXT:    srli a6, a3, 8
+; RV64I-NEXT:    sb a3, 16(a2)
+; RV64I-NEXT:    sb a6, 17(a2)
+; RV64I-NEXT:    sb a5, 18(a2)
+; RV64I-NEXT:    sb a4, 19(a2)
+; RV64I-NEXT:    srli a3, a1, 56
+; RV64I-NEXT:    srli a4, a1, 48
+; RV64I-NEXT:    srli a5, a1, 40
+; RV64I-NEXT:    srli a6, a1, 32
+; RV64I-NEXT:    sb a6, 4(a2)
+; RV64I-NEXT:    sb a5, 5(a2)
+; RV64I-NEXT:    sb a4, 6(a2)
+; RV64I-NEXT:    sb a3, 7(a2)
+; RV64I-NEXT:    srli a3, a1, 24
+; RV64I-NEXT:    srli a4, a1, 16
+; RV64I-NEXT:    srli a5, a1, 8
+; RV64I-NEXT:    sb a1, 0(a2)
+; RV64I-NEXT:    sb a5, 1(a2)
+; RV64I-NEXT:    sb a4, 2(a2)
+; RV64I-NEXT:    sb a3, 3(a2)
 ; RV64I-NEXT:    srli a1, a0, 56
+; RV64I-NEXT:    srli a3, a0, 48
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    srli a5, a0, 32
+; RV64I-NEXT:    sb a5, 12(a2)
+; RV64I-NEXT:    sb a4, 13(a2)
+; RV64I-NEXT:    sb a3, 14(a2)
 ; RV64I-NEXT:    sb a1, 15(a2)
-; RV64I-NEXT:    srli a1, a0, 48
-; RV64I-NEXT:    sb a1, 14(a2)
-; RV64I-NEXT:    srli a1, a0, 40
-; RV64I-NEXT:    sb a1, 13(a2)
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    sb a1, 12(a2)
 ; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a4, a0, 8
+; RV64I-NEXT:    sb a0, 8(a2)
+; RV64I-NEXT:    sb a4, 9(a2)
+; RV64I-NEXT:    sb a3, 10(a2)
 ; RV64I-NEXT:    sb a1, 11(a2)
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    sb a1, 10(a2)
-; RV64I-NEXT:    srli a0, a0, 8
-; RV64I-NEXT:    sb a0, 9(a2)
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -2447,22 +2447,22 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    or a1, a1, a6
 ; RV32I-NEXT:    or a6, a1, t4
 ; RV32I-NEXT:    srai a0, a0, 31
-; RV32I-NEXT:    sw a0, 60(sp)
-; RV32I-NEXT:    sw a0, 56(sp)
-; RV32I-NEXT:    sw a0, 52(sp)
 ; RV32I-NEXT:    sw a0, 48(sp)
-; RV32I-NEXT:    sw a0, 44(sp)
-; RV32I-NEXT:    sw a0, 40(sp)
-; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 52(sp)
+; RV32I-NEXT:    sw a0, 56(sp)
+; RV32I-NEXT:    sw a0, 60(sp)
 ; RV32I-NEXT:    sw a0, 32(sp)
-; RV32I-NEXT:    sw t3, 28(sp)
-; RV32I-NEXT:    sw t2, 24(sp)
-; RV32I-NEXT:    sw t1, 20(sp)
+; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    sw a0, 44(sp)
 ; RV32I-NEXT:    sw t0, 16(sp)
-; RV32I-NEXT:    sw a7, 12(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw t1, 20(sp)
+; RV32I-NEXT:    sw t2, 24(sp)
+; RV32I-NEXT:    sw t3, 28(sp)
 ; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a7, 12(sp)
 ; RV32I-NEXT:    srli a0, a6, 3
 ; RV32I-NEXT:    andi a0, a0, 28
 ; RV32I-NEXT:    mv a1, sp
@@ -2506,62 +2506,62 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sll t0, t2, t0
 ; RV32I-NEXT:    or t0, t1, t0
 ; RV32I-NEXT:    sra a6, t4, a6
-; RV32I-NEXT:    sb a6, 28(a2)
 ; RV32I-NEXT:    srli t1, a6, 24
+; RV32I-NEXT:    srli t2, a6, 16
+; RV32I-NEXT:    srli t3, a6, 8
+; RV32I-NEXT:    sb a6, 28(a2)
+; RV32I-NEXT:    sb t3, 29(a2)
+; RV32I-NEXT:    sb t2, 30(a2)
 ; RV32I-NEXT:    sb t1, 31(a2)
-; RV32I-NEXT:    srli t1, a6, 16
-; RV32I-NEXT:    sb t1, 30(a2)
-; RV32I-NEXT:    srli a6, a6, 8
-; RV32I-NEXT:    sb a6, 29(a2)
-; RV32I-NEXT:    sb t0, 24(a2)
-; RV32I-NEXT:    sb a7, 16(a2)
-; RV32I-NEXT:    sb a5, 20(a2)
-; RV32I-NEXT:    sb a4, 8(a2)
-; RV32I-NEXT:    sb a3, 12(a2)
-; RV32I-NEXT:    sb a1, 0(a2)
-; RV32I-NEXT:    sb a0, 4(a2)
 ; RV32I-NEXT:    srli a6, t0, 24
+; RV32I-NEXT:    srli t1, t0, 16
+; RV32I-NEXT:    srli t2, t0, 8
+; RV32I-NEXT:    sb t0, 24(a2)
+; RV32I-NEXT:    sb t2, 25(a2)
+; RV32I-NEXT:    sb t1, 26(a2)
 ; RV32I-NEXT:    sb a6, 27(a2)
-; RV32I-NEXT:    srli a6, t0, 16
-; RV32I-NEXT:    sb a6, 26(a2)
-; RV32I-NEXT:    srli a6, t0, 8
-; RV32I-NEXT:    sb a6, 25(a2)
 ; RV32I-NEXT:    srli a6, a7, 24
+; RV32I-NEXT:    srli t0, a7, 16
+; RV32I-NEXT:    srli t1, a7, 8
+; RV32I-NEXT:    sb a7, 16(a2)
+; RV32I-NEXT:    sb t1, 17(a2)
+; RV32I-NEXT:    sb t0, 18(a2)
 ; RV32I-NEXT:    sb a6, 19(a2)
-; RV32I-NEXT:    srli a6, a7, 16
-; RV32I-NEXT:    sb a6, 18(a2)
-; RV32I-NEXT:    srli a6, a7, 8
-; RV32I-NEXT:    sb a6, 17(a2)
 ; RV32I-NEXT:    srli a6, a5, 24
+; RV32I-NEXT:    srli a7, a5, 16
+; RV32I-NEXT:    srli t0, a5, 8
+; RV32I-NEXT:    sb a5, 20(a2)
+; RV32I-NEXT:    sb t0, 21(a2)
+; RV32I-NEXT:    sb a7, 22(a2)
 ; RV32I-NEXT:    sb a6, 23(a2)
-; RV32I-NEXT:    srli a6, a5, 16
-; RV32I-NEXT:    sb a6, 22(a2)
-; RV32I-NEXT:    srli a5, a5, 8
-; RV32I-NEXT:    sb a5, 21(a2)
 ; RV32I-NEXT:    srli a5, a4, 24
+; RV32I-NEXT:    srli a6, a4, 16
+; RV32I-NEXT:    srli a7, a4, 8
+; RV32I-NEXT:    sb a4, 8(a2)
+; RV32I-NEXT:    sb a7, 9(a2)
+; RV32I-NEXT:    sb a6, 10(a2)
 ; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a5, a4, 16
-; RV32I-NEXT:    sb a5, 10(a2)
-; RV32I-NEXT:    srli a4, a4, 8
-; RV32I-NEXT:    sb a4, 9(a2)
 ; RV32I-NEXT:    srli a4, a3, 24
+; RV32I-NEXT:    srli a5, a3, 16
+; RV32I-NEXT:    srli a6, a3, 8
+; RV32I-NEXT:    sb a3, 12(a2)
+; RV32I-NEXT:    sb a6, 13(a2)
+; RV32I-NEXT:    sb a5, 14(a2)
 ; RV32I-NEXT:    sb a4, 15(a2)
-; RV32I-NEXT:    srli a4, a3, 16
-; RV32I-NEXT:    sb a4, 14(a2)
-; RV32I-NEXT:    srli a3, a3, 8
-; RV32I-NEXT:    sb a3, 13(a2)
 ; RV32I-NEXT:    srli a3, a1, 24
+; RV32I-NEXT:    srli a4, a1, 16
+; RV32I-NEXT:    srli a5, a1, 8
+; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a5, 1(a2)
+; RV32I-NEXT:    sb a4, 2(a2)
 ; RV32I-NEXT:    sb a3, 3(a2)
-; RV32I-NEXT:    srli a3, a1, 16
-; RV32I-NEXT:    sb a3, 2(a2)
-; RV32I-NEXT:    srli a1, a1, 8
-; RV32I-NEXT:    sb a1, 1(a2)
 ; RV32I-NEXT:    srli a1, a0, 24
+; RV32I-NEXT:    srli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 8
+; RV32I-NEXT:    sb a0, 4(a2)
+; RV32I-NEXT:    sb a4, 5(a2)
+; RV32I-NEXT:    sb a3, 6(a2)
 ; RV32I-NEXT:    sb a1, 7(a2)
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    sb a1, 6(a2)
-; RV32I-NEXT:    srli a0, a0, 8
-; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
   %src = load i256, ptr %src.ptr, align 1
diff --git a/llvm/test/CodeGen/RISCV/xtheadmempair.ll b/llvm/test/CodeGen/RISCV/xtheadmempair.ll
index 3a74bb66d9ec252e03b10652263083545ddb0ccd..3525c40026064a44ea61d6b7595e91a129a1f678 100644
--- a/llvm/test/CodeGen/RISCV/xtheadmempair.ll
+++ b/llvm/test/CodeGen/RISCV/xtheadmempair.ll
@@ -174,10 +174,10 @@ define void @swd(ptr %a, i32 %b, i32%c) {
 define void @sdd(ptr %a, i64 %b, i64%c) {
 ; RV32XTHEADMEMPAIR-LABEL: sdd:
 ; RV32XTHEADMEMPAIR:       # %bb.0:
-; RV32XTHEADMEMPAIR-NEXT:    sw a2, 36(a0)
 ; RV32XTHEADMEMPAIR-NEXT:    sw a1, 32(a0)
-; RV32XTHEADMEMPAIR-NEXT:    sw a4, 44(a0)
+; RV32XTHEADMEMPAIR-NEXT:    sw a2, 36(a0)
 ; RV32XTHEADMEMPAIR-NEXT:    sw a3, 40(a0)
+; RV32XTHEADMEMPAIR-NEXT:    sw a4, 44(a0)
 ; RV32XTHEADMEMPAIR-NEXT:    ret
 ;
 ; RV64XTHEADMEMPAIR-LABEL: sdd:
diff --git a/llvm/test/CodeGen/VE/Scalar/cast.ll b/llvm/test/CodeGen/VE/Scalar/cast.ll
index 44782b342f4d0f18600852e486637f3d078196eb..6f6c93a1e639fb186d121dbcc4db40564a6432f3 100644
--- a/llvm/test/CodeGen/VE/Scalar/cast.ll
+++ b/llvm/test/CodeGen/VE/Scalar/cast.ll
@@ -568,6 +568,44 @@ define float @ull2f(i64 %x) {
   ret float %r
 }
 
+define float @ull2f_nneg(i64 %x) {
+; CHECK-LABEL: ull2f_nneg:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cvt.d.l %s0, %s0
+; CHECK-NEXT:    cvt.s.d %s0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %r = uitofp nneg i64 %x to float
+  ret float %r
+}
+
+define float @ull2f_strict(i32 %x) {
+; CHECK-LABEL: ull2f_strict:
+; CHECK:     # %bb.0:
+; CHECK-NEXT:	adds.l %s11, -16, %s11
+; CHECK-NEXT:		brge.l.t %s11, %s8, .LBB58_2
+; CHECK-NEXT:	# %bb.1:
+; CHECK-NEXT:		ld %s61, 24(, %s14)
+; CHECK-NEXT:		or %s62, 0, %s0
+; CHECK-NEXT:		lea %s63, 315
+; CHECK-NEXT:		shm.l %s63, (%s61)
+; CHECK-NEXT:		shm.l %s8, 8(%s61)
+; CHECK-NEXT:		shm.l %s11, 16(%s61)
+; CHECK-NEXT:		monc
+; CHECK-NEXT:		or %s0, 0, %s62
+; CHECK-NEXT:	.LBB58_2:
+; CHECK-NEXT:		lea %s1, 1127219200
+; CHECK-NEXT:		stl %s1, 12(, %s11)
+; CHECK-NEXT:		stl %s0, 8(, %s11)
+; CHECK-NEXT:		ld %s0, 8(, %s11)
+; CHECK-NEXT:		lea.sl %s1, 1127219200
+; CHECK-NEXT:		fsub.d %s0, %s0, %s1
+; CHECK-NEXT:		cvt.s.d %s0, %s0
+; CHECK-NEXT:		adds.l %s11, 16, %s11
+; CHECK-NEXT:		b.l.t (, %s10)
+  %val = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %val
+}
+
 define double @ull2d(i64 %x) {
 ; CHECK-LABEL: ull2d:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/WebAssembly/profile.ll b/llvm/test/CodeGen/WebAssembly/profile.ll
new file mode 100644
index 0000000000000000000000000000000000000000..27802cc3bf35673168aeb4750988a9b66d8c8a66
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/profile.ll
@@ -0,0 +1,47 @@
+; RUN: llc < %s --filetype=obj | obj2yaml | FileCheck %s
+
+target triple = "wasm32-unknown-unknown"
+
+$__covrec_A = comdat any
+$__covrec_B = comdat any
+
+@__covrec_A = linkonce_odr hidden constant <{ i64, i32, i64, i64, [4 x i8] }> <{
+    i64 -1978722966671112904,
+    i32 4,
+    i64 0,
+    i64 -8102528905418564625,
+    [4 x i8] c"\01\01\04\11"
+}>, section "__llvm_covfun", comdat, align 8
+@__covrec_B = linkonce_odr hidden constant <{ i64, i32, i64, i64, [4 x i8] }> <{
+    i64 8006510647218728891,
+    i32 9,
+    i64 0,
+    i64 -8102528905418564625,
+    [4 x i8] c"\01\01\00\01"
+}>, section "__llvm_covfun", comdat, align 8
+@__llvm_coverage_mapping = private constant { { i32, i32, i32, i32 }, [4 x i8] } {
+    { i32, i32, i32, i32 } { i32 0, i32 87, i32 0, i32 5 },
+    [4 x i8] c"\01\01\00\02"
+}, section "__llvm_covmap", align 8
+
+; CHECK:      - Type:            CUSTOM
+; CHECK-NEXT:   Name:            __llvm_covfun
+; CHECK-NEXT:   Payload:         3845A90EF2298AE4040000000000000000000000EF1B31BAE3088E8F01010411
+; CHECK-NEXT: - Type:            CUSTOM
+; CHECK-NEXT:   Name:            __llvm_covfun
+; CHECK-NEXT:   Payload:         BBEFDA6903D71C6F090000000000000000000000EF1B31BAE3088E8F01010001
+; CHECK-NEXT: - Type:            CUSTOM
+; CHECK-NEXT:   Name:            __llvm_covmap
+; CHECK-NEXT:   Payload:         '0000000057000000000000000500000001010002'
+; CHECK-NEXT: - Type:            CUSTOM
+; CHECK-NEXT:   Name:            linking
+; CHECK-NEXT:   Version:         2
+; CHECK-NEXT:   Comdats:
+; CHECK-NEXT:     - Name:            __covrec_A
+; CHECK-NEXT:       Entries:
+; CHECK-NEXT:         - Kind:            SECTION
+; CHECK-NEXT:           Index:           1
+; CHECK-NEXT:     - Name:            __covrec_B
+; CHECK-NEXT:       Entries:
+; CHECK-NEXT:         - Kind:            SECTION
+; CHECK-NEXT:           Index:           2
diff --git a/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll b/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
index e9bb68e711f8890a5f90cae386af32ce9989d3f7..503b39919b5502d2e7e7b064a6128c1d7c82a2de 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
@@ -45,7 +45,7 @@ define dso_local void @test1(ptr%buf) nounwind {
 ; CHECK-NEXT:    tileloadd (%rbx,%r15), %tmm0
 ; CHECK-NEXT:    tileloadd (%rbx,%r15), %tmm1
 ; CHECK-NEXT:    tilestored %tmm3, 1024(%rsp,%rax) # 1024-byte Folded Spill
-; CHECK-NEXT:    tileloadd {{[-0-9]+}}(%r{{[sb]}}p), %tmm2 # 1024-byte Folded Reload
+; CHECK-NEXT:    tileloadd 1024(%rsp,%rax), %tmm2 # 1024-byte Folded Reload
 ; CHECK-NEXT:    tdpbssd %tmm1, %tmm0, %tmm2
 ; CHECK-NEXT:    tilestored %tmm2, (%rbx,%r15)
 ; CHECK-NEXT:    incl %r14d
@@ -109,8 +109,8 @@ define dso_local void @test1(ptr%buf) nounwind {
 ; EGPR-NEXT:    tileloadd (%rbx,%r15), %tmm1 # EVEX TO VEX Compression encoding: [0xc4,0xa2,0x7b,0x4b,0x0c,0x3b]
 ; EGPR-NEXT:    tilestored %tmm3, 1024(%rsp,%rax) # 1024-byte Folded Spill
 ; EGPR-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7a,0x4b,0x9c,0x04,0x00,0x04,0x00,0x00]
-; EGPR-NEXT:    tileloadd {{[-0-9]+}}(%r{{[sb]}}p), %tmm2 # 1024-byte Folded Reload
-; EGPR-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7b,0x4b,0x94,0x24,0x00,0x04,0x00,0x00]
+; EGPR-NEXT:    tileloadd 1024(%rsp,%rax), %tmm2 # 1024-byte Folded Reload
+; EGPR-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7b,0x4b,0x94,0x04,0x00,0x04,0x00,0x00]
 ; EGPR-NEXT:    tdpbssd %tmm1, %tmm0, %tmm2 # encoding: [0xc4,0xe2,0x73,0x5e,0xd0]
 ; EGPR-NEXT:    tilestored %tmm2, (%rbx,%r15) # EVEX TO VEX Compression encoding: [0xc4,0xa2,0x7a,0x4b,0x14,0x3b]
 ; EGPR-NEXT:    incl %r14d # encoding: [0x41,0xff,0xc6]
diff --git a/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll b/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
index ec087f23248c4cbb1a7604e5966f529117f03c1b..6ef7219cfebdb55b1e4d67efc365826953068517 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
@@ -148,7 +148,7 @@ define void @PR90954(ptr %0, ptr %1, i32 %2) nounwind {
 ; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-NEXT:    movabsq $64, %rax
 ; CHECK-NEXT:    tilestored %tmm0, 3072(%rsp,%rax) # 1024-byte Folded Spill
-; CHECK-NEXT:    tileloadd {{[-0-9]+}}(%r{{[sb]}}p), %tmm1 # 1024-byte Folded Reload
+; CHECK-NEXT:    tileloadd 3072(%rsp,%rax), %tmm1 # 1024-byte Folded Reload
 ; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; CHECK-NEXT:    jmp .LBB1_4
   %4 = shl i32 %2, 4
@@ -212,7 +212,7 @@ define void @multi_use() nounwind {
 ; CHECK-NEXT:    tilezero %tmm0
 ; CHECK-NEXT:    movabsq $64, %rbp
 ; CHECK-NEXT:    tilestored %tmm0, 896(%rsp,%rbp) # 1024-byte Folded Spill
-; CHECK-NEXT:    tileloadd {{[-0-9]+}}(%r{{[sb]}}p), %tmm1 # 1024-byte Folded Reload
+; CHECK-NEXT:    tileloadd 896(%rsp,%rbp), %tmm1 # 1024-byte Folded Reload
 ; CHECK-NEXT:    tdpbf16ps %tmm0, %tmm0, %tmm1
 ; CHECK-NEXT:    tdpbf16ps %tmm0, %tmm0, %tmm0
 ; CHECK-NEXT:    addq $2928, %rsp # imm = 0xB70
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir
index 3b8455684f33d2d079d668652603a4316b64881e..881ceac1d1f7f2e6f0a5779cdb766e9e0a71e3e3 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir
@@ -26,9 +26,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -56,9 +56,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -86,9 +86,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir
index 4965b069715a117c80d5780d10d584c2ce4ddd7f..c2800bef9713de894d322638d1e867542ea0a05a 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir
@@ -26,9 +26,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -56,9 +56,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -86,9 +86,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir
index 77a94581b66fde3c09ee7e8bff5872ef64f0b4e9..e45818af22a35646b56b83352c4b346291bf7383 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir
@@ -28,9 +28,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -58,9 +58,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -88,9 +88,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
index 1d280e9e4bd11f2ee1c9b1d53002d6fea3fc84ae..28c4eaea38841403b1333557bb2dfaf07e12cfe8 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
@@ -33,8 +33,8 @@ selected:        false
 tracksRegLiveness: true
 # CHECK-LABEL: name:            test_mul_vec256
 # CHECK: registers:
-# CHECK:  - { id: 0, class: vecr, preferred-register: '' }
-# CHECK:  - { id: 1, class: vecr, preferred-register: '' }
+# CHECK:  - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK:  - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -56,8 +56,8 @@ selected:        false
 tracksRegLiveness: true
 # CHECK-LABEL: name:            test_add_vec256
 # CHECK: registers:
-# CHECK:  - { id: 0, class: vecr, preferred-register: '' }
-# CHECK:  - { id: 1, class: vecr, preferred-register: '' }
+# CHECK:  - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK:  - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -79,8 +79,8 @@ selected:        false
 tracksRegLiveness: true
 # CHECK-LABEL: name:            test_sub_vec256
 # CHECK: registers:
-# CHECK:  - { id: 0, class: vecr, preferred-register: '' }
-# CHECK:  - { id: 1, class: vecr, preferred-register: '' }
+# CHECK:  - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK:  - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -100,8 +100,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -122,8 +122,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
index 2f8827c7ff9066b35a8bf7112b7d16e4f7443c89..4a19a040b0ce8d0acad5a749dd46bcb31e008d35 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
@@ -33,8 +33,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -53,8 +53,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -73,8 +73,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -93,8 +93,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -115,8 +115,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X32.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X32.mir
index c69345ccf5a2ade3a01b3b6f34bad57cd7f8fb86..8eac3eaf361459ee693edda494f9c7cf36d6451d 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X32.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X32.mir
@@ -14,11 +14,11 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 4, class: gpr, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 2, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 3, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 4, class: gpr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir b/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir
index 4ba8606df5dfc694d86d86a215276373bb6186cc..43c4105c883e1f949e04169437986efbae2c330f 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir
@@ -25,12 +25,12 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X32:                 registers:
-# X32-NEXT:              - { id: 0, class: gr32, preferred-register: '' }
-# X32-NEXT:              - { id: 1, class: gr32, preferred-register: '' }
+# X32-NEXT:              - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# X32-NEXT:              - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 #
 # X32ABI:              registers:
-# X32ABI-NEXT:           - { id: 0, class: low32_addr_access, preferred-register: '' }
-# X32ABI-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
+# X32ABI-NEXT:           - { id: 0, class: low32_addr_access, preferred-register: '', flags: [   ] }
+# X32ABI-NEXT:           - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -60,8 +60,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X32ALL:              registers:
-# X32ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '' }
-# X32ALL-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
+# X32ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# X32ALL-NEXT:           - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir b/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir
index 4a1f63c9879552adaf5da94abd3d91892557c99a..d292bbfcaa98becde54d223abdc63e878460cb62 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir
@@ -25,8 +25,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X64ALL:              registers:
-# X64ALL-NEXT:           - { id: 0, class: gr64, preferred-register: '' }
-# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '' }
+# X64ALL-NEXT:           - { id: 0, class: gr64, preferred-register: '', flags: [   ] }
+# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
@@ -58,8 +58,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X64ALL:              registers:
-# X64ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '' }
-# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '' }
+# X64ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-add-v128.mir b/llvm/test/CodeGen/X86/GlobalISel/select-add-v128.mir
index 987f67bad15b3289c75d051606c974da40d2beda..32898be2e6f5e00bd4fe0882bda131c2c280be81 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-add-v128.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-add-v128.mir
@@ -32,19 +32,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -74,19 +74,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -116,19 +116,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -158,19 +158,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-add-v256.mir b/llvm/test/CodeGen/X86/GlobalISel/select-add-v256.mir
index 3ee959294413dd542743e2700431d06fb84637e9..742cc4378e3ebd77aa7c331c3710ac731816f017 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-add-v256.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-add-v256.mir
@@ -30,19 +30,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # AVX2:            registers:
-# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '' }
+# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr256, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr256, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr256, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr256, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '', flags: [    ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '', flags: [    ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '', flags: [    ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -70,19 +70,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # AVX2:            registers:
-# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '' }
+# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr256, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr256, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr256, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr256, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -110,19 +110,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # AVX2:            registers:
-# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '' }
+# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr256x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr256x, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr256x, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -150,19 +150,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # AVX2:            registers:
-# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '' }
-# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '' }
+# AVX2-NEXT:         - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
+# AVX2-NEXT:         - { id: 2, class: vr256, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr256x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr256x, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr256x, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr256x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir b/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir
index 72a6ed15f63b5b0d573b5e5542a1fa3fbee11211..41e1b5bf22bf1d77d795ca5759e8dd4e7975370a 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir
@@ -35,8 +35,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: gr8, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: gr32, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: gr8, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -61,8 +61,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: gr8, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: gr32, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: gr8, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -87,10 +87,10 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: gr16[[ABCD:(_abcd)?]], preferred-register: '' }
-# X32-NEXT:   - { id: 1, class: gr8_abcd_l, preferred-register: '' }
-# X64-NEXT:   - { id: 1, class: gr8, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: gr32, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: gr16[[ABCD:(_abcd)?]], preferred-register: '', flags: [   ] }
+# X32-NEXT:   - { id: 1, class: gr8_abcd_l, preferred-register: '', flags: [   ] }
+# X64-NEXT:   - { id: 1, class: gr8, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -120,9 +120,9 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: gr32, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: gr16, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: gr32, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: gr16, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -150,10 +150,10 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: gr32[[ABCD:(_abcd)?]], preferred-register: '' }
-# X32-NEXT:   - { id: 1, class: gr8_abcd_l, preferred-register: '' }
-# X64-NEXT:   - { id: 1, class: gr8, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: gr32, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: gr32[[ABCD:(_abcd)?]], preferred-register: '', flags: [   ] }
+# X32-NEXT:   - { id: 1, class: gr8_abcd_l, preferred-register: '', flags: [   ] }
+# X64-NEXT:   - { id: 1, class: gr8, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -183,10 +183,10 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: gr32, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: gr16, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: low32_addr_access_rbp, preferred-register: '' }
-# ALL-NEXT:   - { id: 3, class: low32_addr_access_rbp, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: gr16, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: low32_addr_access_rbp, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 3, class: low32_addr_access_rbp, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec256.mir b/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec256.mir
index 73af03b34ec77c0aa44544b60fc111e59d0bab35..301d63b7f3643f013df2fb9f211b0917900e6d0b 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec256.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec256.mir
@@ -18,12 +18,12 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # AVX:           registers:
-# AVX-NEXT:        - { id: 0, class: vr256, preferred-register: '' }
-# AVX-NEXT:        - { id: 1, class: vr128, preferred-register: '' }
+# AVX-NEXT:        - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX-NEXT:        - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:      registers:
-# AVX512VL-NEXT:   - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512VL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
+# AVX512VL-NEXT:   - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -50,12 +50,12 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # AVX:           registers:
-# AVX-NEXT:        - { id: 0, class: vr256, preferred-register: '' }
-# AVX-NEXT:        - { id: 1, class: vr128, preferred-register: '' }
+# AVX-NEXT:        - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# AVX-NEXT:        - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:      registers:
-# AVX512VL-NEXT:   - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512VL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
+# AVX512VL-NEXT:   - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec512.mir b/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec512.mir
index 5ddf58e6455576055327d17928d80e289500f094..cff8560a4ba45f013cc503d0a7dbc88422f1c52b 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec512.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-extract-vec512.mir
@@ -27,8 +27,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -53,8 +53,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -79,8 +79,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: vr256x, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -105,8 +105,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: vr256x, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: vr512, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-inc.mir b/llvm/test/CodeGen/X86/GlobalISel/select-inc.mir
index 45e2b47176b975e2678501a1c0b9ee512d3c5720..b834155d49f64c00749a3df18007548c928ab39e 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-inc.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-inc.mir
@@ -13,10 +13,10 @@ name:            test_add_i8
 legalized:       true
 regBankSelected: true
 # ALL:      registers:
-# ALL-NEXT:  - { id: 0, class: gr8, preferred-register: '' }
-# INC-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
-# ADD-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
-# ALL-NEXT:  - { id: 2, class: gr8, preferred-register: '' }
+# ALL-NEXT:  - { id: 0, class: gr8, preferred-register: '', flags: [   ] }
+# INC-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
+# ADD-NEXT:  - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
+# ALL-NEXT:  - { id: 2, class: gr8, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr }
   - { id: 1, class: gpr }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
index f24a82899c62e841b419cb1d05708770a5a7cc74..86a83c417d63ee792a22c24c58ca03197fe04162 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
@@ -33,12 +33,12 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NO_AVX512F:       registers:
-# NO_AVX512F-NEXT:    - { id: 0, class: gr64, preferred-register: '' }
-# NO_AVX512F-NEXT:    - { id: 1, class: vr256, preferred-register: '' }
+# NO_AVX512F-NEXT:    - { id: 0, class: gr64, preferred-register: '', flags: [   ] }
+# NO_AVX512F-NEXT:    - { id: 1, class: vr256, preferred-register: '', flags: [   ] }
 #
 # AVX512ALL:        registers:
-# AVX512ALL-NEXT:     - { id: 0, class: gr64, preferred-register: '' }
-# AVX512ALL-NEXT:     - { id: 1, class: vr256x, preferred-register: '' }
+# AVX512ALL-NEXT:     - { id: 0, class: gr64, preferred-register: '', flags: [   ] }
+# AVX512ALL-NEXT:     - { id: 1, class: vr256x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr }
   - { id: 1, class: vecr }
@@ -106,12 +106,12 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NO_AVX512F:       registers:
-# NO_AVX512F-NEXT:    - { id: 0, class: vr256, preferred-register: '' }
-# NO_AVX512F-NEXT:    - { id: 1, class: gr64, preferred-register: '' }
+# NO_AVX512F-NEXT:    - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# NO_AVX512F-NEXT:    - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 # AVX512ALL:        registers:
-# AVX512ALL-NEXT:     - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512ALL-NEXT:     - { id: 1, class: gr64, preferred-register: '' }
+# AVX512ALL-NEXT:     - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512ALL-NEXT:     - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: gpr }
@@ -146,12 +146,12 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NO_AVX512F:       registers:
-# NO_AVX512F-NEXT:    - { id: 0, class: vr256, preferred-register: '' }
-# NO_AVX512F-NEXT:    - { id: 1, class: gr64, preferred-register: '' }
+# NO_AVX512F-NEXT:    - { id: 0, class: vr256, preferred-register: '', flags: [   ] }
+# NO_AVX512F-NEXT:    - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 # AVX512ALL:        registers:
-# AVX512ALL-NEXT:     - { id: 0, class: vr256x, preferred-register: '' }
-# AVX512ALL-NEXT:     - { id: 1, class: gr64, preferred-register: '' }
+# AVX512ALL-NEXT:     - { id: 0, class: vr256x, preferred-register: '', flags: [   ] }
+# AVX512ALL-NEXT:     - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: gpr }
diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-GV.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-GV.mir
index 61f9eb9a72874011e93ee14b2b14a4535ea97dfc..50f6fbd59cd9997966f9c404e5c7819186fdc658 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-GV.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-GV.mir
@@ -15,7 +15,7 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _, preferred-register: '' }
 # CHECK:          %0:_(p0) = G_GLOBAL_VALUE @g_int
diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-GV.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-GV.mir
index a2cf55dc2ba5434166a8aaa283c8535a7b3b5e99..e7c5d9b3679419802a90ed1651a0857c219ce25c 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-GV.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-GV.mir
@@ -15,7 +15,7 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _, preferred-register: '' }
 # CHECK:          %0:_(p0) = G_GLOBAL_VALUE @g_int
diff --git a/llvm/test/CodeGen/X86/PR37310.mir b/llvm/test/CodeGen/X86/PR37310.mir
index fa0368a1b5b3dbf36f6d531d919d686cdd8946c7..6c68f79661bec505dac468a9131cdc1a7a5c4077 100644
--- a/llvm/test/CodeGen/X86/PR37310.mir
+++ b/llvm/test/CodeGen/X86/PR37310.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -no-stack-coloring=false -run-pass stack-coloring -o - %s
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -no-stack-coloring=false -passes stack-coloring -o - %s
 
 # Test to insure that the liveness analysis in the StackColoring
 # pass gracefully handles statically unreachable blocks. See PR 37310.
diff --git a/llvm/test/CodeGen/X86/StackColoring-dbg-invariance.mir b/llvm/test/CodeGen/X86/StackColoring-dbg-invariance.mir
index 9391430cf3800594e1bc52c439e501e277deac6d..348a2901ff6a46fd2c38b0e00c5b9e3a0eccd7de 100644
--- a/llvm/test/CodeGen/X86/StackColoring-dbg-invariance.mir
+++ b/llvm/test/CodeGen/X86/StackColoring-dbg-invariance.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mcpu=corei7 -no-stack-coloring=false -run-pass=stack-coloring -o - %s | FileCheck %s
+# RUN: llc -mcpu=corei7 -no-stack-coloring=false -passes=stack-coloring -o - %s | FileCheck %s
 
 # Difference between test_1 and test_2 is that there is a DBG_VALUE in test_1.
 # If transformation is debug invariant the resulting LEA instruction should be
diff --git a/llvm/test/CodeGen/X86/avx512-cvt.ll b/llvm/test/CodeGen/X86/avx512-cvt.ll
index ff7f7b39c6c8b12de65db1b3ae315d397fe8b8e6..a78d97782e6a3d9f2e6d955f4ddf196db2c38bc4 100644
--- a/llvm/test/CodeGen/X86/avx512-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512-cvt.ll
@@ -312,11 +312,46 @@ define <4 x float> @ulto4f32(<4 x i64> %a) {
   ret <4 x float> %b
 }
 
+define <4 x float> @ulto4f32_nneg(<4 x i64> %a) {
+; NODQ-LABEL: ulto4f32_nneg:
+; NODQ:       # %bb.0:
+; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm1, %xmm1
+; NODQ-NEXT:    vmovq %xmm0, %rax
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm2, %xmm2
+; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3]
+; NODQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; NODQ-NEXT:    vmovq %xmm0, %rax
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm2
+; NODQ-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; NODQ-NEXT:    vpextrq $1, %xmm0, %rax
+; NODQ-NEXT:    vcvtsi2ss %rax, %xmm3, %xmm0
+; NODQ-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; NODQ-NEXT:    vzeroupper
+; NODQ-NEXT:    retq
+;
+; VLDQ-LABEL: ulto4f32_nneg:
+; VLDQ:       # %bb.0:
+; VLDQ-NEXT:    vcvtqq2ps %ymm0, %xmm0
+; VLDQ-NEXT:    vzeroupper
+; VLDQ-NEXT:    retq
+;
+; DQNOVL-LABEL: ulto4f32_nneg:
+; DQNOVL:       # %bb.0:
+; DQNOVL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; DQNOVL-NEXT:    vcvtqq2ps %zmm0, %ymm0
+; DQNOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; DQNOVL-NEXT:    vzeroupper
+; DQNOVL-NEXT:    retq
+  %b = uitofp nneg <4 x i64> %a to <4 x float>
+  ret <4 x float> %b
+}
+
 define <8 x double> @ulto8f64(<8 x i64> %a) {
 ; NODQ-LABEL: ulto8f64:
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; NODQ-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1
+; NODQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)
 ; NODQ-NEXT:    vpsrlq $32, %zmm0, %zmm0
 ; NODQ-NEXT:    vporq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
 ; NODQ-NEXT:    vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
@@ -342,14 +377,14 @@ define <16 x double> @ulto16f64(<16 x i64> %a) {
 ; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
 ; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
 ; NODQ-NEXT:    vmovdqa64 %zmm3, %zmm4
-; NODQ-NEXT:    vpternlogq $248, %zmm2, %zmm0, %zmm4
+; NODQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm0 & zmm2)
 ; NODQ-NEXT:    vpsrlq $32, %zmm0, %zmm0
 ; NODQ-NEXT:    vpbroadcastq {{.*#+}} zmm5 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
 ; NODQ-NEXT:    vporq %zmm5, %zmm0, %zmm0
 ; NODQ-NEXT:    vbroadcastsd {{.*#+}} zmm6 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
 ; NODQ-NEXT:    vsubpd %zmm6, %zmm0, %zmm0
 ; NODQ-NEXT:    vaddpd %zmm0, %zmm4, %zmm0
-; NODQ-NEXT:    vpternlogq $248, %zmm2, %zmm1, %zmm3
+; NODQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm1 & zmm2)
 ; NODQ-NEXT:    vpsrlq $32, %zmm1, %zmm1
 ; NODQ-NEXT:    vporq %zmm5, %zmm1, %zmm1
 ; NODQ-NEXT:    vsubpd %zmm6, %zmm1, %zmm1
@@ -1483,7 +1518,7 @@ define <16 x float> @sbto16f32(<16 x i32> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; NODQ-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
-; NODQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; NODQ-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; NODQ-NEXT:    vcvtdq2ps %zmm0, %zmm0
 ; NODQ-NEXT:    retq
 ;
@@ -1564,7 +1599,7 @@ define <16 x double> @sbto16f64(<16 x double> %a) {
 ; NODQ-NEXT:    vcmpltpd %zmm0, %zmm2, %k0
 ; NODQ-NEXT:    vcmpltpd %zmm1, %zmm2, %k1
 ; NODQ-NEXT:    kunpckbw %k0, %k1, %k1
-; NODQ-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; NODQ-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; NODQ-NEXT:    vcvtdq2pd %ymm1, %zmm0
 ; NODQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; NODQ-NEXT:    vcvtdq2pd %ymm1, %zmm1
@@ -1603,7 +1638,7 @@ define <8 x double> @sbto8f64(<8 x double> %a) {
 ; NOVLDQ:       # %bb.0:
 ; NOVLDQ-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; NOVLDQ-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
-; NOVLDQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; NOVLDQ-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; NOVLDQ-NEXT:    vcvtdq2pd %ymm0, %zmm0
 ; NOVLDQ-NEXT:    retq
 ;
@@ -1864,7 +1899,7 @@ define <16 x float> @ubto16f32(<16 x i32> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; NODQ-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
-; NODQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; NODQ-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; NODQ-NEXT:    vpsrld $31, %zmm0, %zmm0
 ; NODQ-NEXT:    vcvtdq2ps %zmm0, %zmm0
 ; NODQ-NEXT:    retq
@@ -1894,7 +1929,7 @@ define <16 x double> @ubto16f64(<16 x i32> %a) {
 ; NODQ:       # %bb.0:
 ; NODQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; NODQ-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
-; NODQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; NODQ-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; NODQ-NEXT:    vpsrld $31, %zmm0, %zmm1
 ; NODQ-NEXT:    vcvtdq2pd %ymm1, %zmm0
 ; NODQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
diff --git a/llvm/test/CodeGen/X86/ispow2.ll b/llvm/test/CodeGen/X86/ispow2.ll
index 8723432de8b6b0032b651c8a451c4947271dbf55..649d257b28d762c04eb89f69ebc86fafae4777f0 100644
--- a/llvm/test/CodeGen/X86/ispow2.ll
+++ b/llvm/test/CodeGen/X86/ispow2.ll
@@ -102,7 +102,7 @@ define <4 x i1> @is_pow2_non_zero_4xv64(<4 x i64> %xin) {
 ; CHECK-AVX512:       # %bb.0:
 ; CHECK-AVX512-NEXT:    vporq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; CHECK-AVX512-NEXT:    vpopcntq %ymm0, %ymm0
-; CHECK-AVX512-NEXT:    vpcmpeqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %k1
+; CHECK-AVX512-NEXT:    vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %k1
 ; CHECK-AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-AVX512-NEXT:    vzeroupper
@@ -155,7 +155,7 @@ define <4 x i1> @neither_pow2_non_zero_4xv64(<4 x i64> %xin) {
 ; CHECK-AVX512:       # %bb.0:
 ; CHECK-AVX512-NEXT:    vporq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; CHECK-AVX512-NEXT:    vpopcntq %ymm0, %ymm0
-; CHECK-AVX512-NEXT:    vpcmpneqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %k1
+; CHECK-AVX512-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %k1
 ; CHECK-AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-AVX512-NEXT:    vzeroupper
@@ -220,3 +220,44 @@ define <4 x i1> @neither_pow2_non_zero_4xv64_x_maybe_z(<4 x i64> %x) {
   %r = icmp ne <4 x i64> %cnt, <i64 1, i64 1, i64 1, i64 1>
   ret <4 x i1> %r
 }
+
+
+define i1 @ctpop32_eq_one_nonzero(i32 %x) {
+; CHECK-NOBMI-LABEL: ctpop32_eq_one_nonzero:
+; CHECK-NOBMI:       # %bb.0: # %entry
+; CHECK-NOBMI-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NOBMI-NEXT:    leal -1(%rdi), %eax
+; CHECK-NOBMI-NEXT:    testl %eax, %edi
+; CHECK-NOBMI-NEXT:    sete %al
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-LABEL: ctpop32_eq_one_nonzero:
+; CHECK-BMI2:       # %bb.0: # %entry
+; CHECK-BMI2-NEXT:    blsrl %edi, %eax
+; CHECK-BMI2-NEXT:    sete %al
+; CHECK-BMI2-NEXT:    retq
+entry:
+  %popcnt = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp eq i32 %popcnt, 1
+  ret i1 %cmp
+}
+
+define i1 @ctpop32_ne_one_nonzero(i32 %x) {
+; CHECK-NOBMI-LABEL: ctpop32_ne_one_nonzero:
+; CHECK-NOBMI:       # %bb.0: # %entry
+; CHECK-NOBMI-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NOBMI-NEXT:    leal -1(%rdi), %eax
+; CHECK-NOBMI-NEXT:    testl %eax, %edi
+; CHECK-NOBMI-NEXT:    setne %al
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-LABEL: ctpop32_ne_one_nonzero:
+; CHECK-BMI2:       # %bb.0: # %entry
+; CHECK-BMI2-NEXT:    blsrl %edi, %eax
+; CHECK-BMI2-NEXT:    setne %al
+; CHECK-BMI2-NEXT:    retq
+entry:
+  %popcnt = tail call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp ne i32 %popcnt, 1
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll
index ac41a3fe6bb7e44ec5e58fc833a4bbd4d2b09131..6c0aaeb451e14a84fbe013a2ab5cf99dc4c105bc 100644
--- a/llvm/test/CodeGen/X86/known-never-zero.ll
+++ b/llvm/test/CodeGen/X86/known-never-zero.ll
@@ -555,9 +555,9 @@ define <4 x i32> @smax_known_zero_vec(<4 x i32> %x, <4 x i32> %y) {
 ; X86-NEXT:    por %xmm2, %xmm0
 ; X86-NEXT:    pcmpeqd %xmm1, %xmm1
 ; X86-NEXT:    paddd %xmm0, %xmm1
-; X86-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
-; X86-NEXT:    pxor %xmm1, %xmm0
-; X86-NEXT:    pcmpgtd %xmm1, %xmm0
+; X86-NEXT:    pand %xmm1, %xmm0
+; X86-NEXT:    pxor %xmm1, %xmm1
+; X86-NEXT:    pcmpeqd %xmm1, %xmm0
 ; X86-NEXT:    psrld $31, %xmm0
 ; X86-NEXT:    retl
 ;
@@ -566,10 +566,10 @@ define <4 x i32> @smax_known_zero_vec(<4 x i32> %x, <4 x i32> %y) {
 ; X64-NEXT:    vpmaxsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; X64-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; X64-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpminud %xmm1, %xmm0, %xmm1
+; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; X64-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; X64-NEXT:    retq
   %z = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> <i32 54, i32 23, i32 12, i32 1>)
   %r = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %z)
diff --git a/llvm/test/CodeGen/X86/pr108731.ll b/llvm/test/CodeGen/X86/pr108731.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1c6d2deb701afa31c8524886965e1dceeac16041
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr108731.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,NOBMI
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,BMI
+
+define i64 @foo(i64 %w, i64 %x, i64 %y, i64 %z) {
+; NOBMI-LABEL: foo:
+; NOBMI:       # %bb.0: # %Entry
+; NOBMI-NEXT:    movq %rcx, %rax
+; NOBMI-NEXT:    andq %rdx, %rsi
+; NOBMI-NEXT:    notq %rsi
+; NOBMI-NEXT:    andq %rdi, %rsi
+; NOBMI-NEXT:    notq %rax
+; NOBMI-NEXT:    orq %rdx, %rax
+; NOBMI-NEXT:    andq %rsi, %rax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: foo:
+; BMI:       # %bb.0: # %Entry
+; BMI-NEXT:    andq %rdx, %rsi
+; BMI-NEXT:    andnq %rdi, %rsi, %rax
+; BMI-NEXT:    andnq %rcx, %rdx, %rcx
+; BMI-NEXT:    andnq %rax, %rcx, %rax
+; BMI-NEXT:    retq
+Entry:
+  %and1 = and i64 %y, %x
+  %xor1 = xor i64 %and1, -1
+  %and2 = and i64 %xor1, %w
+  %.not = xor i64 %z, -1
+  %or1 = or i64 %.not, %y
+  %and3 = and i64 %and2, %or1
+  ret i64 %and3
+}
+
+define <16 x i8> @fooVec(<16 x i8> %w, <16 x i8> %x, <16 x i8> %y, <16 x i8> %z) {
+; NOBMI-LABEL: fooVec:
+; NOBMI:       # %bb.0: # %Entry
+; NOBMI-NEXT:    andps %xmm2, %xmm1
+; NOBMI-NEXT:    andnps %xmm0, %xmm1
+; NOBMI-NEXT:    andnps %xmm3, %xmm2
+; NOBMI-NEXT:    andnps %xmm1, %xmm2
+; NOBMI-NEXT:    movaps %xmm2, %xmm0
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: fooVec:
+; BMI:       # %bb.0: # %Entry
+; BMI-NEXT:    vandps %xmm1, %xmm2, %xmm1
+; BMI-NEXT:    vandnps %xmm0, %xmm1, %xmm0
+; BMI-NEXT:    vandnps %xmm3, %xmm2, %xmm1
+; BMI-NEXT:    vandnps %xmm0, %xmm1, %xmm0
+; BMI-NEXT:    retq
+Entry:
+  %and1 = and <16 x i8> %y, %x
+  %xor1 = xor <16 x i8> %and1, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %and2 = and <16 x i8> %xor1, %w
+  %.not = xor <16 x i8> %z, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %or1 = or <16 x i8> %.not, %y
+  %and3 = and <16 x i8> %and2, %or1
+  ret <16 x i8> %and3
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/pr48064.mir b/llvm/test/CodeGen/X86/pr48064.mir
index a6c927185881f153083a66a5e067b103f4943df5..9712a3ca27aa49539093363a5e849547deafdfff 100644
--- a/llvm/test/CodeGen/X86/pr48064.mir
+++ b/llvm/test/CodeGen/X86/pr48064.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple="i386-pc-windows-msvc" -run-pass=stack-coloring %s -o - | FileCheck %s
+# RUN: llc -mtriple="i386-pc-windows-msvc" -passes=stack-coloring %s -o - | FileCheck %s
 
 # There is a problem with the exception handler, we found in windows, when set
 # LifetimeStartOnFirstUse=true for stack-coloring in default. Take the following
diff --git a/llvm/test/CodeGen/X86/stack-protector-trap-unreachable.ll b/llvm/test/CodeGen/X86/stack-protector-trap-unreachable.ll
new file mode 100644
index 0000000000000000000000000000000000000000..093bb44050809f3b7872e518cb77c803811254a3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/stack-protector-trap-unreachable.ll
@@ -0,0 +1,40 @@
+;; Make sure we emit trap instructions after stack protector checks iff NoTrapAfterNoReturn is false.
+
+; RUN: llc -enable-selectiondag-sp -mtriple=x86_64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -enable-selectiondag-sp -mtriple=x86_64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -enable-selectiondag-sp -mtriple=x86_64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,TRAP_UNREACHABLE %s
+
+; RUN: llc -enable-selectiondag-sp=false -mtriple=x86_64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -enable-selectiondag-sp=false -mtriple=x86_64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -enable-selectiondag-sp=false -mtriple=x86_64 -fast-isel=false -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,TRAP_UNREACHABLE %s
+
+;; Make sure FastISel doesn't break anything.
+; RUN: llc -mtriple=x86_64 -fast-isel -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable=false --o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=x86_64 -fast-isel -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,NO_TRAP_UNREACHABLE %s
+; RUN: llc -mtriple=x86_64 -fast-isel -global-isel=false -verify-machineinstrs -print-after=finalize-isel \
+; RUN:     -trap-unreachable -no-trap-after-noreturn=false -o /dev/null 2>&1 %s | FileCheck --check-prefixes=CHECK,TRAP_UNREACHABLE %s
+
+; CHECK-LABEL: Machine code for function test
+; CHECK: bb.0.entry:
+; CHECK: CALL64{{.*}}__stack_chk_fail
+; CHECK-NEXT: ADJCALLSTACKUP64
+; TRAP_UNREACHABLE-NEXT: TRAP
+; NO_TRAP_UNREACHABLE-NOT: TRAP
+; NO_TRAP_UNREACHABLE-EMPTY:
+
+define void @test() nounwind ssp {
+entry:
+  %buf = alloca [8 x i8]
+  %result = call i32(ptr) @callee(ptr %buf) nounwind
+  ret void
+}
+
+declare i32 @callee(ptr) nounwind
diff --git a/llvm/test/CodeGen/X86/tls-align.ll b/llvm/test/CodeGen/X86/tls-align.ll
index e996c00dbf1d4ac63d3fcf2a0f12b18dd9cbcdcc..94f9b9045cf24c3a2a9b3adf386ea662b0b0a253 100644
--- a/llvm/test/CodeGen/X86/tls-align.ll
+++ b/llvm/test/CodeGen/X86/tls-align.ll
@@ -1,4 +1,3 @@
-; REQUIRES: x86-registered-target
 ; RUN: opt -passes=instcombine -S < %s | FileCheck %s
 
 %class.Arr = type <{ [160 x %class.Derived], i32, [4 x i8] }>
diff --git a/llvm/test/CodeGen/X86/unreachable-trap.ll b/llvm/test/CodeGen/X86/unreachable-trap.ll
index d2704bf7b462055857694737bab5ec473da2a91a..486ee0f7a0e24635848e3c77adf54f561321acab 100644
--- a/llvm/test/CodeGen/X86/unreachable-trap.ll
+++ b/llvm/test/CodeGen/X86/unreachable-trap.ll
@@ -1,7 +1,15 @@
 ; RUN: llc -o - %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK
 ; RUN: llc -o - %s -mtriple=x86_64-windows-msvc | FileCheck %s --check-prefixes=CHECK
-; RUN: llc -o - %s -mtriple=x86_64-scei-ps4 | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
 ; RUN: llc -o - %s -mtriple=x86_64-apple-darwin | FileCheck %s --check-prefixes=CHECK,NO_TRAP_AFTER_NORETURN
+
+; On PS4/PS5, always emit trap instructions regardless of of trap-unreachable or no-trap-after-noreturn.
+; RUN: llc -o - %s -mtriple=x86_64-scei-ps4 -trap-unreachable | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-sie-ps5 -trap-unreachable | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-scei-ps4 -trap-unreachable=false | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-sie-ps5 -trap-unreachable=false | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-scei-ps4 -trap-unreachable -no-trap-after-noreturn=false | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-sie-ps5 -trap-unreachable -no-trap-after-noreturn=false | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+
 ; RUN: llc --trap-unreachable -o - %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
 ; RUN: llc --trap-unreachable -global-isel -o - %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
 ; RUN: llc --trap-unreachable -fast-isel -o - %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
diff --git a/llvm/test/CodeGen/X86/vector-shift-lut.ll b/llvm/test/CodeGen/X86/vector-shift-lut.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0bf2006090893cb566017a30d3c7a71917e6ca2e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-shift-lut.ll
@@ -0,0 +1,1831 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512BW
+
+; Shifts of uniform vXi8 constant values can be performed by a PSHUFB LUT
+
+define <16 x i8> @uniform_shl_v16i8(<16 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_shl_v16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    paddb %xmm3, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT:    psubb %xmm3, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_shl_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: uniform_shl_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; AVX-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: uniform_shl_v16i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_shl_v16i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_shl_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovq {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    retq
+  %shift = shl <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, %a
+  ret <16 x i8> %shift
+}
+
+define <16 x i8> @uniform_lshr_v16i8(<16 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_lshr_v16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    paddb %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_lshr_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: uniform_lshr_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: uniform_lshr_v16i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; XOP-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_lshr_v16i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm1 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_lshr_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovd {{.*#+}} xmm1 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    retq
+  %shift = lshr <16 x i8> <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>, %a
+  ret <16 x i8> %shift
+}
+
+define <16 x i8> @uniform_ashr_v16i8(<16 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_ashr_v16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    paddb %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_ashr_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: uniform_ashr_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd {{.*#+}} xmm1 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; XOP-LABEL: uniform_ashr_v16i8:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; XOP-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOP-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_ashr_v16i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm1 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_ashr_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovd {{.*#+}} xmm1 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    retq
+  %shift = ashr <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>, %a
+  ret <16 x i8> %shift
+}
+
+define <32 x i8> @uniform_shl_v32i8(<32 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_shl_v32i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; SSE2-NEXT:    movdqa %xmm7, %xmm0
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    por %xmm0, %xmm7
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm8
+; SSE2-NEXT:    pandn %xmm7, %xmm8
+; SSE2-NEXT:    psllw $2, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    por %xmm8, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    por %xmm2, %xmm7
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pandn %xmm7, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm7
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_shl_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm2 = [4,8,16,32,64,128,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    pshufb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uniform_shl_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = [4,8,16,32,64,128,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uniform_shl_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: uniform_shl_v32i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm0, %xmm2, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: uniform_shl_v32i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_shl_v32i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_shl_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0,4,8,16,32,64,128,0,0]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT:    retq
+  %shift = shl <32 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>, %a
+  ret <32 x i8> %shift
+}
+
+define <32 x i8> @uniform_lshr_v32i8(<32 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_lshr_v32i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5]
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pandn %xmm0, %xmm7
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pand %xmm0, %xmm6
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pandn %xmm6, %xmm7
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm6
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm7
+; SSE2-NEXT:    pandn %xmm6, %xmm7
+; SSE2-NEXT:    psrlw $2, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_lshr_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = [5,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    pshufb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uniform_lshr_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = [5,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uniform_lshr_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: uniform_lshr_v32i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5]
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm2, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: uniform_lshr_v32i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_lshr_v32i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_lshr_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0,5,2,1,0,0,0,0,0]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT:    retq
+  %shift = lshr <32 x i8> <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>, %a
+  ret <32 x i8> %shift
+}
+
+define <32 x i8> @uniform_ashr_v32i8(<32 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_ashr_v32i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6]
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pandn %xmm0, %xmm7
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pand %xmm0, %xmm6
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pandn %xmm6, %xmm7
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm6
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm7
+; SSE2-NEXT:    pandn %xmm6, %xmm7
+; SSE2-NEXT:    psrlw $2, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_ashr_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = [6,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pshufb %xmm0, %xmm3
+; SSE41-NEXT:    pshufb %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uniform_ashr_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = [6,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uniform_ashr_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: uniform_ashr_v32i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6]
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm2, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: uniform_ashr_v32i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_ashr_v32i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_ashr_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0,6,3,1,0,0,0,0,0]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT:    retq
+  %shift = ashr <32 x i8> <i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6>, %a
+  ret <32 x i8> %shift
+}
+
+define <64 x i8> @uniform_shl_v64i8(<64 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_shl_v64i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm9
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pandn %xmm5, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112]
+; SSE2-NEXT:    pand %xmm7, %xmm9
+; SSE2-NEXT:    por %xmm0, %xmm9
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pandn %xmm9, %xmm10
+; SSE2-NEXT:    psllw $2, %xmm9
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm9, %xmm0
+; SSE2-NEXT:    por %xmm10, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm9, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    pand %xmm7, %xmm9
+; SSE2-NEXT:    por %xmm1, %xmm9
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm10
+; SSE2-NEXT:    pandn %xmm9, %xmm10
+; SSE2-NEXT:    psllw $2, %xmm9
+; SSE2-NEXT:    pand %xmm8, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm1
+; SSE2-NEXT:    por %xmm10, %xmm1
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm4
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pandn %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm9
+; SSE2-NEXT:    por %xmm4, %xmm9
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm10
+; SSE2-NEXT:    pandn %xmm9, %xmm10
+; SSE2-NEXT:    psllw $2, %xmm9
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
+; SSE2-NEXT:    por %xmm10, %xmm4
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm9
+; SSE2-NEXT:    pandn %xmm5, %xmm9
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    por %xmm9, %xmm2
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pandn %xmm2, %xmm7
+; SSE2-NEXT:    psllw $2, %xmm2
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pandn %xmm5, %xmm2
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_shl_v64i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm4 = [7,14,28,56,112,224,192,128,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pshufb %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pshufb %xmm1, %xmm6
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pshufb %xmm2, %xmm7
+; SSE41-NEXT:    pshufb %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    movdqa %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uniform_shl_v64i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovq {{.*#+}} xmm3 = [7,14,28,56,112,224,192,128,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uniform_shl_v64i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: uniform_shl_v64i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vpshlb %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: uniform_shl_v64i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; XOPAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_shl_v64i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_shl_v64i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128,7,14,28,56,112,224,192,128]
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+  %shift = shl <64 x i8> <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>, %a
+  ret <64 x i8> %shift
+}
+
+define <64 x i8> @uniform_lshr_v64i8(<64 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_lshr_v64i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE2-NEXT:    pandn %xmm6, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm8
+; SSE2-NEXT:    pandn %xmm0, %xmm8
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm6, %xmm1
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm8
+; SSE2-NEXT:    pandn %xmm1, %xmm8
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm6, %xmm5
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm8
+; SSE2-NEXT:    pandn %xmm5, %xmm8
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    pand %xmm7, %xmm5
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    psllw $5, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_lshr_v64i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovsxbq {{.*#+}} xmm4 = [1,0]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pshufb %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pshufb %xmm1, %xmm6
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pshufb %xmm2, %xmm7
+; SSE41-NEXT:    pshufb %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    movdqa %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uniform_lshr_v64i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [1,0]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uniform_lshr_v64i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [1,0,1,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: uniform_lshr_v64i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, %xmm4, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm4, %xmm1
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: uniform_lshr_v64i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [1,0,1,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; XOPAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_lshr_v64i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [1,0,1,0]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_lshr_v64i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+  %shift = lshr <64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, %a
+  ret <64 x i8> %shift
+}
+
+define <64 x i8> @uniform_ashr_v64i8(<64 x i8> %a) nounwind {
+; SSE2-LABEL: uniform_ashr_v64i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; SSE2-NEXT:    pandn %xmm6, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm8
+; SSE2-NEXT:    pandn %xmm0, %xmm8
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm6, %xmm1
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm8
+; SSE2-NEXT:    pandn %xmm1, %xmm8
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm6, %xmm5
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm8
+; SSE2-NEXT:    pandn %xmm5, %xmm8
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    pand %xmm7, %xmm5
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    psllw $5, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: uniform_ashr_v64i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm4 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pshufb %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm6
+; SSE41-NEXT:    pshufb %xmm1, %xmm6
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pshufb %xmm2, %xmm7
+; SSE41-NEXT:    pshufb %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    movdqa %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uniform_ashr_v64i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uniform_ashr_v64i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: uniform_ashr_v64i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
+; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, %xmm4, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm4, %xmm1
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: uniform_ashr_v64i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; XOPAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: uniform_ashr_v64i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: uniform_ashr_v64i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0]
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+  %shift = ashr <64 x i8> <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>, %a
+  ret <64 x i8> %shift
+}
+
+; Test where each 128-bit lane has a different value
+
+define <32 x i8> @perlane_shl_v32i8(<32 x i8> %a) nounwind {
+; SSE2-LABEL: perlane_shl_v32i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT:    por %xmm1, %xmm5
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pandn %xmm5, %xmm6
+; SSE2-NEXT:    psllw $2, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm6, %xmm1
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    paddb %xmm2, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    psubb %xmm2, %xmm5
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pandn %xmm5, %xmm6
+; SSE2-NEXT:    psllw $2, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: perlane_shl_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm2
+; SSE41-NEXT:    movq {{.*#+}} xmm3 = [2,4,8,16,32,64,128,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: perlane_shl_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = [2,4,8,16,32,64,128,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: perlane_shl_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: perlane_shl_v32i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: perlane_shl_v32i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: perlane_shl_v32i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: perlane_shl_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT:    retq
+  %shift = shl <32 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>, %a
+  ret <32 x i8> %shift
+}
+
+define <32 x i8> @perlane_lshr_v32i8(<32 x i8> %a) nounwind {
+; SSE2-LABEL: perlane_lshr_v32i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    paddb %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm1
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    psrlw $2, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm3, %xmm4
+; SSE2-NEXT:    psrlw $1, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm3
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm4
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: perlane_lshr_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = [4,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: perlane_lshr_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = [4,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: perlane_lshr_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [259,0,66052,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: perlane_lshr_v32i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm2
+; XOPAVX1-NEXT:    vpshlb %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: perlane_lshr_v32i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [259,0,66052,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: perlane_lshr_v32i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [259,0,66052,0]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: perlane_lshr_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [259,0,66052,0]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT:    retq
+  %shift = lshr <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>, %a
+  ret <32 x i8> %shift
+}
+
+define <32 x i8> @perlane_ashr_v32i8(<32 x i8> %a) nounwind {
+; SSE2-LABEL: perlane_ashr_v32i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    paddb %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; SSE2-NEXT:    movdqa %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    psrlw $2, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm3, %xmm6
+; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pandn %xmm6, %xmm5
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm5
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm6
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pandn %xmm5, %xmm7
+; SSE2-NEXT:    psrlw $2, %xmm5
+; SSE2-NEXT:    pand %xmm3, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pandn %xmm6, %xmm1
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: perlane_ashr_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = [5,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = [6,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: perlane_ashr_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = [6,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vmovd {{.*#+}} xmm2 = [5,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: perlane_ashr_v32i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [66053,0,66310,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: perlane_ashr_v32i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm2
+; XOPAVX1-NEXT:    vpshlb %xmm2, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: perlane_ashr_v32i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [66053,0,66310,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: perlane_ashr_v32i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [66053,0,66310,0]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: perlane_ashr_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxdq {{.*#+}} ymm1 = [66053,0,66310,0]
+; AVX512BW-NEXT:    vpshufb %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT:    retq
+  %shift = ashr <32 x i8> <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6>, %a
+  ret <32 x i8> %shift
+}
+
+define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
+; SSE2-LABEL: perlane_shl_v64i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pandn %xmm4, %xmm7
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm7
+; SSE2-NEXT:    pandn %xmm1, %xmm7
+; SSE2-NEXT:    psllw $2, %xmm1
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm4, %xmm3
+; SSE2-NEXT:    paddb %xmm4, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; SSE2-NEXT:    pand %xmm1, %xmm7
+; SSE2-NEXT:    paddb %xmm1, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    psubb %xmm1, %xmm7
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn %xmm7, %xmm1
+; SSE2-NEXT:    psllw $2, %xmm7
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: perlane_shl_v64i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{.*#+}} xmm4 = [7,14,28,56,112,224,192,128,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm4
+; SSE41-NEXT:    movq {{.*#+}} xmm5 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm2, %xmm5
+; SSE41-NEXT:    movq {{.*#+}} xmm6 = [2,4,8,16,32,64,128,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm3, %xmm6
+; SSE41-NEXT:    xorps %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm6, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: perlane_shl_v64i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovq {{.*#+}} xmm3 = [2,4,8,16,32,64,128,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vmovq {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = [7,14,28,56,112,224,192,128,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: perlane_shl_v64i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,14,28,56,112,224,192,128,u,u,u,u,u,u,u,u,0,0,0,0,0,0,0,0,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: perlane_shl_v64i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpshlb %xmm1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; XOPAVX1-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: perlane_shl_v64i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,14,28,56,112,224,192,128,u,u,u,u,u,u,u,u,0,0,0,0,0,0,0,0,u,u,u,u,u,u,u,u]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; XOPAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: perlane_shl_v64i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,14,28,56,112,224,192,128,u,u,u,u,u,u,u,u,0,0,0,0,0,0,0,0,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: perlane_shl_v64i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [7,14,28,56,112,224,192,128,u,u,u,u,u,u,u,u,0,0,0,0,0,0,0,0,u,u,u,u,u,u,u,u,1,2,4,8,16,32,64,128,u,u,u,u,u,u,u,u,2,4,8,16,32,64,128,0,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+  %shift = shl <64 x i8> <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>, %a
+  ret <64 x i8> %shift
+}
+
+define <64 x i8> @perlane_lshr_v64i8(<64 x i8> %a) nounwind {
+; SSE2-LABEL: perlane_lshr_v64i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    paddb %xmm1, %xmm6
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm6, %xmm8
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa %xmm8, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    psrlw $2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm7, %xmm8
+; SSE2-NEXT:    pand %xmm2, %xmm8
+; SSE2-NEXT:    por %xmm1, %xmm8
+; SSE2-NEXT:    paddb %xmm6, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm6, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm2
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm8
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa %xmm8, %xmm9
+; SSE2-NEXT:    pandn %xmm2, %xmm9
+; SSE2-NEXT:    psrlw $2, %xmm2
+; SSE2-NEXT:    pand %xmm7, %xmm8
+; SSE2-NEXT:    pand %xmm2, %xmm8
+; SSE2-NEXT:    por %xmm9, %xmm8
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm5
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm5
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm8
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT:    movdqa %xmm8, %xmm9
+; SSE2-NEXT:    pandn %xmm5, %xmm9
+; SSE2-NEXT:    psrlw $2, %xmm5
+; SSE2-NEXT:    pand %xmm7, %xmm8
+; SSE2-NEXT:    pand %xmm5, %xmm8
+; SSE2-NEXT:    por %xmm9, %xmm8
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    psrlw $1, %xmm8
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    psllw $5, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm3
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm7
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm7
+; SSE2-NEXT:    paddb %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm7, %xmm0
+; SSE2-NEXT:    psrlw $1, %xmm7
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: perlane_lshr_v64i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd {{.*#+}} xmm4 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm0, %xmm4
+; SSE41-NEXT:    movd {{.*#+}} xmm5 = [4,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm1, %xmm5
+; SSE41-NEXT:    movd {{.*#+}} xmm6 = [5,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm2, %xmm6
+; SSE41-NEXT:    movd {{.*#+}} xmm7 = [6,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm3, %xmm7
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm6, %xmm2
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: perlane_lshr_v64i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = [4,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = [6,3,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = [5,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: perlane_lshr_v64i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm2 = [259,0,66052,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm2 = [66053,0,66310,0]
+; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: perlane_lshr_v64i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm2, %xmm3
+; XOPAVX1-NEXT:    vpshlb %xmm3, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm2, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm3
+; XOPAVX1-NEXT:    vpshlb %xmm3, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: perlane_lshr_v64i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpmovsxdq {{.*#+}} ymm2 = [259,0,66052,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; XOPAVX2-NEXT:    vpmovsxdq {{.*#+}} ymm2 = [66053,0,66310,0]
+; XOPAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: perlane_lshr_v64i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm2 = [66053,0,66310,0]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm2 = [259,0,66052,0]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: perlane_lshr_v64i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxdq {{.*#+}} zmm1 = [259,0,66052,0,66053,0,66310,0]
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+  %shift = lshr <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6>, %a
+  ret <64 x i8> %shift
+}
+
+define <64 x i8> @perlane_ashr_v64i8(<64 x i8> %a) nounwind {
+; SSE2-LABEL: perlane_ashr_v64i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    paddb %xmm1, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pandn %xmm0, %xmm6
+; SSE2-NEXT:    paddb %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm6, %xmm5
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    psllw $5, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm6
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT:    pandn %xmm5, %xmm6
+; SSE2-NEXT:    paddb %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    pand %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    psllw $5, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm2
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm6
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: perlane_ashr_v64i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovsxbq {{.*#+}} xmm4 = [1,0]
+; SSE41-NEXT:    pshufb %xmm1, %xmm4
+; SSE41-NEXT:    movd {{.*#+}} xmm5 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm2, %xmm5
+; SSE41-NEXT:    movd {{.*#+}} xmm6 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; SSE41-NEXT:    pshufb %xmm3, %xmm6
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm6, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: perlane_ashr_v64i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = [3,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = [2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpmovsxbq {{.*#+}} xmm2 = [1,0]
+; AVX1-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: perlane_ashr_v64i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [0,0,1,0]
+; AVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpmovsxwq {{.*#+}} ymm2 = [258,0,259,0]
+; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: perlane_ashr_v64i8:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm3
+; XOPAVX1-NEXT:    vpshlb %xmm3, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vpshlb %xmm1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vpsubb %xmm0, %xmm2, %xmm0
+; XOPAVX1-NEXT:    vpshlb %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: perlane_ashr_v64i8:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [0,0,1,0]
+; XOPAVX2-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; XOPAVX2-NEXT:    vpmovsxwq {{.*#+}} ymm2 = [258,0,259,0]
+; XOPAVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    retq
+;
+; AVX512DQ-LABEL: perlane_ashr_v64i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512DQ-NEXT:    vpmovsxwq {{.*#+}} ymm2 = [258,0,259,0]
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [0,0,1,0]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512BW-LABEL: perlane_ashr_v64i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpmovsxwq {{.*#+}} zmm1 = [0,0,1,0,258,0,259,0]
+; AVX512BW-NEXT:    vpshufb %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+  %shift = ashr <64 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>, %a
+  ret <64 x i8> %shift
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; SSE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
index 61814b48e6b3a35432b677935657e1dce6192455..04cfa3cedd58fd166d3191f3f02d3f32ac93ead5 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
@@ -151,10 +151,9 @@ define <64 x i8> @combine_permi2q_pshufb_as_permi2d_mask(<8 x i64> %a0, <8 x i64
 define <64 x i8> @combine_vpermi2var_v64i8_with_mask(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> %a2) {
 ; CHECK-LABEL: combine_vpermi2var_v64i8_with_mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpermt2b %zmm2, %zmm1, %zmm0
 ; CHECK-NEXT:    vpmovb2m %zmm1, %k0
-; CHECK-NEXT:    vpmovm2b %k0, %zmm1
-; CHECK-NEXT:    vpandnq %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    knotq %k0, %k1
+; CHECK-NEXT:    vpermt2b %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; CHECK-NEXT:    ret{{[l|q]}}
   %perm = tail call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> %a2)
   %cmp = icmp slt <64 x i8> %a1, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/vselect-packss.ll b/llvm/test/CodeGen/X86/vselect-packss.ll
index 7078eed042b43ee88b66294a7a911dff8c555a90..5b14e2782ee1c91d87fd813d5e976d753c68e364 100644
--- a/llvm/test/CodeGen/X86/vselect-packss.ll
+++ b/llvm/test/CodeGen/X86/vselect-packss.ll
@@ -66,7 +66,7 @@ define <16 x i8> @vselect_packss_v16i16(<16 x i16> %a0, <16 x i16> %a1, <16 x i8
 ; AVX512VL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm3 ^ (xmm0 & (xmm2 ^ xmm3))
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
@@ -82,7 +82,7 @@ define <16 x i8> @vselect_packss_v16i16(<16 x i16> %a0, <16 x i16> %a1, <16 x i8
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpcmpeqw %ymm1, %ymm0, %k0
 ; AVX512BWVL-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512BWVL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm3 ^ (xmm0 & (xmm2 ^ xmm3))
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i16> %a0, %a1
@@ -156,7 +156,7 @@ define <16 x i8> @vselect_packss_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i8
 ; AVX512F-LABEL: vselect_packss_v16i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -165,9 +165,9 @@ define <16 x i8> @vselect_packss_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i8
 ; AVX512VL-LABEL: vselect_packss_v16i32:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k1
-; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm3 ^ (xmm0 & (xmm2 ^ xmm3))
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
@@ -183,7 +183,7 @@ define <16 x i8> @vselect_packss_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i8
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
 ; AVX512BWVL-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512BWVL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm3 ^ (xmm0 & (xmm2 ^ xmm3))
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i32> %a0, %a1
@@ -312,7 +312,7 @@ define <16 x i8> @vselect_packss_v16i64(<16 x i64> %a0, <16 x i64> %a1, <16 x i8
 ; AVX512F-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
 ; AVX512F-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
 ; AVX512F-NEXT:    kunpckbw %k0, %k1, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vpblendvb %xmm0, %xmm4, %xmm5, %xmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -323,9 +323,9 @@ define <16 x i8> @vselect_packss_v16i64(<16 x i64> %a0, <16 x i64> %a1, <16 x i8
 ; AVX512VL-NEXT:    vpcmpeqq %zmm2, %zmm0, %k0
 ; AVX512VL-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
 ; AVX512VL-NEXT:    kunpckbw %k0, %k1, %k1
-; AVX512VL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $202, %xmm5, %xmm4, %xmm0
+; AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm5 ^ (xmm0 & (xmm4 ^ xmm5))
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
@@ -345,7 +345,7 @@ define <16 x i8> @vselect_packss_v16i64(<16 x i64> %a0, <16 x i64> %a1, <16 x i8
 ; AVX512BWVL-NEXT:    vpcmpeqq %zmm3, %zmm1, %k1
 ; AVX512BWVL-NEXT:    kunpckbw %k0, %k1, %k0
 ; AVX512BWVL-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512BWVL-NEXT:    vpternlogq $202, %xmm5, %xmm4, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm5 ^ (xmm0 & (xmm4 ^ xmm5))
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i64> %a0, %a1
@@ -415,7 +415,7 @@ define <16 x i8> @vselect_packss(<16 x i16> %a0, <16 x i16> %a1, <16 x i8> %a2,
 ; AVX512VL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512VL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm3 ^ (xmm0 & (xmm2 ^ xmm3))
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
@@ -433,7 +433,7 @@ define <16 x i8> @vselect_packss(<16 x i16> %a0, <16 x i16> %a1, <16 x i8> %a2,
 ; AVX512BWVL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
+; AVX512BWVL-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm3 ^ (xmm0 & (xmm2 ^ xmm3))
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp eq <16 x i16> %a0, %a1
diff --git a/llvm/test/DebugInfo/COFF/fortran-contained-proc.ll b/llvm/test/DebugInfo/COFF/fortran-contained-proc.ll
index f8b66aae212e798dd3eee52f935412140e68655d..82b39d4fa34cb087af1b4859de8867dac213d3ed 100644
--- a/llvm/test/DebugInfo/COFF/fortran-contained-proc.ll
+++ b/llvm/test/DebugInfo/COFF/fortran-contained-proc.ll
@@ -74,8 +74,8 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #3
 ; Function Attrs: mustprogress nofree nosync nounwind readnone speculatable willreturn
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
 
-attributes #0 = { nounwind uwtable "denormal-fp-math"="preserve_sign,preserve_sign" "frame-pointer"="none" "intel-lang"="fortran" "min-legal-vector-width"="0" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" }
-attributes #1 = { mustprogress nofree norecurse nosync nounwind uwtable willreturn writeonly "denormal-fp-math"="preserve_sign,preserve_sign" "frame-pointer"="none" "intel-lang"="fortran" "min-legal-vector-width"="0" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" }
+attributes #0 = { nounwind uwtable "denormal-fp-math"="preserve-sign,preserve-sign" "frame-pointer"="none" "intel-lang"="fortran" "min-legal-vector-width"="0" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" }
+attributes #1 = { mustprogress nofree norecurse nosync nounwind uwtable willreturn writeonly "denormal-fp-math"="preserve-sign,preserve-sign" "frame-pointer"="none" "intel-lang"="fortran" "min-legal-vector-width"="0" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" }
 attributes #2 = { nofree "intel-lang"="fortran" }
 attributes #3 = { mustprogress nofree nosync nounwind readnone speculatable willreturn }
 attributes #4 = { nounwind }
diff --git a/llvm/test/DebugInfo/Generic/sugared-constants.ll b/llvm/test/DebugInfo/Generic/sugared-constants.ll
index c8ee035dbca1dc558e86be5a65355dace19d72e8..221586fd1bc6f9571d32356089f1a2dbda17c841 100644
--- a/llvm/test/DebugInfo/Generic/sugared-constants.ll
+++ b/llvm/test/DebugInfo/Generic/sugared-constants.ll
@@ -1,7 +1,3 @@
-; xfail this test on hexagon because upstream llc is not emitting the
-; correct DWARF info. Downstream llc is.
-; XFAIL: target=hexagon-{{.*}}
-
 ; RUN: %llc_dwarf -O0 -filetype=obj %s -o - | llvm-dwarfdump -v -debug-info - | FileCheck %s
 ; Use correct signedness when emitting constants of derived (sugared) types.
 
diff --git a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir
index 5df70096e9305d11dba311d2559c3ea2b5bbe20d..d6171ec30b12711ebabe6993371496e3baa3b695 100644
--- a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir
+++ b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir
@@ -3,11 +3,13 @@
 ## This tests that LiveDebugVariables does not trim non-inlined variable
 ## location.
 
+# CHECK: ![[VARA:.*]] = !DILocalVariable(name: "a"
 # CHECK: ![[VARB:.*]] = !DILocalVariable(name: "b"
 # CHECK: ![[VARC:.*]] = !DILocalVariable(name: "c"
 # CHECK: $at = COPY $a2
 # CHECK-NEXT: DBG_VALUE $at, $noreg, ![[VARC]], !DIExpression(), debug-location
 # CHECK: $s0 = COPY $a1
+# CHECK-NEXT: DBG_VALUE $a0, $noreg, ![[VARA]], !DIExpression(), debug-location
 # CHECK-NEXT: DBG_VALUE $s0, $noreg, ![[VARB]], !DIExpression(), debug-location
 
 --- |
diff --git a/llvm/test/DebugInfo/Mips/livedebugvariables-reorder.mir b/llvm/test/DebugInfo/Mips/livedebugvariables-reorder.mir
new file mode 100644
index 0000000000000000000000000000000000000000..bc24c5272d31877fc5fd0eb7e417fe107605ae38
--- /dev/null
+++ b/llvm/test/DebugInfo/Mips/livedebugvariables-reorder.mir
@@ -0,0 +1,84 @@
+# RUN: llc -emit-call-site-info %s -mtriple=mips -start-before=register-coalescer -stop-after=virtregrewriter -o - | FileCheck %s
+
+# Verify that LiveDebugVariables does not reorder the stack location DBG_VALUE
+# and the fragmented DBG_VALUEs for aaa, as the latter may represent a stale
+# value. It should also not reorder the DBG_VALUEs for the different variables,
+# as that results in noise in pass before/after diffs.
+
+# CHECK-DAG: ![[aaa:[0-9]+]] = !DILocalVariable(name: "aaa"
+# CHECK-DAG: ![[bbb:[0-9]+]] = !DILocalVariable(name: "bbb"
+# CHECK-DAG: ![[ccc:[0-9]+]] = !DILocalVariable(name: "ccc"
+# CHECK-DAG: ![[ddd:[0-9]+]] = !DILocalVariable(name: "ddd"
+
+# CHECK: DBG_VALUE 1001, $noreg, ![[aaa]], !DIExpression(DW_OP_LLVM_fragment, 0, 16)
+# CHECK-NEXT: DBG_VALUE 1002, $noreg, ![[aaa]], !DIExpression(DW_OP_LLVM_fragment, 16, 16)
+# CHECK-NEXT: DBG_VALUE 222, $noreg, ![[bbb]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE 333, $noreg, ![[ccc]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE 444, $noreg, ![[ddd]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE %stack.0, 0, ![[aaa]], !DIExpression()
+
+--- |
+  target datalayout = "E-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64"
+  target triple = "mips"
+
+  define dso_local i32 @main() local_unnamed_addr #0 !dbg !3 {
+  entry:
+    ret i32 0, !dbg !12
+  }
+
+  declare !dbg !14 dso_local i32 @foo() local_unnamed_addr
+
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+  attributes #0 = { "frame-pointer"="all" }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!2}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 19.0.0git", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "main.c", directory: "/", checksumkind: CSK_MD5, checksum: "9edfcd32ce51b21ab508a4a0755aa78b")
+  !2 = !{i32 2, !"Debug Info Version", i32 3}
+  !3 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 12, type: !4, scopeLine: 12, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !7)
+  !4 = !DISubroutineType(types: !5)
+  !5 = !{!6}
+  !6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !7 = !{!8, !9, !10, !11}
+  !8 = !DILocalVariable(name: "aaa", scope: !3, file: !1, line: 13, type: !6)
+  !9 = !DILocalVariable(name: "bbb", scope: !3, file: !1, line: 13, type: !6)
+  !10 = !DILocalVariable(name: "ccc", scope: !3, file: !1, line: 13, type: !6)
+  !11 = !DILocalVariable(name: "ddd", scope: !3, file: !1, line: 13, type: !6)
+  !12 = !DILocation(line: 13, scope: !3)
+  !13 = !{i64 2147496201}
+  !14 = !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !4, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+
+...
+---
+name:            main
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gpr32 }
+frameInfo:
+  maxAlignment:    1
+  adjustsStack:    true
+  hasCalls:        true
+callSites:
+  - { bb: 0, offset: 1 }
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    ADJCALLSTACKDOWN 16, 0, implicit-def dead $sp, implicit $sp, debug-location !12
+    JAL @foo, csr_o32_fpxx, implicit-def dead $ra, implicit-def $sp, implicit-def $v0, debug-location !12
+    ADJCALLSTACKUP 16, 0, implicit-def dead $sp, implicit $sp, debug-location !12
+    %0:gpr32 = COPY killed $v0, debug-location !12
+    DBG_VALUE 1001, $noreg, !8, !DIExpression(DW_OP_LLVM_fragment, 0, 16), debug-location !12
+    DBG_VALUE 1002, $noreg, !8, !DIExpression(DW_OP_LLVM_fragment, 16, 16), debug-location !12
+    DBG_VALUE 222, $noreg, !9, !DIExpression(), debug-location !12
+    DBG_VALUE 333, $noreg, !10, !DIExpression(), debug-location !12
+    DBG_VALUE 444, $noreg, !11, !DIExpression(), debug-location !12
+    DBG_VALUE %0, $noreg, !8, !DIExpression(), debug-location !12
+    INLINEASM &"", 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $v0, 12 /* clobber */, implicit-def dead early-clobber $v1, 12 /* clobber */, implicit-def dead early-clobber $a0, 12 /* clobber */, implicit-def dead early-clobber $a1, 12 /* clobber */, implicit-def dead early-clobber $a2, 12 /* clobber */, implicit-def dead early-clobber $a3, 12 /* clobber */, implicit-def dead early-clobber $t0, 12 /* clobber */, implicit-def dead early-clobber $t1, 12 /* clobber */, implicit-def dead early-clobber $t2, 12 /* clobber */, implicit-def dead early-clobber $t3, 12 /* clobber */, implicit-def dead early-clobber $t4, 12 /* clobber */, implicit-def dead early-clobber $t5, 12 /* clobber */, implicit-def dead early-clobber $t6, 12 /* clobber */, implicit-def dead early-clobber $t7, 12 /* clobber */, implicit-def dead early-clobber $t8, 12 /* clobber */, implicit-def dead early-clobber $t9, 12 /* clobber */, implicit-def dead early-clobber $gp, 12 /* clobber */, implicit-def dead early-clobber $ra, 12 /* clobber */, implicit-def dead early-clobber $s0, 12 /* clobber */, implicit-def dead early-clobber $s1, 12 /* clobber */, implicit-def dead early-clobber $s2, 12 /* clobber */, implicit-def dead early-clobber $s3, 12 /* clobber */, implicit-def dead early-clobber $s4, 12 /* clobber */, implicit-def dead early-clobber $s5, 12 /* clobber */, implicit-def dead early-clobber $s6, 12 /* clobber */, implicit-def dead early-clobber $s7, 12 /* clobber */, implicit-def dead early-clobber $at, !13, debug-location !12
+    $v0 = COPY killed %0, debug-location !12
+    RetRA implicit killed $v0, debug-location !12
+
+...
diff --git a/llvm/test/DebugInfo/X86/live-debug-vars-discard-invalid.mir b/llvm/test/DebugInfo/X86/live-debug-vars-discard-invalid.mir
index 08130b47b4c5c90be96509aedd9acd30ad0663ca..7c8aa966d3d643d2cd3b0bdd4a5b78208d1359b2 100644
--- a/llvm/test/DebugInfo/X86/live-debug-vars-discard-invalid.mir
+++ b/llvm/test/DebugInfo/X86/live-debug-vars-discard-invalid.mir
@@ -118,15 +118,15 @@ body:             |
 
 # CHECK-LABEL: bb.3:
 # CHECK:        dead renamable $rcx = IMPLICIT_DEF
-# CHECK-NEXT:   DBG_VALUE 0, $noreg, !23, !DIExpression()
 # CHECK-NEXT:   DBG_VALUE $rcx, $noreg, !18, !DIExpression()
+# CHECK-NEXT:   DBG_VALUE 0, $noreg, !23, !DIExpression()
 
 # CHECK-LABEL: bb.4:
 # CHECK:        liveins: $rax
 # CHECK:        DBG_VALUE $rax, $noreg, !18, !DIExpression()
 # CHECK-NEXT:   renamable $rax = BTS64ri8 killed renamable $rax, 0, implicit-def $eflags
-# CHECK-NEXT:   DBG_VALUE 0, $noreg, !23, !DIExpression()
 # CHECK-NEXT:   DBG_VALUE $rax, $noreg, !18, !DIExpression()
+# CHECK-NEXT:   DBG_VALUE 0, $noreg, !23, !DIExpression()
 # CHECK-NEXT:   renamable $rax = BTS64ri8 killed renamable $rax, 0, implicit-def $eflags
 # CHECK-NEXT:   DBG_VALUE $rax, $noreg, !18, !DIExpression()
 # CHECK-NEXT:   dead renamable $rax = BTS64ri8 killed renamable $rax, 0, implicit-def $eflags
diff --git a/llvm/test/DebugInfo/X86/sdag-salvage-add.ll b/llvm/test/DebugInfo/X86/sdag-salvage-add.ll
index 5a77b9504206cbf465953cf8b19a1d0b7f87aef3..bd3ff01f9a8266cc734473a025ae2aa2eddad688 100644
--- a/llvm/test/DebugInfo/X86/sdag-salvage-add.ll
+++ b/llvm/test/DebugInfo/X86/sdag-salvage-add.ll
@@ -27,21 +27,18 @@
 ; instruction selection as it is folded into the load. As a result, we should
 ; refer to s4 and myVar with complex expressions.
 ;
-; NB: instruction referencing and DBG_VALUE modes produce debug insts in a
-; different order.
-;
 ; CHECK:         ![[S4:.*]] = !DILocalVariable(name: "s4",
 ; CHECK:         ![[MYVAR:.*]] = !DILocalVariable(name: "myVar",
 ; CHECK:         $rax = MOV64rm
 ; INSTRREF-SAME: debug-instr-number 2,
 ; INSTRREF-NEXT: DBG_INSTR_REF ![[S4]],
-; DBGVALUE-NEXT: DBG_VALUE $rax, $noreg, ![[MYVAR]],
+; DBGVALUE-NEXT: DBG_VALUE $rax, $noreg, ![[S4]],
 ; DBGVALUE-SAME:       !DIExpression(DW_OP_plus_uconst, 4096, DW_OP_stack_value)
 ; INSTRREF-SAME:       !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_plus_uconst, 4096, DW_OP_stack_value)
 ; INSTRREF-SAME: dbg-instr-ref(2, 0)
 
 ; INSTRREF:      DBG_INSTR_REF ![[MYVAR]],
-; DBGVALUE:      DBG_VALUE $rax, $noreg, ![[S4]],
+; DBGVALUE:      DBG_VALUE $rax, $noreg, ![[MYVAR]],
 ; DBGVALUE-SAME:           !DIExpression(DW_OP_plus_uconst, 4096, DW_OP_stack_value)
 ; INSTRREF-SAME:           !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_plus_uconst, 4096, DW_OP_stack_value)
 ; INSTRREF-SAME: dbg-instr-ref(2, 0)
diff --git a/llvm/test/Instrumentation/InstrProfiling/profiling.ll b/llvm/test/Instrumentation/InstrProfiling/profiling.ll
index e7678a9dce08919c0e8e4ed2cfbc17995207293c..74dd54cef50fb5c2523334afdc61e6af1cce7131 100644
--- a/llvm/test/Instrumentation/InstrProfiling/profiling.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/profiling.ll
@@ -114,11 +114,6 @@ declare void @llvm.instrprof.increment(ptr, i64, i32, i32)
 ; PS:   %[[REG:.*]] = load i32, ptr @__llvm_profile_runtime
 ; XCOFF-NOT: define .* __llvm_profile_runtime_user
 
-; WASM:      define internal void @__llvm_profile_register_functions() unnamed_addr {
-; WASM-NEXT:   call void @__llvm_profile_register_function(ptr @__profd_foo)
-; WASM-NEXT:   call void @__llvm_profile_register_function(ptr @__profd_foo_weak)
-; WASM:        call void @__llvm_profile_register_names_function(ptr @__llvm_prf_nm
-; WASM-NEXT:   ret void
-; WASM-NEXT: }
+; WASM-NOT:   internal void @__llvm_profile_register_functions()
 
 ; XCOFF-NOT:  internal void @__llvm_profile_register_functions() 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll b/llvm/test/Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8c23d9542242648ba787c41b798c901fda6c36df
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "E-m:m-i8:8:32-i16:16:32-i64:64-n32:64-S128"
+target triple = "mips64--linux"
+
+define i32 @foo(i32 %guard, ...) {
+  %vl = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.va_start(ptr %vl)
+  call void @llvm.va_end(ptr %vl)
+  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  ret i32 0
+}
+
+; First, check allocation of the save area.
+
+; CHECK-LABEL: @foo
+; CHECK: [[A:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls
+; CHECK: [[B:%.*]] = add i64 0, [[A]]
+; CHECK: [[C:%.*]] = alloca {{.*}} [[B]]
+
+; CHECK: call void @llvm.memset.p0.i64(ptr align 8 [[C]], i8 0, i64 [[B]], i1 false)
+
+; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 800)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
+
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.va_start(ptr) #2
+declare void @llvm.va_end(ptr) #2
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+
+define i32 @bar() {
+  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; Save the incoming shadow value from the arguments in the __msan_va_arg_tls
+; array.  The first argument is stored at position 4, since it's right
+; justified.
+; CHECK-LABEL: @bar
+; CHECK: store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 4) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check multiple fixed arguments.
+declare i32 @foo2(i32 %g1, i32 %g2, ...)
+define i32 @bar2() {
+  %1 = call i32 (i32, i32, ...) @foo2(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar2
+; CHECK: store i64 0, ptr @__msan_va_arg_tls, align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
+; passed to a variadic function.
+define dso_local i64 @many_args() {
+entry:
+  %ret = call i64 (i64, ...) @sum(i64 120,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1
+  )
+  ret i64 %ret
+}
+
+; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed.
+; CHECK-LABEL: @many_args
+; CHECK: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 792)
+; CHECK-NOT: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 800)
+declare i64 @sum(i64 %n, ...)
+
+; CHECK: declare void @__msan_maybe_warning_1(i8 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_1(i8 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_2(i16 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_2(i16 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_4(i32 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_4(i32 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_8(i64 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_8(i64 signext, ptr, i32 signext)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8c23d9542242648ba787c41b798c901fda6c36df
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "E-m:m-i8:8:32-i16:16:32-i64:64-n32:64-S128"
+target triple = "mips64--linux"
+
+define i32 @foo(i32 %guard, ...) {
+  %vl = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.va_start(ptr %vl)
+  call void @llvm.va_end(ptr %vl)
+  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  ret i32 0
+}
+
+; First, check allocation of the save area.
+
+; CHECK-LABEL: @foo
+; CHECK: [[A:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls
+; CHECK: [[B:%.*]] = add i64 0, [[A]]
+; CHECK: [[C:%.*]] = alloca {{.*}} [[B]]
+
+; CHECK: call void @llvm.memset.p0.i64(ptr align 8 [[C]], i8 0, i64 [[B]], i1 false)
+
+; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 800)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
+
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.va_start(ptr) #2
+declare void @llvm.va_end(ptr) #2
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+
+define i32 @bar() {
+  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; Save the incoming shadow value from the arguments in the __msan_va_arg_tls
+; array.  The first argument is stored at position 4, since it's right
+; justified.
+; CHECK-LABEL: @bar
+; CHECK: store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 4) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check multiple fixed arguments.
+declare i32 @foo2(i32 %g1, i32 %g2, ...)
+define i32 @bar2() {
+  %1 = call i32 (i32, i32, ...) @foo2(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar2
+; CHECK: store i64 0, ptr @__msan_va_arg_tls, align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
+; passed to a variadic function.
+define dso_local i64 @many_args() {
+entry:
+  %ret = call i64 (i64, ...) @sum(i64 120,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1
+  )
+  ret i64 %ret
+}
+
+; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed.
+; CHECK-LABEL: @many_args
+; CHECK: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 792)
+; CHECK-NOT: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 800)
+declare i64 @sum(i64 %n, ...)
+
+; CHECK: declare void @__msan_maybe_warning_1(i8 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_1(i8 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_2(i16 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_2(i16 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_4(i32 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_4(i32 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_8(i64 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_8(i64 signext, ptr, i32 signext)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll
new file mode 100644
index 0000000000000000000000000000000000000000..17f4b826be0beefdab6b24ac8902af87fa0194b8
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll
@@ -0,0 +1,82 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:m-i8:8:32-i16:16:32-i64:64-n32:64-S128"
+target triple = "mips64el--linux"
+
+define i32 @foo(i32 %guard, ...) {
+  %vl = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.va_start(ptr %vl)
+  call void @llvm.va_end(ptr %vl)
+  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  ret i32 0
+}
+
+; First, check allocation of the save area.
+
+; CHECK-LABEL: @foo
+; CHECK: [[A:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls
+; CHECK: [[B:%.*]] = add i64 0, [[A]]
+; CHECK: [[C:%.*]] = alloca {{.*}} [[B]]
+
+; CHECK: call void @llvm.memset.p0.i64(ptr align 8 [[C]], i8 0, i64 [[B]], i1 false)
+
+; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 800)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
+
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.va_start(ptr) #2
+declare void @llvm.va_end(ptr) #2
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+
+define i32 @bar() {
+  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; Save the incoming shadow value from the arguments in the __msan_va_arg_tls
+; array.
+; CHECK-LABEL: @bar
+; CHECK: store i32 0, ptr @__msan_va_arg_tls, align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check multiple fixed arguments.
+declare i32 @foo2(i32 %g1, i32 %g2, ...)
+define i32 @bar2() {
+  %1 = call i32 (i32, i32, ...) @foo2(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar2
+; CHECK: store i64 0, ptr @__msan_va_arg_tls, align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
+; passed to a variadic function.
+define dso_local i64 @many_args() {
+entry:
+  %ret = call i64 (i64, ...) @sum(i64 120,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1
+  )
+  ret i64 %ret
+}
+
+; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed.
+; CHECK-LABEL: @many_args
+; CHECK: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 792)
+; CHECK-NOT: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 800)
+declare i64 @sum(i64 %n, ...)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/kernel-ppcle.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/kernel-ppcle.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b4e472a134abd951cc139a21b8760c4fa70b7015
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/kernel-ppcle.ll
@@ -0,0 +1,149 @@
+; RUN: opt < %s -S -msan-kernel=1 -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le--linux"
+
+define void @Store1(ptr %p, i8 %x) sanitize_memory {
+entry:
+  store i8 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store1(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_store_1(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i8 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store2(ptr %p, i16 %x) sanitize_memory {
+entry:
+  store i16 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store2(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_store_2(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i16 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store4(ptr %p, i32 %x) sanitize_memory {
+entry:
+  store i32 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store4(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_store_4(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i32 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store8(ptr %p, i64 %x) sanitize_memory {
+entry:
+  store i64 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store8(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_store_8(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i64 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store16(ptr %p, i128 %x) sanitize_memory {
+entry:
+  store i128 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store16(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_store_n(ptr %p, i64 16)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i128 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define i8 @Load1(ptr %p) sanitize_memory {
+entry:
+  %0 = load i8, ptr %p
+  ret i8 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load1(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_load_1(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i8, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i8 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i8 {{.+}}
+
+define i16 @Load2(ptr %p) sanitize_memory {
+entry:
+  %0 = load i16, ptr %p
+  ret i16 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load2(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_load_2(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i16, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i16 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i16 {{.+}}
+
+define i32 @Load4(ptr %p) sanitize_memory {
+entry:
+  %0 = load i32, ptr %p
+  ret i32 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load4(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_load_4(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i32, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i32 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i32 {{.+}}
+
+define i64 @Load8(ptr %p) sanitize_memory {
+entry:
+  %0 = load i64, ptr %p
+  ret i64 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load8(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_load_8(ptr %p)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i64, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i64 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i64 {{.+}}
+
+define i128 @Load16(ptr %p) sanitize_memory {
+entry:
+  %0 = load i128, ptr %p
+  ret i128 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load16(
+; CHECK: [[META:%[a-z0-9_]+]] = call { ptr, ptr } @__msan_metadata_ptr_for_load_n(ptr %p, i64 16)
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i128, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i128 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i128 {{.+}}
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll
new file mode 100644
index 0000000000000000000000000000000000000000..db09c5a4771868505aa7e31d41b422dc6d1134da
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll
@@ -0,0 +1,125 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64--linux"
+
+define i32 @foo(i32 %guard, ...) {
+  %vl = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.va_start(ptr %vl)
+  call void @llvm.va_end(ptr %vl)
+  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  ret i32 0
+}
+
+; First, check allocation of the save area.
+
+; CHECK-LABEL: @foo
+; CHECK: [[A:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls
+; CHECK: [[B:%.*]] = add i64 0, [[A]]
+; CHECK: [[C:%.*]] = alloca {{.*}} [[B]]
+
+; CHECK: call void @llvm.memset.p0.i64(ptr align 8 [[C]], i8 0, i64 [[B]], i1 false)
+
+; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 800)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
+
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.va_start(ptr) #2
+declare void @llvm.va_end(ptr) #2
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+
+define i32 @bar() {
+  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; Save the incoming shadow value from the arguments in the __msan_va_arg_tls
+; array.  The first argument is stored at position 4, since it's right
+; justified.
+; CHECK-LABEL: @bar
+; CHECK: store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 4) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check vector argument.
+define i32 @bar2() {
+  %1 = call i32 (i32, ...) @foo(i32 0, <2 x i64> <i64 1, i64 2>)
+  ret i32 %1
+}
+
+; The vector is at offset 16 of parameter save area, but __msan_va_arg_tls
+; corresponds to offset 8+ of parameter save area - so the offset from
+; __msan_va_arg_tls is actually misaligned.
+; CHECK-LABEL: @bar2
+; CHECK: store <2 x i64> zeroinitializer, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check i64 array.
+define i32 @bar4() {
+  %1 = call i32 (i32, ...) @foo(i32 0, [2 x i64] [i64 1, i64 2])
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar4
+; CHECK: store [2 x i64] zeroinitializer, ptr @__msan_va_arg_tls, align 8
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check i128 array.
+define i32 @bar5() {
+  %1 = call i32 (i32, ...) @foo(i32 0, [2 x i128] [i128 1, i128 2])
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar5
+; CHECK: store [2 x i128] zeroinitializer, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 40, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check 8-aligned byval.
+define i32 @bar6(ptr %arg) {
+  %1 = call i32 (i32, ...) @foo(i32 0, ptr byval([2 x i64]) align 8 %arg)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar6
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 @__msan_va_arg_tls, ptr align 8 {{.*}}, i64 16, i1 false)
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check 16-aligned byval.
+define i32 @bar7(ptr %arg) {
+  %1 = call i32 (i32, ...) @foo(i32 0, ptr byval([4 x i64]) align 16 %arg)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar7
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), ptr align 8 {{.*}}, i64 32, i1 false)
+; CHECK: store {{.*}} 40, {{.*}} @__msan_va_arg_overflow_size_tls
+
+
+; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
+; passed to a variadic function.
+define dso_local i64 @many_args() {
+entry:
+  %ret = call i64 (i64, ...) @sum(i64 120,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1
+  )
+  ret i64 %ret
+}
+
+; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed.
+; CHECK-LABEL: @many_args
+; CHECK: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 792)
+; CHECK-NOT: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 800)
+declare i64 @sum(i64 %n, ...)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll
new file mode 100644
index 0000000000000000000000000000000000000000..63e11dc7cadd090427b0a744e8e8ea9bd1859f67
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll
@@ -0,0 +1,123 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le--linux"
+
+define i32 @foo(i32 %guard, ...) {
+  %vl = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.va_start(ptr %vl)
+  call void @llvm.va_end(ptr %vl)
+  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  ret i32 0
+}
+
+; First, check allocation of the save area.
+
+; CHECK-LABEL: @foo
+; CHECK: [[A:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls
+; CHECK: [[B:%.*]] = add i64 0, [[A]]
+; CHECK: [[C:%.*]] = alloca {{.*}} [[B]]
+
+; CHECK: call void @llvm.memset.p0.i64(ptr align 8 [[C]], i8 0, i64 [[B]], i1 false)
+
+; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 800)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
+
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.va_start(ptr) #2
+declare void @llvm.va_end(ptr) #2
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+
+define i32 @bar() {
+  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; Save the incoming shadow value from the arguments in the __msan_va_arg_tls
+; array.
+; CHECK-LABEL: @bar
+; CHECK: store i32 0, ptr @__msan_va_arg_tls, align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check vector argument.
+define i32 @bar2() {
+  %1 = call i32 (i32, ...) @foo(i32 0, <2 x i64> <i64 1, i64 2>)
+  ret i32 %1
+}
+
+; The vector is at offset 16 of parameter save area, but __msan_va_arg_tls
+; corresponds to offset 8+ of parameter save area - so the offset from
+; __msan_va_arg_tls is actually misaligned.
+; CHECK-LABEL: @bar2
+; CHECK: store <2 x i64> zeroinitializer, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check i64 array.
+define i32 @bar4() {
+  %1 = call i32 (i32, ...) @foo(i32 0, [2 x i64] [i64 1, i64 2])
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar4
+; CHECK: store [2 x i64] zeroinitializer, ptr @__msan_va_arg_tls, align 8
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check i128 array.
+define i32 @bar5() {
+  %1 = call i32 (i32, ...) @foo(i32 0, [2 x i128] [i128 1, i128 2])
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar5
+; CHECK: store [2 x i128] zeroinitializer, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 40, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check 8-aligned byval.
+define i32 @bar6(ptr %arg) {
+  %1 = call i32 (i32, ...) @foo(i32 0, ptr byval([2 x i64]) align 8 %arg)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar6
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 @__msan_va_arg_tls, ptr align 8 {{.*}}, i64 16, i1 false)
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check 16-aligned byval.
+define i32 @bar7(ptr %arg) {
+  %1 = call i32 (i32, ...) @foo(i32 0, ptr byval([4 x i64]) align 16 %arg)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar7
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), ptr align 8 {{.*}}, i64 32, i1 false)
+; CHECK: store {{.*}} 40, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
+; passed to a variadic function.
+define dso_local i64 @many_args() {
+entry:
+  %ret = call i64 (i64, ...) @sum(i64 120,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1
+  )
+  ret i64 %ret
+}
+
+; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed.
+; CHECK-LABEL: @many_args
+; CHECK: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 792)
+; CHECK-NOT: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 800)
+declare i64 @sum(i64 %n, ...)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll b/llvm/test/Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8c23d9542242648ba787c41b798c901fda6c36df
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "E-m:m-i8:8:32-i16:16:32-i64:64-n32:64-S128"
+target triple = "mips64--linux"
+
+define i32 @foo(i32 %guard, ...) {
+  %vl = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.va_start(ptr %vl)
+  call void @llvm.va_end(ptr %vl)
+  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  ret i32 0
+}
+
+; First, check allocation of the save area.
+
+; CHECK-LABEL: @foo
+; CHECK: [[A:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls
+; CHECK: [[B:%.*]] = add i64 0, [[A]]
+; CHECK: [[C:%.*]] = alloca {{.*}} [[B]]
+
+; CHECK: call void @llvm.memset.p0.i64(ptr align 8 [[C]], i8 0, i64 [[B]], i1 false)
+
+; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 800)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
+
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.va_start(ptr) #2
+declare void @llvm.va_end(ptr) #2
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+
+define i32 @bar() {
+  %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; Save the incoming shadow value from the arguments in the __msan_va_arg_tls
+; array.  The first argument is stored at position 4, since it's right
+; justified.
+; CHECK-LABEL: @bar
+; CHECK: store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 4) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK: store {{.*}} 24, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Check multiple fixed arguments.
+declare i32 @foo2(i32 %g1, i32 %g2, ...)
+define i32 @bar2() {
+  %1 = call i32 (i32, i32, ...) @foo2(i32 0, i32 1, i64 2, double 3.000000e+00)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @bar2
+; CHECK: store i64 0, ptr @__msan_va_arg_tls, align 8
+; CHECK: store i64 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 8) to ptr), align 8
+; CHECK: store {{.*}} 16, {{.*}} @__msan_va_arg_overflow_size_tls
+
+; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
+; passed to a variadic function.
+define dso_local i64 @many_args() {
+entry:
+  %ret = call i64 (i64, ...) @sum(i64 120,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1
+  )
+  ret i64 %ret
+}
+
+; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed.
+; CHECK-LABEL: @many_args
+; CHECK: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 792)
+; CHECK-NOT: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 800)
+declare i64 @sum(i64 %n, ...)
+
+; CHECK: declare void @__msan_maybe_warning_1(i8 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_1(i8 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_2(i16 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_2(i16 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_4(i32 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_4(i32 signext, ptr, i32 signext)
+; CHECK: declare void @__msan_maybe_warning_8(i64 signext, i32 signext)
+; CHECK: declare void @__msan_maybe_store_origin_8(i64 signext, ptr, i32 signext)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-x86.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3d5e4005de5b10c658415e4aecbfa733dac70352
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-x86.ll
@@ -0,0 +1,1457 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes
+; RUN: opt %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <4 x double> @test_x86_avx_addsub_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_addsub_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_addsub_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_addsub_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define <4 x double> @test_x86_avx_blendv_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx_blendv_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[A2:%.*]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr <4 x i64> [[TMP4]], <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc <4 x i64> [[TMP5]] to <4 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ashr <4 x i64> [[TMP1]], <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i64> [[TMP7]] to <4 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x double> [[A1:%.*]] to <4 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x double> [[A0:%.*]] to <4 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <4 x i64> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i64> [[TMP12]], [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i64> [[TMP13]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP8]], <4 x i64> [[TMP14]], <4 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> [[A0]], <4 x double> [[A1]], <4 x double> [[A2]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double>, <4 x double>, <4 x double>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_blendv_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx_blendv_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[A2:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr <8 x i32> [[TMP4]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc <8 x i32> [[TMP5]] to <8 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ashr <8 x i32> [[TMP1]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <8 x i32> [[TMP7]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP6]], <8 x i32> [[TMP2]], <8 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x float> [[A1:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x float> [[A0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <8 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[TMP12]], [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP13]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP8]], <8 x i32> [[TMP14]], <8 x i32> [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> [[A0]], <8 x float> [[A1]], <8 x float> [[A2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone
+
+
+define <4 x double> @test_x86_avx_cmp_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_cmp_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i64>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <4 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double> %a0, <4 x double> %a1, i8 7) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_cmp_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_cmp_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <8 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 7) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_x86_avx_cmp_ps_256_pseudo_op(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_cmp_ps_256_pseudo_op(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <8 x i1> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[A2:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <8 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = sext <8 x i1> [[TMP7]] to <8 x i32>
+; CHECK-NEXT:    [[A3:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A2]], i8 1)
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP1]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <8 x i1> [[TMP10]] to <8 x i32>
+; CHECK-NEXT:    [[A4:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A3]], i8 2)
+; CHECK-NEXT:    [[TMP12:%.*]] = or <8 x i32> [[TMP1]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <8 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = sext <8 x i1> [[TMP13]] to <8 x i32>
+; CHECK-NEXT:    [[A5:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A4]], i8 3)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP1]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <8 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <8 x i1> [[TMP16]] to <8 x i32>
+; CHECK-NEXT:    [[A6:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A5]], i8 4)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i32> [[TMP1]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[A7:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A6]], i8 5)
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i32> [[TMP1]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp ne <8 x i32> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = sext <8 x i1> [[TMP22]] to <8 x i32>
+; CHECK-NEXT:    [[A8:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A7]], i8 6)
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP1]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = sext <8 x i1> [[TMP25]] to <8 x i32>
+; CHECK-NEXT:    [[A9:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A8]], i8 7)
+; CHECK-NEXT:    [[TMP27:%.*]] = or <8 x i32> [[TMP1]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ne <8 x i32> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = sext <8 x i1> [[TMP28]] to <8 x i32>
+; CHECK-NEXT:    [[A10:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A9]], i8 8)
+; CHECK-NEXT:    [[TMP30:%.*]] = or <8 x i32> [[TMP1]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ne <8 x i32> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP32:%.*]] = sext <8 x i1> [[TMP31]] to <8 x i32>
+; CHECK-NEXT:    [[A11:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A10]], i8 9)
+; CHECK-NEXT:    [[TMP33:%.*]] = or <8 x i32> [[TMP1]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <8 x i32> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = sext <8 x i1> [[TMP34]] to <8 x i32>
+; CHECK-NEXT:    [[A12:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A11]], i8 10)
+; CHECK-NEXT:    [[TMP36:%.*]] = or <8 x i32> [[TMP1]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <8 x i32> [[TMP36]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = sext <8 x i1> [[TMP37]] to <8 x i32>
+; CHECK-NEXT:    [[A13:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A12]], i8 11)
+; CHECK-NEXT:    [[TMP39:%.*]] = or <8 x i32> [[TMP1]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp ne <8 x i32> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP41:%.*]] = sext <8 x i1> [[TMP40]] to <8 x i32>
+; CHECK-NEXT:    [[A14:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A13]], i8 12)
+; CHECK-NEXT:    [[TMP42:%.*]] = or <8 x i32> [[TMP1]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <8 x i32> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = sext <8 x i1> [[TMP43]] to <8 x i32>
+; CHECK-NEXT:    [[A15:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A14]], i8 13)
+; CHECK-NEXT:    [[TMP45:%.*]] = or <8 x i32> [[TMP1]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <8 x i32> [[TMP45]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = sext <8 x i1> [[TMP46]] to <8 x i32>
+; CHECK-NEXT:    [[A16:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A15]], i8 14)
+; CHECK-NEXT:    [[TMP48:%.*]] = or <8 x i32> [[TMP1]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp ne <8 x i32> [[TMP48]], zeroinitializer
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <8 x i1> [[TMP49]] to <8 x i32>
+; CHECK-NEXT:    [[A17:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A16]], i8 15)
+; CHECK-NEXT:    [[TMP51:%.*]] = or <8 x i32> [[TMP1]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <8 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <8 x i1> [[TMP52]] to <8 x i32>
+; CHECK-NEXT:    [[A18:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A17]], i8 16)
+; CHECK-NEXT:    [[TMP54:%.*]] = or <8 x i32> [[TMP1]], [[TMP53]]
+; CHECK-NEXT:    [[TMP55:%.*]] = icmp ne <8 x i32> [[TMP54]], zeroinitializer
+; CHECK-NEXT:    [[TMP56:%.*]] = sext <8 x i1> [[TMP55]] to <8 x i32>
+; CHECK-NEXT:    [[A19:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A18]], i8 17)
+; CHECK-NEXT:    [[TMP57:%.*]] = or <8 x i32> [[TMP1]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = icmp ne <8 x i32> [[TMP57]], zeroinitializer
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <8 x i1> [[TMP58]] to <8 x i32>
+; CHECK-NEXT:    [[A20:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A19]], i8 18)
+; CHECK-NEXT:    [[TMP60:%.*]] = or <8 x i32> [[TMP1]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <8 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <8 x i1> [[TMP61]] to <8 x i32>
+; CHECK-NEXT:    [[A21:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A20]], i8 19)
+; CHECK-NEXT:    [[TMP63:%.*]] = or <8 x i32> [[TMP1]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <8 x i32> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <8 x i1> [[TMP64]] to <8 x i32>
+; CHECK-NEXT:    [[A22:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A21]], i8 20)
+; CHECK-NEXT:    [[TMP66:%.*]] = or <8 x i32> [[TMP1]], [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = icmp ne <8 x i32> [[TMP66]], zeroinitializer
+; CHECK-NEXT:    [[TMP68:%.*]] = sext <8 x i1> [[TMP67]] to <8 x i32>
+; CHECK-NEXT:    [[A23:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A22]], i8 21)
+; CHECK-NEXT:    [[TMP69:%.*]] = or <8 x i32> [[TMP1]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = icmp ne <8 x i32> [[TMP69]], zeroinitializer
+; CHECK-NEXT:    [[TMP71:%.*]] = sext <8 x i1> [[TMP70]] to <8 x i32>
+; CHECK-NEXT:    [[A24:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A23]], i8 22)
+; CHECK-NEXT:    [[TMP72:%.*]] = or <8 x i32> [[TMP1]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp ne <8 x i32> [[TMP72]], zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = sext <8 x i1> [[TMP73]] to <8 x i32>
+; CHECK-NEXT:    [[A25:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A24]], i8 23)
+; CHECK-NEXT:    [[TMP75:%.*]] = or <8 x i32> [[TMP1]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = icmp ne <8 x i32> [[TMP75]], zeroinitializer
+; CHECK-NEXT:    [[TMP77:%.*]] = sext <8 x i1> [[TMP76]] to <8 x i32>
+; CHECK-NEXT:    [[A26:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A25]], i8 24)
+; CHECK-NEXT:    [[TMP78:%.*]] = or <8 x i32> [[TMP1]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = icmp ne <8 x i32> [[TMP78]], zeroinitializer
+; CHECK-NEXT:    [[TMP80:%.*]] = sext <8 x i1> [[TMP79]] to <8 x i32>
+; CHECK-NEXT:    [[A27:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A26]], i8 25)
+; CHECK-NEXT:    [[TMP81:%.*]] = or <8 x i32> [[TMP1]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = icmp ne <8 x i32> [[TMP81]], zeroinitializer
+; CHECK-NEXT:    [[TMP83:%.*]] = sext <8 x i1> [[TMP82]] to <8 x i32>
+; CHECK-NEXT:    [[A28:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A27]], i8 26)
+; CHECK-NEXT:    [[TMP84:%.*]] = or <8 x i32> [[TMP1]], [[TMP83]]
+; CHECK-NEXT:    [[TMP85:%.*]] = icmp ne <8 x i32> [[TMP84]], zeroinitializer
+; CHECK-NEXT:    [[TMP86:%.*]] = sext <8 x i1> [[TMP85]] to <8 x i32>
+; CHECK-NEXT:    [[A29:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A28]], i8 27)
+; CHECK-NEXT:    [[TMP87:%.*]] = or <8 x i32> [[TMP1]], [[TMP86]]
+; CHECK-NEXT:    [[TMP88:%.*]] = icmp ne <8 x i32> [[TMP87]], zeroinitializer
+; CHECK-NEXT:    [[TMP89:%.*]] = sext <8 x i1> [[TMP88]] to <8 x i32>
+; CHECK-NEXT:    [[A30:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A29]], i8 28)
+; CHECK-NEXT:    [[TMP90:%.*]] = or <8 x i32> [[TMP1]], [[TMP89]]
+; CHECK-NEXT:    [[TMP91:%.*]] = icmp ne <8 x i32> [[TMP90]], zeroinitializer
+; CHECK-NEXT:    [[TMP92:%.*]] = sext <8 x i1> [[TMP91]] to <8 x i32>
+; CHECK-NEXT:    [[A31:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A30]], i8 29)
+; CHECK-NEXT:    [[TMP93:%.*]] = or <8 x i32> [[TMP1]], [[TMP92]]
+; CHECK-NEXT:    [[TMP94:%.*]] = icmp ne <8 x i32> [[TMP93]], zeroinitializer
+; CHECK-NEXT:    [[TMP95:%.*]] = sext <8 x i1> [[TMP94]] to <8 x i32>
+; CHECK-NEXT:    [[A32:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A31]], i8 30)
+; CHECK-NEXT:    [[TMP96:%.*]] = or <8 x i32> [[TMP1]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp ne <8 x i32> [[TMP96]], zeroinitializer
+; CHECK-NEXT:    [[TMP98:%.*]] = sext <8 x i1> [[TMP97]] to <8 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> [[A0]], <8 x float> [[A32]], i8 31)
+; CHECK-NEXT:    store <8 x i32> [[TMP98]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %a2 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 0) ; <<8 x float>> [#uses=1]
+  %a3 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a2, i8 1) ; <<8 x float>> [#uses=1]
+  %a4 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a3, i8 2) ; <<8 x float>> [#uses=1]
+  %a5 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a4, i8 3) ; <<8 x float>> [#uses=1]
+  %a6 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a5, i8 4) ; <<8 x float>> [#uses=1]
+  %a7 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a6, i8 5) ; <<8 x float>> [#uses=1]
+  %a8 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a7, i8 6) ; <<8 x float>> [#uses=1]
+  %a9 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a8, i8 7) ; <<8 x float>> [#uses=1]
+  %a10 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a9, i8 8) ; <<8 x float>> [#uses=1]
+  %a11 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a10, i8 9) ; <<8 x float>> [#uses=1]
+  %a12 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a11, i8 10) ; <<8 x float>> [#uses=1]
+  %a13 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a12, i8 11) ; <<8 x float>> [#uses=1]
+  %a14 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a13, i8 12) ; <<8 x float>> [#uses=1]
+  %a15 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a14, i8 13) ; <<8 x float>> [#uses=1]
+  %a16 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a15, i8 14) ; <<8 x float>> [#uses=1]
+  %a17 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a16, i8 15) ; <<8 x float>> [#uses=1]
+  %a18 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a17, i8 16) ; <<8 x float>> [#uses=1]
+  %a19 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a18, i8 17) ; <<8 x float>> [#uses=1]
+  %a20 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a19, i8 18) ; <<8 x float>> [#uses=1]
+  %a21 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a20, i8 19) ; <<8 x float>> [#uses=1]
+  %a22 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a21, i8 20) ; <<8 x float>> [#uses=1]
+  %a23 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a22, i8 21) ; <<8 x float>> [#uses=1]
+  %a24 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a23, i8 22) ; <<8 x float>> [#uses=1]
+  %a25 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a24, i8 23) ; <<8 x float>> [#uses=1]
+  %a26 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a25, i8 24) ; <<8 x float>> [#uses=1]
+  %a27 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a26, i8 25) ; <<8 x float>> [#uses=1]
+  %a28 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a27, i8 26) ; <<8 x float>> [#uses=1]
+  %a29 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a28, i8 27) ; <<8 x float>> [#uses=1]
+  %a30 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a29, i8 28) ; <<8 x float>> [#uses=1]
+  %a31 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a30, i8 29) ; <<8 x float>> [#uses=1]
+  %a32 = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a31, i8 30) ; <<8 x float>> [#uses=1]
+  %res = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a32, i8 31) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+
+
+define <4 x float> @test_x86_avx_cvt_pd2_ps_256(<4 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_cvt_pd2_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.cvt.pd2.ps.256(<4 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx.cvt.pd2.ps.256(<4 x double> %a0) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx.cvt.pd2.ps.256(<4 x double>) nounwind readnone
+
+
+define <4 x i32> @test_x86_avx_cvt_pd2dq_256(<4 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_cvt_pd2dq_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double> %a0) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx_cvt_ps2dq_256(<8 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_cvt_ps2dq_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float> %a0) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float>) nounwind readnone
+
+
+define <4 x i32> @test_x86_avx_cvtt_pd2dq_256(<4 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_cvtt_pd2dq_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx.cvtt.pd2dq.256(<4 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx.cvtt.pd2dq.256(<4 x double> %a0) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx.cvtt.pd2dq.256(<4 x double>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx_cvtt_ps2dq_256(<8 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_cvtt_ps2dq_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx.cvtt.ps2dq.256(<8 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx.cvtt.ps2dq.256(<8 x float> %a0) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx.cvtt.ps2dq.256(<8 x float>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_dp_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_dp_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <8 x i1> <i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x i32> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSDPP:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[_MSDPP]], <8 x i1> zeroinitializer, <8 x i1> <i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>, <8 x i32> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP7]])
+; CHECK-NEXT:    [[_MSDPP1:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[_MSDPP1]], <8 x i1> zeroinitializer, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[_MSDPP2:%.*]] = sext <8 x i1> [[TMP10]] to <8 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]], i8 -18)
+; CHECK-NEXT:    store <8 x i32> [[_MSDPP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 -18) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
+
+
+define <4 x double> @test_x86_avx_hadd_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_hadd_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_hadd_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_hadd_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define <4 x double> @test_x86_avx_hsub_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_hsub_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_hsub_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_hsub_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define <32 x i8> @test_x86_avx_ldu_dq_256(ptr %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_ldu_dq_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx.ldu.dq.256(ptr [[A0]])
+; CHECK-NEXT:    store <32 x i8> [[_MSLD]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx.ldu.dq.256(ptr %a0) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx.ldu.dq.256(ptr) nounwind readonly
+
+
+define <2 x double> @test_x86_avx_maskload_pd(ptr %a0, <2 x i64> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx_maskload_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.maskload.pd(ptr [[A0:%.*]], <2 x i64> [[MASK:%.*]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx.maskload.pd(ptr %a0, <2 x i64> %mask) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx.maskload.pd(ptr, <2 x i64>) nounwind readonly
+
+
+define <4 x double> @test_x86_avx_maskload_pd_256(ptr %a0, <4 x i64> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx_maskload_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr [[A0:%.*]], <4 x i64> [[MASK:%.*]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr %a0, <4 x i64> %mask) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.maskload.pd.256(ptr, <4 x i64>) nounwind readonly
+
+
+define <4 x float> @test_x86_avx_maskload_ps(ptr %a0, <4 x i32> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx_maskload_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[A0:%.*]], <4 x i32> [[MASK:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx.maskload.ps(ptr %a0, <4 x i32> %mask) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx.maskload.ps(ptr, <4 x i32>) nounwind readonly
+
+
+define <8 x float> @test_x86_avx_maskload_ps_256(ptr %a0, <8 x i32> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx_maskload_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr [[A0:%.*]], <8 x i32> [[MASK:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr %a0, <8 x i32> %mask) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>) nounwind readonly
+
+
+define void @test_x86_avx_maskstore_pd(ptr %a0, <2 x i64> %mask, <2 x double> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx_maskstore_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd(ptr [[A0:%.*]], <2 x i64> [[MASK:%.*]], <2 x double> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx.maskstore.pd(ptr %a0, <2 x i64> %mask, <2 x double> %a2)
+  ret void
+}
+declare void @llvm.x86.avx.maskstore.pd(ptr, <2 x i64>, <2 x double>) nounwind
+
+
+define void @test_x86_avx_maskstore_pd_256(ptr %a0, <4 x i64> %mask, <4 x double> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx_maskstore_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.pd.256(ptr [[A0:%.*]], <4 x i64> [[MASK:%.*]], <4 x double> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx.maskstore.pd.256(ptr %a0, <4 x i64> %mask, <4 x double> %a2)
+  ret void
+}
+declare void @llvm.x86.avx.maskstore.pd.256(ptr, <4 x i64>, <4 x double>) nounwind
+
+
+define void @test_x86_avx_maskstore_ps(ptr %a0, <4 x i32> %mask, <4 x float> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx_maskstore_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps(ptr [[A0:%.*]], <4 x i32> [[MASK:%.*]], <4 x float> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx.maskstore.ps(ptr %a0, <4 x i32> %mask, <4 x float> %a2)
+  ret void
+}
+declare void @llvm.x86.avx.maskstore.ps(ptr, <4 x i32>, <4 x float>) nounwind
+
+
+define void @test_x86_avx_maskstore_ps_256(ptr %a0, <8 x i32> %mask, <8 x float> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx_maskstore_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx.maskstore.ps.256(ptr [[A0:%.*]], <8 x i32> [[MASK:%.*]], <8 x float> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx.maskstore.ps.256(ptr %a0, <8 x i32> %mask, <8 x float> %a2)
+  ret void
+}
+declare void @llvm.x86.avx.maskstore.ps.256(ptr, <8 x i32>, <8 x float>) nounwind
+
+
+define <4 x double> @test_x86_avx_max_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_max_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.max.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_max_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_max_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.max.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define <4 x double> @test_x86_avx_min_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_min_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.min.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.min.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.min.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_min_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_min_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.min.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define i32 @test_x86_avx_movmsk_pd_256(<4 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_movmsk_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.movmsk.pd.256(<4 x double>) nounwind readnone
+
+
+define i32 @test_x86_avx_movmsk_ps_256(<8 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_movmsk_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.movmsk.ps.256(<8 x float>) nounwind readnone
+
+
+define i32 @test_x86_avx_ptestc_256(<4 x i64> %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_ptestc_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.ptestc.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.ptestc.256(<4 x i64> %a0, <4 x i64> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.ptestc.256(<4 x i64>, <4 x i64>) nounwind readnone
+
+
+define i32 @test_x86_avx_ptestnzc_256(<4 x i64> %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_ptestnzc_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.ptestnzc.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.ptestnzc.256(<4 x i64> %a0, <4 x i64> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.ptestnzc.256(<4 x i64>, <4 x i64>) nounwind readnone
+
+
+define i32 @test_x86_avx_ptestz_256(<4 x i64> %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_ptestz_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.ptestz.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.ptestz.256(<4 x i64> %a0, <4 x i64> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.ptestz.256(<4 x i64>, <4 x i64>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_rcp_ps_256(<8 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_rcp_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float> %a0) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float>) nounwind readnone
+
+
+define <4 x double> @test_x86_avx_round_pd_256(<4 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_round_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a0, i32 7) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.round.pd.256(<4 x double>, i32) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_round_ps_256(<8 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_round_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a0, i32 7) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.round.ps.256(<8 x float>, i32) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_rsqrt_ps_256(<8 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_rsqrt_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float> %a0) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float>) nounwind readnone
+
+define <2 x double> @test_x86_avx_vpermilvar_pd(<2 x double> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vpermilvar_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A0:%.*]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double>, <2 x i64>) nounwind readnone
+
+
+define <4 x double> @test_x86_avx_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vpermilvar_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0:%.*]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double>, <4 x i64>) nounwind readnone
+
+define <4 x double> @test_x86_avx_vpermilvar_pd_256_2(<4 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx_vpermilvar_pd_256_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0:%.*]], <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 0, i64 2>) ; <<4 x double>> [#uses=1]
+  ret <4 x double> %res
+}
+
+define <4 x float> @test_x86_avx_vpermilvar_ps(<4 x float> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vpermilvar_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0:%.*]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+define <4 x float> @test_x86_avx_vpermilvar_ps_load(<4 x float> %a0, ptr %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vpermilvar_ps_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A2:%.*]] = load <4 x i32>, ptr [[A1:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0:%.*]], <4 x i32> [[A2]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a2 = load <4 x i32>, ptr %a1
+  %res = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a2) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>, <4 x i32>) nounwind readnone
+
+
+define <8 x float> @test_x86_avx_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vpermilvar_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>, <8 x i32>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestc_pd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestc_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i1> [[TMP4]] to i2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i2 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestc.pd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestc.pd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestc.pd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestc_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestc_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> %a0, <4 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestc.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestc_ps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestc_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestc.ps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestc.ps(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestc.ps(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestc_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestc_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> %a0, <8 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestc.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestnzc_pd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestnzc_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i1> [[TMP4]] to i2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i2 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestnzc.pd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestnzc.pd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestnzc.pd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestnzc_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestnzc_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestnzc.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestnzc.pd.256(<4 x double> %a0, <4 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestnzc.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestnzc_ps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestnzc_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestnzc.ps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestnzc.ps(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestnzc.ps(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestnzc_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestnzc_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestnzc.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestnzc.ps.256(<8 x float> %a0, <8 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestnzc.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestz_pd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestz_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i1> [[TMP4]] to i2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i2 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestz.pd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestz.pd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestz.pd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestz_pd_256(<4 x double> %a0, <4 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestz_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestz.pd.256(<4 x double> [[A0:%.*]], <4 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestz.pd.256(<4 x double> %a0, <4 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestz.pd.256(<4 x double>, <4 x double>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestz_ps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestz_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i1> [[TMP4]] to i4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i4 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestz.ps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestz.ps(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestz.ps(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_avx_vtestz_ps_256(<8 x float> %a0, <8 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx_vtestz_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx.vtestz.ps.256(<8 x float> [[A0:%.*]], <8 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx.vtestz.ps.256(<8 x float> %a0, <8 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx.vtestz.ps.256(<8 x float>, <8 x float>) nounwind readnone
+
+
+define void @test_x86_avx_vzeroall() #0 {
+; CHECK-LABEL: @test_x86_avx_vzeroall(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @llvm.x86.avx.vzeroall()
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx.vzeroall()
+  ret void
+}
+declare void @llvm.x86.avx.vzeroall() nounwind
+
+
+define void @test_x86_avx_vzeroupper() #0 {
+; CHECK-LABEL: @test_x86_avx_vzeroupper(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @llvm.x86.avx.vzeroupper()
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx.vzeroupper()
+  ret void
+}
+declare void @llvm.x86.avx.vzeroupper() nounwind
+
+define void @movnt_dq(ptr %p, <2 x i64> %a1) nounwind #0 {
+; CHECK-LABEL: @movnt_dq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[A2:%.*]] = add <2 x i64> [[A1:%.*]], <i64 1, i64 1>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[_MSPROP]], <2 x i64> <i64 -1, i64 -1>, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[A3:%.*]] = shufflevector <2 x i64> [[A2]], <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr [[TMP7]], align 32
+; CHECK-NEXT:    store <4 x i64> [[A3]], ptr [[P]], align 32, !nontemporal [[META1:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  %a2 = add <2 x i64> %a1, <i64 1, i64 1>
+  %a3 = shufflevector <2 x i64> %a2, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  tail call void @llvm.x86.avx.movnt.dq.256(ptr %p, <4 x i64> %a3) nounwind
+  ret void
+}
+declare void @llvm.x86.avx.movnt.dq.256(ptr, <4 x i64>) nounwind
+
+define void @movnt_ps(ptr %p, <8 x float> %a) nounwind #0 {
+; CHECK-LABEL: @movnt_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store <8 x i32> [[TMP2]], ptr [[TMP7]], align 32
+; CHECK-NEXT:    store <8 x float> [[A:%.*]], ptr [[P]], align 32, !nontemporal [[META1]]
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.movnt.ps.256(ptr %p, <8 x float> %a) nounwind
+  ret void
+}
+declare void @llvm.x86.avx.movnt.ps.256(ptr, <8 x float>) nounwind
+
+define void @movnt_pd(ptr %p, <4 x double> %a1) nounwind #0 {
+  ; add operation forces the execution domain.
+; CHECK-LABEL: @movnt_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[A2:%.*]] = fadd <4 x double> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store <4 x i64> [[_MSPROP]], ptr [[TMP7]], align 32
+; CHECK-NEXT:    store <4 x double> [[A2]], ptr [[P]], align 32, !nontemporal [[META1]]
+; CHECK-NEXT:    ret void
+;
+  %a2 = fadd <4 x double> %a1, <double 0x0, double 0x0, double 0x0, double 0x0>
+  tail call void @llvm.x86.avx.movnt.pd.256(ptr %p, <4 x double> %a2) nounwind
+  ret void
+}
+declare void @llvm.x86.avx.movnt.pd.256(ptr, <4 x double>) nounwind
+
+
+define <2 x i64> @test_x86_pclmulqdq(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_pclmulqdq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], i8 0)
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a0, <2 x i64> %a1, i8 0) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8) nounwind readnone
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-x86.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5efb7eb4078987240306b619d80843d302b4ee80
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-x86.ll
@@ -0,0 +1,2154 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <16 x i16> @test_x86_avx2_packssdw(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_packssdw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[TMP4]], <8 x i32> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %a0, <8 x i32> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_packssdw_fold() #0 {
+; CHECK-LABEL: @test_x86_avx2_packssdw_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
+  ret <16 x i16> %res
+}
+
+
+define <32 x i8> @test_x86_avx2_packsswb(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_packsswb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i16>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> [[TMP4]], <16 x i16> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %a0, <16 x i16> %a1) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <32 x i8> @test_x86_avx2_packsswb_fold() #0 {
+; CHECK-LABEL: @test_x86_avx2_packsswb_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> zeroinitializer, <16 x i16> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> <i16 0, i16 255, i16 256, i16 -1, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 -1, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
+  ret <32 x i8> %res
+}
+
+
+define <32 x i8> @test_x86_avx2_packuswb(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_packuswb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <16 x i1> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i16>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> [[TMP4]], <16 x i16> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %a0, <16 x i16> %a1) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <32 x i8> @test_x86_avx2_packuswb_fold() #0 {
+; CHECK-LABEL: @test_x86_avx2_packuswb_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> zeroinitializer, <16 x i16> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> <i16 0, i16 255, i16 256, i16 -1, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 -1, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
+  ret <32 x i8> %res
+}
+
+
+define <32 x i8> @test_x86_avx2_pavg_b(<32 x i8> %a0, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pavg_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pavg.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.pavg.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx2.pavg.b(<32 x i8>, <32 x i8>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_pavg_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pavg_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pavg.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.pavg.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.pavg.w(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_pmadd_wd(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pmadd_wd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a0, <16 x i16> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define i32 @test_x86_avx2_pmovmskb(<32 x i8> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_pmovmskb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <32 x i8> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx2.pmovmskb(<32 x i8>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_pmulh_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pmulh_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_pmulhu_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pmulhu_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx2_psad_bw(<32 x i8> %a0, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psad_bw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = lshr <4 x i64> [[TMP6]], <i64 48, i64 48, i64 48, i64 48>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %a0, <32 x i8> %a1) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_psll_d(<8 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psll_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %a0, <4 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32>, <4 x i32>) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx2_psll_q(<4 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psll_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64>, <2 x i64>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psll_w(<16 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psll_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <16 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16>, <8 x i16>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_pslli_d(<8 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_pslli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %a0, i32 7) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32>, i32) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx2_pslli_q(<4 x i64> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_pslli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <4 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64>, i32) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_pslli_w(<16 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_pslli_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %a0, i32 7) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16>, i32) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_psra_d(<8 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psra_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %a0, <4 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psra_w(<16 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psra_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <16 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_psrai_d(<8 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrai_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %a0, i32 7) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32>, i32) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psrai_w(<16 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrai_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %a0, i32 7) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16>, i32) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_psrl_d(<8 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrl_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <8 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %a0, <4 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32>, <4 x i32>) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx2_psrl_q(<4 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrl_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <4 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64>, <2 x i64>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psrl_w(<16 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrl_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i256
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i256 [[TMP6]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <16 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16>, <8 x i16>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psrl_w_load(<16 x i16> %a0, ptr %p) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrl_w_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i128 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i1 [[TMP10]] to i256
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i256 [[TMP11]] to <16 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[TMP2]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i16> [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <16 x i16> [[TMP14]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %a1 = load <8 x i16>, ptr %p
+  %res = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+
+
+define <8 x i32> @test_x86_avx2_psrli_d(<8 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %a0, i32 7) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32>, i32) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx2_psrli_q(<4 x i64> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <4 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64>, i32) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psrli_w(<16 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrli_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <16 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %a0, i32 7) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16>, i32) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_phadd_d(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_phadd_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32>, <8 x i32>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_phadd_sw(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_phadd_sw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_phadd_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_phadd_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_phsub_d(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_phsub_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32>, <8 x i32>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_phsub_sw(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_phsub_sw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_phsub_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_phsub_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_pmadd_ub_sw(<32 x i8> %a0, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pmadd_ub_sw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>) nounwind readnone
+
+define <16 x i16> @test_x86_avx2_pmadd_ub_sw_load_op0(ptr %ptr, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pmadd_ub_sw_load_op0(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A0:%.*]] = load <32 x i8>, ptr [[PTR:%.*]], align 32
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i8>, ptr [[TMP7]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i8> [[_MSLD]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i8> [[TMP8]] to <16 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <16 x i1> [[TMP10]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[TMP11]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %a0 = load <32 x i8>, ptr %ptr
+  %res = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_x86_avx2_pmul_hr_sw(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pmul_hr_sw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <32 x i8> @test_x86_avx2_pshuf_b(<32 x i8> %a0, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pshuf_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %a0, <32 x i8> %a1) ; <<16 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>) nounwind readnone
+
+
+define <32 x i8> @test_x86_avx2_psign_b(<32 x i8> %a0, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psign_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8>, <32 x i8>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_psign_d(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psign_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32> %a0, <8 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32>, <8 x i32>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psign_w(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psign_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16>, <16 x i16>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_mpsadbw(<32 x i8> %a0, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_mpsadbw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i8> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %a0, <32 x i8> %a1, i8 7) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8>, <32 x i8>, i8) nounwind readnone
+
+define <16 x i16> @test_x86_avx2_mpsadbw_load_op0(ptr %ptr, <32 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_mpsadbw_load_op0(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A0:%.*]] = load <32 x i8>, ptr [[PTR:%.*]], align 32
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i8>, ptr [[TMP7]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i8> [[_MSLD]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> [[A0]], <32 x i8> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %a0 = load <32 x i8>, ptr %ptr
+  %res = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %a0, <32 x i8> %a1, i8 7) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_x86_avx2_packusdw(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_packusdw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[TMP4]], <8 x i32> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %a0, <8 x i32> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32>, <8 x i32>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_packusdw_fold() #0 {
+; CHECK-LABEL: @test_x86_avx2_packusdw_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[RES]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
+  ret <16 x i16> %res
+}
+
+
+define <32 x i8> @test_x86_avx2_pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx2_pblendvb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr <32 x i8> [[A2:%.*]], <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc <32 x i8> [[TMP4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ashr <32 x i8> [[TMP1]], <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc <32 x i8> [[TMP6]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[TMP2]], <32 x i8> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i8> [[A1:%.*]], [[A0:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i8> [[TMP9]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i8> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP7]], <32 x i8> [[TMP11]], <32 x i8> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> [[A0]], <32 x i8> [[A1]], <32 x i8> [[A2]])
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8>, <32 x i8>, <32 x i8>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_pblendw(<16 x i16> %a0, <16 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pblendw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[TMP2]], <16 x i32> <i32 16, i32 17, i32 18, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]], <16 x i32> <i32 16, i32 17, i32 18, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %res = call <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16> %a0, <16 x i16> %a1, i8 7) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16>, <16 x i16>, i8) nounwind readnone
+
+
+define <4 x i32> @test_x86_avx2_pblendd_128(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pblendd_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]], <4 x i32> <i32 4, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.pblendd.128(<4 x i32> %a0, <4 x i32> %a1, i8 7) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx2.pblendd.128(<4 x i32>, <4 x i32>, i8) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_pblendd_256(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_pblendd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 8, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]], <8 x i32> <i32 8, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.pblendd.256(<8 x i32> %a0, <8 x i32> %a1, i8 7) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.pblendd.256(<8 x i32>, <8 x i32>, i8) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_permd(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_permd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>) nounwind readonly
+
+
+define <8 x float> @test_x86_avx2_permps(<8 x float> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_permps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx2.permps(<8 x float> [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1]
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind readonly
+
+
+define <2 x i64> @test_x86_avx2_maskload_q(ptr %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskload_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr [[A0:%.*]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx2.maskload.q(ptr %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.avx2.maskload.q(ptr, <2 x i64>) nounwind readonly
+
+
+define <4 x i64> @test_x86_avx2_maskload_q_256(ptr %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskload_q_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr [[A0:%.*]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr, <4 x i64>) nounwind readonly
+
+
+define <4 x i32> @test_x86_avx2_maskload_d(ptr %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskload_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr [[A0:%.*]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.maskload.d(ptr %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx2.maskload.d(ptr, <4 x i32>) nounwind readonly
+
+
+define <8 x i32> @test_x86_avx2_maskload_d_256(ptr %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskload_d_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF0]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr [[A0:%.*]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr, <8 x i32>) nounwind readonly
+
+
+define void @test_x86_avx2_maskstore_q(ptr %a0, <2 x i64> %a1, <2 x i64> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskstore_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q(ptr [[A0:%.*]], <2 x i64> [[A1:%.*]], <2 x i64> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx2.maskstore.q(ptr %a0, <2 x i64> %a1, <2 x i64> %a2)
+  ret void
+}
+declare void @llvm.x86.avx2.maskstore.q(ptr, <2 x i64>, <2 x i64>) nounwind
+
+
+define void @test_x86_avx2_maskstore_q_256(ptr %a0, <4 x i64> %a1, <4 x i64> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskstore_q_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.q.256(ptr [[A0:%.*]], <4 x i64> [[A1:%.*]], <4 x i64> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx2.maskstore.q.256(ptr %a0, <4 x i64> %a1, <4 x i64> %a2)
+  ret void
+}
+declare void @llvm.x86.avx2.maskstore.q.256(ptr, <4 x i64>, <4 x i64>) nounwind
+
+
+define void @test_x86_avx2_maskstore_d(ptr %a0, <4 x i32> %a1, <4 x i32> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskstore_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d(ptr [[A0:%.*]], <4 x i32> [[A1:%.*]], <4 x i32> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx2.maskstore.d(ptr %a0, <4 x i32> %a1, <4 x i32> %a2)
+  ret void
+}
+declare void @llvm.x86.avx2.maskstore.d(ptr, <4 x i32>, <4 x i32>) nounwind
+
+
+define void @test_x86_avx2_maskstore_d_256(ptr %a0, <8 x i32> %a1, <8 x i32> %a2) #0 {
+; CHECK-LABEL: @test_x86_avx2_maskstore_d_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    call void @llvm.x86.avx2.maskstore.d.256(ptr [[A0:%.*]], <8 x i32> [[A1:%.*]], <8 x i32> [[A2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx2.maskstore.d.256(ptr %a0, <8 x i32> %a1, <8 x i32> %a2)
+  ret void
+}
+declare void @llvm.x86.avx2.maskstore.d.256(ptr, <8 x i32>, <8 x i32>) nounwind
+
+
+define <4 x i32> @test_x86_avx2_psllv_d(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_x86_avx2_psllv_d_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_d_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> <i32 1, i32 1, i32 1, i32 -1>, <4 x i32> <i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <4 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES2]]
+;
+  %res0 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+  %res1 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> <i32 1, i32 1, i32 1, i32 -1>, <4 x i32> <i32 1, i32 1, i32 1,  i32 -1>)
+  %res2 = add <4 x i32> %res0, %res1
+  ret <4 x i32> %res2
+}
+declare <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32>, <4 x i32>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_psllv_d_256(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_d_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> [[TMP1]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> [[A0:%.*]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_x86_avx2_psllv_d_256_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_d_256_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> zeroinitializer, <8 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> zeroinitializer, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES2]]
+;
+  %res0 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
+  %res1 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1>)
+  %res2 = add <8 x i32> %res0, %res1
+  ret <8 x i32> %res2
+}
+declare <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
+
+
+define <2 x i64> @test_x86_avx2_psllv_q(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <2 x i1> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+define <2 x i64> @test_x86_avx2_psllv_q_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_q_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> zeroinitializer, <2 x i64> <i64 1, i64 -1>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> <i64 4, i64 -1>, <2 x i64> <i64 1, i64 -1>)
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> <i64 4, i64 -1>, <2 x i64> <i64 1, i64 -1>)
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx2_psllv_q_256(<4 x i64> %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_q_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> [[TMP1]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_x86_avx2_psllv_q_256_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psllv_q_256_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> zeroinitializer, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    store <4 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
+
+
+define <4 x i32> @test_x86_avx2_psrlv_d(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_x86_avx2_psrlv_d_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_d_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 4, i32 4, i32 4, i32 -1>, <4 x i32> <i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <4 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES2]]
+;
+  %res0 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+  %res1 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 4, i32 4, i32 4, i32 -1>, <4 x i32> <i32 1, i32 1, i32 1,  i32 -1>)
+  %res2 = add <4 x i32> %res0, %res1
+  ret <4 x i32> %res2
+}
+declare <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32>, <4 x i32>) nounwind readnone
+
+
+define <8 x i32> @test_x86_avx2_psrlv_d_256(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_d_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> [[TMP1]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> [[A0:%.*]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_x86_avx2_psrlv_d_256_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_d_256_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> zeroinitializer, <8 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1, i32 2, i32 0, i32 34, i32 -2>)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> zeroinitializer, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 -1>)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES2:%.*]] = add <8 x i32> [[RES0]], [[RES1]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES2]]
+;
+  %res0 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
+  %res1 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1>)
+  %res2 = add <8 x i32> %res0, %res1
+  ret <8 x i32> %res2
+}
+declare <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
+
+
+define <2 x i64> @test_x86_avx2_psrlv_q(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <2 x i1> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_x86_avx2_psrlv_q_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_q_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> zeroinitializer, <2 x i64> <i64 1, i64 -1>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> <i64 4, i64 4>, <2 x i64> <i64 1, i64 -1>)
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> <i64 4, i64 4>, <2 x i64> <i64 1, i64 -1>)
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define <4 x i64> @test_x86_avx2_psrlv_q_256(<4 x i64> %a0, <4 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_q_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> [[TMP1]], <4 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i64> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1]])
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
+  ret <4 x i64> %res
+}
+
+
+define <4 x i64> @test_x86_avx2_psrlv_q_256_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psrlv_q_256_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> zeroinitializer, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 4>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+; CHECK-NEXT:    store <4 x i64> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 4>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
+
+
+define <4 x i32> @test_x86_avx2_psrav_d(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrav_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_x86_avx2_psrav_d_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psrav_d_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 18, i32 35, i32 52>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> <i32 2, i32 9, i32 -12, i32 23>, <4 x i32> <i32 1, i32 18, i32 35, i32 52>)
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> <i32 2, i32 9, i32 -12, i32 23>, <4 x i32> <i32 1, i32 18, i32 35, i32 52>)
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32>, <4 x i32>) nounwind readnone
+
+define <8 x i32> @test_x86_avx2_psrav_d_256(<8 x i32> %a0, <8 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx2_psrav_d_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> [[TMP1]], <8 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> [[A0:%.*]], <8 x i32> [[A1]])
+; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_x86_avx2_psrav_d_256_const() #0 {
+; CHECK-LABEL: @test_x86_avx2_psrav_d_256_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> zeroinitializer, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>)
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> <i32 2, i32 9, i32 -12, i32 23, i32 -26, i32 37, i32 -40, i32 51>, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>)
+; CHECK-NEXT:    store <8 x i32> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> <i32 2, i32 9, i32 -12, i32 23, i32 -26, i32 37, i32 -40, i32 51>, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>)
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32>, <8 x i32>) nounwind readnone
+
+define <2 x double> @test_x86_avx2_gather_d_pd(<2 x double> %a0, ptr %a1, <4 x i32> %idx, <2 x double> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double> [[A0:%.*]], ptr [[A1:%.*]], <4 x i32> [[IDX:%.*]], <2 x double> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double> %a0,
+  ptr %a1, <4 x i32> %idx, <2 x double> %mask, i8 2) ;
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double>, ptr,
+  <4 x i32>, <2 x double>, i8) nounwind readonly
+
+define <4 x double> @test_x86_avx2_gather_d_pd_256(<4 x double> %a0, ptr %a1, <4 x i32> %idx, <4 x double> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP4]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> [[A0:%.*]], ptr [[A1:%.*]], <4 x i32> [[IDX:%.*]], <4 x double> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> %a0,
+  ptr %a1, <4 x i32> %idx, <4 x double> %mask, i8 2) ;
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double>, ptr,
+  <4 x i32>, <4 x double>, i8) nounwind readonly
+
+define <2 x double> @test_x86_avx2_gather_q_pd(<2 x double> %a0, ptr %a1, <2 x i64> %idx, <2 x double> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx2.gather.q.pd(<2 x double> [[A0:%.*]], ptr [[A1:%.*]], <2 x i64> [[IDX:%.*]], <2 x double> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.avx2.gather.q.pd(<2 x double> %a0,
+  ptr %a1, <2 x i64> %idx, <2 x double> %mask, i8 2) ;
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx2.gather.q.pd(<2 x double>, ptr,
+  <2 x i64>, <2 x double>, i8) nounwind readonly
+
+define <4 x double> @test_x86_avx2_gather_q_pd_256(<4 x double> %a0, ptr %a1, <4 x i64> %idx, <4 x double> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP4]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> [[A0:%.*]], ptr [[A1:%.*]], <4 x i64> [[IDX:%.*]], <4 x double> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  %res = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> %a0,
+  ptr %a1, <4 x i64> %idx, <4 x double> %mask, i8 2) ;
+  ret <4 x double> %res
+}
+declare <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double>, ptr,
+  <4 x i64>, <4 x double>, i8) nounwind readonly
+
+define <4 x float> @test_x86_avx2_gather_d_ps(<4 x float> %a0, ptr %a1, <4 x i32> %idx, <4 x float> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float> [[A0:%.*]], ptr [[A1:%.*]], <4 x i32> [[IDX:%.*]], <4 x float> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float> %a0,
+  ptr %a1, <4 x i32> %idx, <4 x float> %mask, i8 2) ;
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float>, ptr,
+  <4 x i32>, <4 x float>, i8) nounwind readonly
+
+define <8 x float> @test_x86_avx2_gather_d_ps_256(<8 x float> %a0, ptr %a1, <8 x i32> %idx, <8 x float> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP4]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> [[A0:%.*]], ptr [[A1:%.*]], <8 x i32> [[IDX:%.*]], <8 x float> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> %a0,
+  ptr %a1, <8 x i32> %idx, <8 x float> %mask, i8 2) ;
+  ret <8 x float> %res
+}
+declare <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float>, ptr,
+  <8 x i32>, <8 x float>, i8) nounwind readonly
+
+define <4 x float> @test_x86_avx2_gather_q_ps(<4 x float> %a0, ptr %a1, <2 x i64> %idx, <4 x float> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx2.gather.q.ps(<4 x float> [[A0:%.*]], ptr [[A1:%.*]], <2 x i64> [[IDX:%.*]], <4 x float> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx2.gather.q.ps(<4 x float> %a0,
+  ptr %a1, <2 x i64> %idx, <4 x float> %mask, i8 2) ;
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx2.gather.q.ps(<4 x float>, ptr,
+  <2 x i64>, <4 x float>, i8) nounwind readonly
+
+define <4 x float> @test_x86_avx2_gather_q_ps_256(<4 x float> %a0, ptr %a1, <4 x i64> %idx, <4 x float> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> [[A0:%.*]], ptr [[A1:%.*]], <4 x i64> [[IDX:%.*]], <4 x float> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> %a0,
+  ptr %a1, <4 x i64> %idx, <4 x float> %mask, i8 2) ;
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float>, ptr,
+  <4 x i64>, <4 x float>, i8) nounwind readonly
+
+define <2 x i64> @test_x86_avx2_gather_d_q(<2 x i64> %a0, ptr %a1, <4 x i32> %idx, <2 x i64> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.gather.d.q(<2 x i64> [[A0:%.*]], ptr [[A1:%.*]], <4 x i32> [[IDX:%.*]], <2 x i64> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx2.gather.d.q(<2 x i64> %a0,
+  ptr %a1, <4 x i32> %idx, <2 x i64> %mask, i8 2) ;
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.avx2.gather.d.q(<2 x i64>, ptr,
+  <4 x i32>, <2 x i64>, i8) nounwind readonly
+
+define <4 x i64> @test_x86_avx2_gather_d_q_256(<4 x i64> %a0, ptr %a1, <4 x i32> %idx, <4 x i64> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_q_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP4]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> [[A0:%.*]], ptr [[A1:%.*]], <4 x i32> [[IDX:%.*]], <4 x i64> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> %a0,
+  ptr %a1, <4 x i32> %idx, <4 x i64> %mask, i8 2) ;
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64>, ptr,
+  <4 x i32>, <4 x i64>, i8) nounwind readonly
+
+define <2 x i64> @test_x86_avx2_gather_q_q(<2 x i64> %a0, ptr %a1, <2 x i64> %idx, <2 x i64> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> [[A0:%.*]], ptr [[A1:%.*]], <2 x i64> [[IDX:%.*]], <2 x i64> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> %a0,
+  ptr %a1, <2 x i64> %idx, <2 x i64> %mask, i8 2) ;
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64>, ptr,
+  <2 x i64>, <2 x i64>, i8) nounwind readonly
+
+define <4 x i64> @test_x86_avx2_gather_q_q_256(<4 x i64> %a0, ptr %a1, <4 x i64> %idx, <4 x i64> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_q_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP4]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> [[A0:%.*]], ptr [[A1:%.*]], <4 x i64> [[IDX:%.*]], <4 x i64> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %res = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> %a0,
+  ptr %a1, <4 x i64> %idx, <4 x i64> %mask, i8 2) ;
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64>, ptr,
+  <4 x i64>, <4 x i64>, i8) nounwind readonly
+
+define <4 x i32> @test_x86_avx2_gather_d_d(<4 x i32> %a0, ptr %a1, <4 x i32> %idx, <4 x i32> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32> [[A0:%.*]], ptr [[A1:%.*]], <4 x i32> [[IDX:%.*]], <4 x i32> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32> %a0,
+  ptr %a1, <4 x i32> %idx, <4 x i32> %mask, i8 2) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32>, ptr,
+  <4 x i32>, <4 x i32>, i8) nounwind readonly
+
+define <8 x i32> @test_x86_avx2_gather_d_d_256(<8 x i32> %a0, ptr %a1, <8 x i32> %idx, <8 x i32> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_d_d_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP4]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> [[A0:%.*]], ptr [[A1:%.*]], <8 x i32> [[IDX:%.*]], <8 x i32> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> %a0,
+  ptr %a1, <8 x i32> %idx, <8 x i32> %mask, i8 2) ;
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32>, ptr,
+  <8 x i32>, <8 x i32>, i8) nounwind readonly
+
+define <4 x i32> @test_x86_avx2_gather_q_d(<4 x i32> %a0, ptr %a1, <2 x i64> %idx, <4 x i32> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.gather.q.d(<4 x i32> [[A0:%.*]], ptr [[A1:%.*]], <2 x i64> [[IDX:%.*]], <4 x i32> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.gather.q.d(<4 x i32> %a0,
+  ptr %a1, <2 x i64> %idx, <4 x i32> %mask, i8 2) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx2.gather.q.d(<4 x i32>, ptr,
+  <2 x i64>, <4 x i32>, i8) nounwind readonly
+
+define <4 x i32> @test_x86_avx2_gather_q_d_256(<4 x i32> %a0, ptr %a1, <4 x i64> %idx, <4 x i32> %mask) #0 {
+; CHECK-LABEL: @test_x86_avx2_gather_q_d_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> [[A0:%.*]], ptr [[A1:%.*]], <4 x i64> [[IDX:%.*]], <4 x i32> [[MASK:%.*]], i8 2)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> %a0,
+  ptr %a1, <4 x i64> %idx, <4 x i32> %mask, i8 2) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32>, ptr,
+  <4 x i64>, <4 x i32>, i8) nounwind readonly
+
+define <8 x float>  @test_gather_mask(<8 x float> %a0, ptr %a, <8 x i32> %idx, <8 x float> %mask, ptr nocapture %out) #0 {
+; CHECK-LABEL: @test_gather_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP4]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF0]]
+; CHECK:       9:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       10:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> [[A0:%.*]], ptr [[A:%.*]], <8 x i32> [[IDX:%.*]], <8 x float> [[MASK:%.*]], i8 4)
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[OUT:%.*]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP13]], 87960930222080
+; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr [[TMP15]], align 4
+; CHECK-NEXT:    store <8 x float> [[MASK]], ptr [[OUT]], align 4
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> %a0,
+  ptr %a, <8 x i32> %idx, <8 x float> %mask, i8 4) ;
+
+  store <8 x float> %mask, ptr %out, align 4
+
+  ret <8 x float> %res
+}
+
+define <2 x i64> @test_mask_demanded_bits(<2 x i64> %a0, ptr %a1, <2 x i64> %idx, <2 x i1> %mask) #0 {
+; CHECK-LABEL: @test_mask_demanded_bits(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i1>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[MASK1:%.*]] = sext <2 x i1> [[MASK:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[_MSPROP]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> [[A0:%.*]], ptr [[A1:%.*]], <2 x i64> [[IDX:%.*]], <2 x i64> [[MASK1]], i8 2)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %mask1 = sext <2 x i1> %mask to <2 x i64>
+  %res = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> %a0,
+  ptr %a1, <2 x i64> %idx, <2 x i64> %mask1, i8 2) ;
+  ret <2 x i64> %res
+}
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1d2e38eb5e63d86945931590bb476832428a01d4
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
@@ -0,0 +1,3626 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <1 x i64> @llvm.x86.ssse3.phadd.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test1(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test1(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phadd.w(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %2 = bitcast <4 x i16> %1 to <1 x i64>
+  %3 = bitcast <4 x i16> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.phadd.w(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <4 x i16>
+  %6 = bitcast <4 x i16> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.mmx.pcmpgt.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test88(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test88(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pcmpgt.d(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pcmpgt.d(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pcmpgt.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test87(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test87(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pcmpgt.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pcmpgt.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pcmpgt.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test86(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test86(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pcmpgt.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pcmpgt.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pcmpeq.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test85(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test85(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pcmpeq.d(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pcmpeq.d(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pcmpeq.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test84(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test84(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pcmpeq.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pcmpeq.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pcmpeq.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test83(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test83(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pcmpeq.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pcmpeq.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.punpckldq(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test82(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test82(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.punpckldq(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.punpckldq(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.punpcklwd(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test81(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test81(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.punpcklwd(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.punpcklwd(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.punpcklbw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test80(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test80(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.punpcklbw(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.punpcklbw(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.punpckhdq(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test79(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test79(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.punpckhdq(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.punpckhdq(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.punpckhwd(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test78(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test78(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.punpckhwd(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.punpckhwd(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.punpckhbw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test77(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test77(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.punpckhbw(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.punpckhbw(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.packuswb(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test76(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test76(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP16:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP16]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <1 x i64> [[TMP17]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <4 x i16> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP23]] to <4 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <4 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = sext <4 x i1> [[TMP12]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <1 x i64> @llvm.x86.mmx.packsswb(<1 x i64> [[TMP14]], <1 x i64> [[TMP15]])
+; CHECK-NEXT:    [[TMP24:%.*]] = tail call <1 x i64> @llvm.x86.mmx.packuswb(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[_MSPROP_VECTOR_PACK]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP24]] to <8 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i8> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i8> [[TMP4]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <1 x i64> [[TMP21]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP22]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.packuswb(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.packssdw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test75(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test75(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP16:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP16]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <1 x i64> [[TMP17]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <2 x i32> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP23]] to <2 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP7]] to <2 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <2 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <2 x i1> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <2 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = sext <2 x i1> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i32> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <1 x i64> @llvm.x86.mmx.packssdw(<1 x i64> [[TMP14]], <1 x i64> [[TMP15]])
+; CHECK-NEXT:    [[TMP24:%.*]] = tail call <1 x i64> @llvm.x86.mmx.packssdw(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[_MSPROP_VECTOR_PACK]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP24]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <4 x i16> [[TMP4]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <1 x i64> [[TMP21]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP22]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.packssdw(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.packsswb(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test74(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test74(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP16:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP16]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <1 x i64> [[TMP17]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <4 x i16> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP23]] to <4 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <4 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = sext <4 x i1> [[TMP12]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <1 x i64> @llvm.x86.mmx.packsswb(<1 x i64> [[TMP14]], <1 x i64> [[TMP15]])
+; CHECK-NEXT:    [[TMP24:%.*]] = tail call <1 x i64> @llvm.x86.mmx.packsswb(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[_MSPROP_VECTOR_PACK]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP24]] to <8 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i8> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i8> [[TMP4]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <1 x i64> [[TMP21]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP22]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.packsswb(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrai.d(<1 x i64>, i32) nounwind readnone
+
+define i64 @test73(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test73(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.psrai.d(<1 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrai.d(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.psrai.d(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to <2 x i32>
+  %3 = bitcast <2 x i32> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrai.w(<1 x i64>, i32) nounwind readnone
+
+define i64 @test72(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test72(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.psrai.w(<1 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrai.w(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.psrai.w(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to <4 x i16>
+  %3 = bitcast <4 x i16> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+define i64 @test72_2(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test72_2(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.psrai.w(<1 x i64> [[TMP1]], i32 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrai.w(<1 x i64> [[MMX_VAR_I]], i32 0) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.psrai.w(<1 x i64> %mmx_var.i, i32 0) nounwind
+  %2 = bitcast <1 x i64> %1 to <4 x i16>
+  %3 = bitcast <4 x i16> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrli.q(<1 x i64>, i32) nounwind readnone
+
+define i64 @test71(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test71(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.x86.mmx.psrli.q(<1 x i64> [[TMP2]], i32 3)
+; CHECK-NEXT:    [[TMP6:%.*]] = or <1 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrli.q(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var.i = bitcast i64 %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.psrli.q(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to i64
+  ret i64 %2
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrli.d(<1 x i64>, i32) nounwind readnone
+
+define i64 @test70(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test70(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.psrli.d(<1 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrli.d(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.psrli.d(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to <2 x i32>
+  %3 = bitcast <2 x i32> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+define i64 @test70_2(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test70_2(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.psrli.d(<1 x i64> [[TMP1]], i32 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrli.d(<1 x i64> [[MMX_VAR_I]], i32 0) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.psrli.d(<1 x i64> %mmx_var.i, i32 0) nounwind
+  %2 = bitcast <1 x i64> %1 to <2 x i32>
+  %3 = bitcast <2 x i32> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrli.w(<1 x i64>, i32) nounwind readnone
+
+define i64 @test69(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test69(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.psrli.w(<1 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrli.w(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.psrli.w(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to <4 x i16>
+  %3 = bitcast <4 x i16> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+declare <1 x i64> @llvm.x86.mmx.pslli.q(<1 x i64>, i32) nounwind readnone
+
+define i64 @test68(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test68(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.x86.mmx.pslli.q(<1 x i64> [[TMP2]], i32 3)
+; CHECK-NEXT:    [[TMP6:%.*]] = or <1 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pslli.q(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var.i = bitcast i64 %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.pslli.q(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to i64
+  ret i64 %2
+}
+
+declare <1 x i64> @llvm.x86.mmx.pslli.d(<1 x i64>, i32) nounwind readnone
+
+define i64 @test67(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test67(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.pslli.d(<1 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pslli.d(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.pslli.d(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to <2 x i32>
+  %3 = bitcast <2 x i32> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+declare <1 x i64> @llvm.x86.mmx.pslli.w(<1 x i64>, i32) nounwind readnone
+
+define i64 @test66(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test66(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.pslli.w(<1 x i64> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pslli.w(<1 x i64> [[MMX_VAR_I]], i32 3) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.pslli.w(<1 x i64> %mmx_var.i, i32 3) nounwind
+  %2 = bitcast <1 x i64> %1 to <4 x i16>
+  %3 = bitcast <4 x i16> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+define i64 @test66_2(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test66_2(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <1 x i64> @llvm.x86.mmx.pslli.w(<1 x i64> [[TMP1]], i32 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = or <1 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pslli.w(<1 x i64> [[MMX_VAR_I]], i32 0) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP11]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i16> [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = tail call <1 x i64> @llvm.x86.mmx.pslli.w(<1 x i64> %mmx_var.i, i32 0) nounwind
+  %2 = bitcast <1 x i64> %1 to <4 x i16>
+  %3 = bitcast <4 x i16> %2 to <1 x i64>
+  %4 = extractelement <1 x i64> %3, i32 0
+  ret i64 %4
+}
+
+declare <1 x i64> @llvm.x86.mmx.psra.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test65(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test65(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i1 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i64 [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.mmx.psra.d(<1 x i64> [[TMP4]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <1 x i64> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psra.d(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP21]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i32> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <1 x i64> [[TMP16]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP17]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psra.d(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psra.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test64(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test64(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i1 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i64 [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.mmx.psra.w(<1 x i64> [[TMP4]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <1 x i64> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psra.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[TMP12]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP21]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <1 x i64> [[TMP16]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP17]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psra.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrl.q(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test63(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test63(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <1 x i64> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i64 [[_MSPROP1]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i1 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = call <1 x i64> @llvm.x86.mmx.psrl.q(<1 x i64> [[TMP3]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP16:%.*]] = or <1 x i64> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP12:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrl.q(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP12]] to i64
+; CHECK-NEXT:    store i64 [[TMP11]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var.i = bitcast i64 %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psrl.q(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to i64
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrl.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test62(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test62(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i1 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i64 [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.mmx.psrl.d(<1 x i64> [[TMP4]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <1 x i64> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrl.d(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP21]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i32> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <1 x i64> [[TMP16]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP17]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psrl.d(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psrl.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test61(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test61(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i1 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i64 [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.mmx.psrl.w(<1 x i64> [[TMP4]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <1 x i64> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psrl.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[TMP12]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP21]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <1 x i64> [[TMP16]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP17]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psrl.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psll.q(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test60(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test60(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <1 x i64> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i64 [[_MSPROP1]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i1 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP10]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = call <1 x i64> @llvm.x86.mmx.psll.q(<1 x i64> [[TMP3]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP16:%.*]] = or <1 x i64> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP12:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psll.q(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP12]] to i64
+; CHECK-NEXT:    store i64 [[TMP11]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var.i = bitcast i64 %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psll.q(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to i64
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.mmx.psll.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test59(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test59(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i1 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i64 [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.mmx.psll.d(<1 x i64> [[TMP4]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <1 x i64> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psll.d(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP21]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i32> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <1 x i64> [[TMP16]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP17]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psll.d(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test58(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test58(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <1 x i64> [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i1 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i64 [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64> [[TMP4]], <1 x i64> [[MMX_VAR1_I]])
+; CHECK-NEXT:    [[TMP12:%.*]] = or <1 x i64> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[TMP12]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP21]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <1 x i64> [[TMP16]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP17]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1.i = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pxor(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test56(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test56(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pxor(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pxor(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.por(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test55(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test55(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.por(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.por(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pandn(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test54(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test54(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pandn(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pandn(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pand(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test53(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test53(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pand(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pand(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmull.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test52(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test52(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmull.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmull.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+define i64 @test51(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test51(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmull.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmull.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmulh.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test50(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test50(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmulh.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmulh.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test49(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test49(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP13:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP13]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <2 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <2 x i1> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <1 x i64> [[TMP14]] to <2 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP3]] to <1 x i64>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <2 x i32> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <1 x i64> [[TMP21]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP18]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psubus.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test48(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test48(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psubus.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psubus.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psubus.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test47(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test47(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psubus.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psubus.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psubs.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test46(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test46(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psubs.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psubs.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psubs.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test45(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test45(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psubs.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psubs.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+define i64 @test44(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test44(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP1]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <1 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psub.q(<1 x i64> [[MMX_VAR]], <1 x i64> [[MMX_VAR1]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = bitcast <1 x i64> [[_MSPROP3]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP6]] to i64
+; CHECK-NEXT:    store i64 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var = bitcast i64 %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1 = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psub.q(<1 x i64> %mmx_var, <1 x i64> %mmx_var1)
+  %3 = bitcast <1 x i64> %2 to i64
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.mmx.psub.q(<1 x i64>, <1 x i64>) nounwind readnone
+
+declare <1 x i64> @llvm.x86.mmx.psub.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test43(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test43(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psub.d(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psub.d(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psub.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test42(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test42(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psub.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psub.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psub.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test41(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test41(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psub.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psub.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.paddus.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test40(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test40(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.paddus.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.paddus.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.paddus.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test39(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test39(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.paddus.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.paddus.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.padds.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test38(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test38(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.padds.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.padds.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.padds.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test37(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test37(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.padds.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.padds.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.padd.q(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test36(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test36(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <1 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i64 [[_MSPROP1]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <1 x i64> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call <1 x i64> @llvm.x86.mmx.padd.q(<1 x i64> [[MMX_VAR]], <1 x i64> [[MMX_VAR1]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = bitcast <1 x i64> [[_MSPROP3]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP6]] to i64
+; CHECK-NEXT:    store i64 [[_MSPROP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var = bitcast i64 %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1 = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.padd.q(<1 x i64> %mmx_var, <1 x i64> %mmx_var1)
+  %3 = bitcast <1 x i64> %2 to i64
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test35(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test35(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.padd.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test34(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test34(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.padd.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.padd.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.padd.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test33(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test33(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.padd.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.padd.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.psad.bw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test32(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test32(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP5]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP12]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP4]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP16:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i1 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP10]], 48
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i64 [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = tail call <1 x i64> @llvm.x86.mmx.psad.bw(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <1 x i64> [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP14]] to i64
+; CHECK-NEXT:    store i64 [[TMP15]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.psad.bw(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to i64
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmins.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test31(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test31(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmins.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmins.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pminu.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test30(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test30(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pminu.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pminu.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmaxs.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test29(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test29(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmaxs.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmaxs.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmaxu.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test28(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test28(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmaxu.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmaxu.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pavg.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test27(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test27(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pavg.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pavg.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pavg.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test26(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test26(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pavg.b(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pavg.b(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare void @llvm.x86.mmx.movnt.dq(ptr, <1 x i64>) nounwind
+
+define void @test25(ptr %p, <1 x i64> %a) nounwind optsize ssp #0 {
+; CHECK-LABEL: define void @test25(
+; CHECK-SAME: ptr [[P:%.*]], <1 x i64> [[A:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    store <1 x i64> [[TMP3]], ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    tail call void @llvm.x86.mmx.movnt.dq(ptr [[P]], <1 x i64> [[MMX_VAR_I]]) #[[ATTR2]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var.i = bitcast i64 %0 to <1 x i64>
+  tail call void @llvm.x86.mmx.movnt.dq(ptr %p, <1 x i64> %mmx_var.i) nounwind
+  ret void
+}
+
+declare i32 @llvm.x86.mmx.pmovmskb(<1 x i64>) nounwind readnone
+
+define i32 @test24(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i32 @test24(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP2]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP6]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP4]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.x86.mmx.pmovmskb(<1 x i64> [[MMX_VAR_I]]) #[[ATTR2]]
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %0 to <1 x i64>
+  %1 = tail call i32 @llvm.x86.mmx.pmovmskb(<1 x i64> %mmx_var.i) nounwind
+  ret i32 %1
+}
+
+declare void @llvm.x86.mmx.maskmovq(<1 x i64>, <1 x i64>, ptr) nounwind
+
+define void @test23(<1 x i64> %d, <1 x i64> %n, ptr %p) nounwind optsize ssp #0 {
+; CHECK-LABEL: define void @test23(
+; CHECK-SAME: <1 x i64> [[D:%.*]], <1 x i64> [[N:%.*]], ptr [[P:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP4]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[N]] to <8 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP6]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[D]] to <8 x i8>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i8> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i8> [[TMP3]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[TMP9]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    tail call void @llvm.x86.mmx.maskmovq(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]], ptr [[P]]) #[[ATTR2]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast <1 x i64> %n to <8 x i8>
+  %1 = bitcast <1 x i64> %d to <8 x i8>
+  %mmx_var.i = bitcast <8 x i8> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <8 x i8> %0 to <1 x i64>
+  tail call void @llvm.x86.mmx.maskmovq(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i, ptr %p) nounwind
+  ret void
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmulhu.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test22(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test22(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP8]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP14]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmulhu.w(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP15]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <1 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP13]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %mmx_var.i = bitcast <4 x i16> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmulhu.w(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64>, i8) nounwind readnone
+
+define i64 @test21(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test21(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP13:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP13]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %1 = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> %1, i8 3) nounwind readnone
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+define i32 @test21_2(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i32 @test21_2(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP13:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP13]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %1 = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> %1, i8 3) nounwind readnone
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <2 x i32>
+  %5 = extractelement <2 x i32> %4, i32 0
+  ret i32 %5
+}
+
+declare <1 x i64> @llvm.x86.mmx.pmulu.dq(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test20(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test20(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP5]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP4]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i32> [[TMP9]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmulu.dq(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
+; CHECK-NEXT:    [[_MSPROP:%.*]] = bitcast <1 x i64> [[_MSPROP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
+; CHECK-NEXT:    store i64 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %mmx_var.i = bitcast <2 x i32> %1 to <1 x i64>
+  %mmx_var1.i = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.pmulu.dq(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i) nounwind
+  %3 = bitcast <1 x i64> %2 to i64
+  ret i64 %3
+}
+
+declare <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64>) nounwind readnone
+
+define <2 x double> @test19(<1 x i64> %a) #0 {
+; CHECK-LABEL: define <2 x double> @test19(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[TMP4]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP7]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP5]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64> [[TMP8]]) #[[ATTR5]]
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %1 = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64> %1) nounwind readnone
+  ret <2 x double> %2
+}
+
+declare <1 x i64> @llvm.x86.sse.cvttpd2pi(<2 x double>) nounwind readnone
+
+define i64 @test18(<2 x double> %a) #0 {
+; CHECK-LABEL: define i64 @test18(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP8:%.*]] = tail call <1 x i64> @llvm.x86.sse.cvttpd2pi(<2 x double> [[A]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <1 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+entry:
+  %0 = tail call <1 x i64> @llvm.x86.sse.cvttpd2pi(<2 x double> %a) nounwind readnone
+  %1 = bitcast <1 x i64> %0 to <2 x i32>
+  %2 = bitcast <2 x i32> %1 to <1 x i64>
+  %3 = extractelement <1 x i64> %2, i32 0
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double>) nounwind readnone
+
+define i64 @test17(<2 x double> %a) #0 {
+; CHECK-LABEL: define i64 @test17(
+; CHECK-SAME: <2 x double> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF0]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP8:%.*]] = tail call <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double> [[A]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <1 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+entry:
+  %0 = tail call <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double> %a) nounwind readnone
+  %1 = bitcast <1 x i64> %0 to <2 x i32>
+  %2 = bitcast <2 x i32> %1 to <1 x i64>
+  %3 = extractelement <1 x i64> %2, i32 0
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.mmx.palignr.b(<1 x i64>, <1 x i64>, i8) nounwind readnone
+
+define i64 @test16(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test16(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP6:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <1 x i64> [[A]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64 [[_MSPROP]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR:%.*]] = bitcast i64 [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = extractelement <1 x i64> [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> [[B]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[_MSPROP1]] to <1 x i64>
+; CHECK-NEXT:    [[MMX_VAR1:%.*]] = bitcast i64 [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP4]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP5]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP12]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[TMP10:%.*]] = tail call <1 x i64> @llvm.x86.mmx.palignr.b(<1 x i64> [[MMX_VAR]], <1 x i64> [[MMX_VAR1]], i8 16)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
+; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+entry:
+  %0 = extractelement <1 x i64> %a, i32 0
+  %mmx_var = bitcast i64 %0 to <1 x i64>
+  %1 = extractelement <1 x i64> %b, i32 0
+  %mmx_var1 = bitcast i64 %1 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.mmx.palignr.b(<1 x i64> %mmx_var, <1 x i64> %mmx_var1, i8 16)
+  %3 = bitcast <1 x i64> %2 to i64
+  ret i64 %3
+}
+
+declare <1 x i64> @llvm.x86.ssse3.pabs.d(<1 x i64>) nounwind readnone
+
+define i64 @test15(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test15(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x i32> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pabs.d(<1 x i64> [[TMP1]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP11]] to <2 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x i32> [[TMP6]] to <1 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i32> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <1 x i64> [[TMP9]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP10]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <2 x i32>
+  %1 = bitcast <2 x i32> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.ssse3.pabs.d(<1 x i64> %1) nounwind readnone
+  %3 = bitcast <1 x i64> %2 to <2 x i32>
+  %4 = bitcast <2 x i32> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.ssse3.pabs.w(<1 x i64>) nounwind readnone
+
+define i64 @test14(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test14(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pabs.w(<1 x i64> [[TMP1]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP11]] to <4 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP12]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP6]] to <1 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <1 x i64> [[TMP9]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP10]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <4 x i16>
+  %1 = bitcast <4 x i16> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.ssse3.pabs.w(<1 x i64> %1) nounwind readnone
+  %3 = bitcast <1 x i64> %2 to <4 x i16>
+  %4 = bitcast <4 x i16> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.ssse3.pabs.b(<1 x i64>) nounwind readnone
+
+define i64 @test13(<1 x i64> %a) #0 {
+; CHECK-LABEL: define i64 @test13(
+; CHECK-SAME: <1 x i64> [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[TMP7]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i8> [[TMP8]] to <1 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pabs.b(<1 x i64> [[TMP1]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP11]] to <8 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP12]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP6]] to <1 x i64>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i8> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <1 x i64> [[TMP9]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP10]]
+;
+entry:
+  %0 = bitcast <1 x i64> %a to <8 x i8>
+  %1 = bitcast <8 x i8> %0 to <1 x i64>
+  %2 = tail call <1 x i64> @llvm.x86.ssse3.pabs.b(<1 x i64> %1) nounwind readnone
+  %3 = bitcast <1 x i64> %2 to <8 x i8>
+  %4 = bitcast <8 x i8> %3 to <1 x i64>
+  %5 = extractelement <1 x i64> %4, i32 0
+  ret i64 %5
+}
+
+declare <1 x i64> @llvm.x86.ssse3.psign.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test12(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test12(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i32> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.psign.d(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i32> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %2 = bitcast <2 x i32> %1 to <1 x i64>
+  %3 = bitcast <2 x i32> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.psign.d(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <2 x i32>
+  %6 = bitcast <2 x i32> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.psign.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test11(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test11(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.psign.w(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %2 = bitcast <4 x i16> %1 to <1 x i64>
+  %3 = bitcast <4 x i16> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.psign.w(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <4 x i16>
+  %6 = bitcast <4 x i16> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.psign.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test10(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test10(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i8> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.psign.b(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i8> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %2 = bitcast <8 x i8> %1 to <1 x i64>
+  %3 = bitcast <8 x i8> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.psign.b(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <8 x i8>
+  %6 = bitcast <8 x i8> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.pshuf.b(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test9(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test9(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i8> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pshuf.b(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x i8> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %2 = bitcast <8 x i8> %1 to <1 x i64>
+  %3 = bitcast <8 x i8> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.pshuf.b(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <8 x i8>
+  %6 = bitcast <8 x i8> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.pmul.hr.sw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test8(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test8(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pmul.hr.sw(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %2 = bitcast <4 x i16> %1 to <1 x i64>
+  %3 = bitcast <4 x i16> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.pmul.hr.sw(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <4 x i16>
+  %6 = bitcast <4 x i16> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test7(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test7(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <1 x i64> [[TMP9]] to <8 x i8>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <1 x i64> [[TMP15]] to <8 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i8> [[TMP18]] to <1 x i64>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP17]] to <1 x i64>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = or <1 x i64> [[TMP21]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = sext <4 x i1> [[TMP12]] to <4 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP24:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64> [[TMP22]], <1 x i64> [[TMP23]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP14]] to <8 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP24]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i8> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <1 x i64> [[TMP19]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP20]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <8 x i8>
+  %1 = bitcast <1 x i64> %a to <8 x i8>
+  %2 = bitcast <8 x i8> %1 to <1 x i64>
+  %3 = bitcast <8 x i8> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <8 x i8>
+  %6 = bitcast <8 x i8> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.phsub.sw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test6(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test6(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.sw(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %2 = bitcast <4 x i16> %1 to <1 x i64>
+  %3 = bitcast <4 x i16> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.phsub.sw(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <4 x i16>
+  %6 = bitcast <4 x i16> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.phsub.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test5(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test5(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i32> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.d(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i32> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %2 = bitcast <2 x i32> %1 to <1 x i64>
+  %3 = bitcast <2 x i32> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.phsub.d(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <2 x i32>
+  %6 = bitcast <2 x i32> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.phsub.w(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test4(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test4(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phsub.w(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %2 = bitcast <4 x i16> %1 to <1 x i64>
+  %3 = bitcast <4 x i16> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.phsub.w(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <4 x i16>
+  %6 = bitcast <4 x i16> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.phadd.sw(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test3(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test3(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <4 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <4 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phadd.sw(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <4 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <4 x i16>
+  %1 = bitcast <1 x i64> %a to <4 x i16>
+  %2 = bitcast <4 x i16> %1 to <1 x i64>
+  %3 = bitcast <4 x i16> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.phadd.sw(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <4 x i16>
+  %6 = bitcast <4 x i16> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+declare <1 x i64> @llvm.x86.ssse3.phadd.d(<1 x i64>, <1 x i64>) nounwind readnone
+
+define i64 @test2(<1 x i64> %a, <1 x i64> %b) #0 {
+; CHECK-LABEL: define i64 @test2(
+; CHECK-SAME: <1 x i64> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP9:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[B]] to <2 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP10]] to <2 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <2 x i32> [[TMP13]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP1]] to <1 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i32> [[TMP12]] to <1 x i64>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP0]] to <1 x i64>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.phadd.d(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <2 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i32> [[TMP19]] to <1 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+entry:
+  %0 = bitcast <1 x i64> %b to <2 x i32>
+  %1 = bitcast <1 x i64> %a to <2 x i32>
+  %2 = bitcast <2 x i32> %1 to <1 x i64>
+  %3 = bitcast <2 x i32> %0 to <1 x i64>
+  %4 = tail call <1 x i64> @llvm.x86.ssse3.phadd.d(<1 x i64> %2, <1 x i64> %3) nounwind readnone
+  %5 = bitcast <1 x i64> %4 to <2 x i32>
+  %6 = bitcast <2 x i32> %5 to <1 x i64>
+  %7 = extractelement <1 x i64> %6, i32 0
+  ret i64 %7
+}
+
+define <4 x float> @test89(<4 x float> %a, <1 x i64> %b) nounwind #0 {
+; ALL-LABEL: test89:
+; ALL:       # %bb.0:
+; ALL-NEXT:    cvtpi2ps %mm0, %xmm0
+; ALL-NEXT:    ret{{[l|q]}}
+; CHECK-LABEL: define <4 x float> @test89(
+; CHECK-SAME: <4 x float> [[A:%.*]], <1 x i64> [[B:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[TMP4]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[C:%.*]] = tail call <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float> [[A]], <1 x i64> [[B]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[C]]
+;
+  %c = tail call <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float> %a, <1 x i64> %b)
+  ret <4 x float> %c
+}
+
+declare <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, <1 x i64>) nounwind readnone
+
+define void @test90() #0 {
+; ALL-LABEL: test90:
+; ALL:       # %bb.0:
+; ALL-NEXT:    emms
+; ALL-NEXT:    ret{{[l|q]}}
+; CHECK-LABEL: define void @test90(
+; CHECK-SAME: ) #[[ATTR1]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    call void @llvm.x86.mmx.emms()
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.mmx.emms()
+  ret void
+}
+
+declare void @llvm.x86.mmx.emms()
+
+define <1 x i64> @test_mm_insert_pi16(<1 x i64> %a.coerce, i32 %d) nounwind #0 {
+; CHECK-LABEL: define <1 x i64> @test_mm_insert_pi16(
+; CHECK-SAME: <1 x i64> [[A_COERCE:%.*]], i32 [[D:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP3:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <1 x i64> [[TMP3]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pinsr.w(<1 x i64> [[A_COERCE]], i32 [[D]], i32 2)
+; CHECK-NEXT:    store <1 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMP9]]
+;
+entry:
+  %1 = tail call <1 x i64> @llvm.x86.mmx.pinsr.w(<1 x i64> %a.coerce, i32 %d, i32 2)
+  ret <1 x i64> %1
+}
+
+declare <1 x i64> @llvm.x86.mmx.pinsr.w(<1 x i64>, i32, i32 immarg)
+
+define i32 @test_mm_extract_pi16(<1 x i64> %a.coerce) nounwind #0 {
+; CHECK-LABEL: define i32 @test_mm_extract_pi16(
+; CHECK-SAME: <1 x i64> [[A_COERCE:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP2]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.x86.mmx.pextr.w(<1 x i64> [[A_COERCE]], i32 2)
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  %1 = tail call i32 @llvm.x86.mmx.pextr.w(<1 x i64> %a.coerce, i32 2)
+  ret i32 %1
+}
+
+declare i32 @llvm.x86.mmx.pextr.w(<1 x i64>, i32 immarg)
+
+attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/msan_x86_bts_asm.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/msan_x86_bts_asm.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dd2fecb081be234e077a1c950aa69eaf806c2a48
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/msan_x86_bts_asm.ll
@@ -0,0 +1,89 @@
+; Test for the conservative assembly handling mode used by KMSAN.
+; RUN: opt < %s -msan-kernel=1 -msan-check-access-address=0                    \
+; RUN: -msan-handle-asm-conservative=0 -S -passes=msan 2>&1 | FileCheck        \
+; RUN: "-check-prefix=CHECK" %s
+; RUN: opt < %s -msan-kernel=1 -msan-check-access-address=0                    \
+; RUN: -msan-handle-asm-conservative=1 -S -passes=msan 2>&1 | FileCheck        \
+; RUN: "-check-prefixes=CHECK,CHECK-CONS" %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The IR below was generated from the following source:
+;  int main() {
+;    bool bit;
+;    unsigned long value = 2;
+;    long nr = 0;
+;    unsigned long *addr = &value;
+;    asm("btsq %2, %1; setc %0" : "=qm" (bit), "=m" (addr): "Ir" (nr));
+;    if (bit)
+;      return 0;
+;    else
+;      return 1;
+;  }
+;
+; In the regular instrumentation mode MSan is unable to understand that |bit|
+; is initialized by the asm() call, and therefore reports a false positive on
+; the if-statement.
+; The conservative assembly handling mode initializes every memory location
+; passed by pointer into an asm() call. This prevents false positive reports,
+; but may introduce false negatives.
+;
+; This test makes sure that the conservative mode unpoisons the shadow of |bit|
+; by writing 0 to it.
+
+define dso_local i32 @main() sanitize_memory {
+entry:
+  %retval = alloca i32, align 4
+  %bit = alloca i8, align 1
+  %value = alloca i64, align 8
+  %nr = alloca i64, align 8
+  %addr = alloca ptr, align 8
+  store i32 0, ptr %retval, align 4
+  store i64 2, ptr %value, align 8
+  store i64 0, ptr %nr, align 8
+  store ptr %value, ptr %addr, align 8
+  %0 = load i64, ptr %nr, align 8
+  call void asm "btsq $2, $1; setc $0", "=*qm,=*m,Ir,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i8) %bit, ptr elementtype(ptr) %addr, i64 %0)
+  %1 = load i8, ptr %bit, align 1
+  %tobool = trunc i8 %1 to i1
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  ret i32 0
+
+if.else:                                          ; preds = %entry
+  ret i32 1
+}
+
+; %nr is first poisoned, then unpoisoned (written to). Need to optimize this in the future.
+; CHECK: call void @__msan_poison_alloca(ptr %nr{{.*}})
+; CHECK: call { ptr, ptr } @__msan_metadata_ptr_for_store_8(ptr %nr)
+
+; Hooks for inputs usually go before the assembly statement. But here we have none,
+; because %nr is passed by value. However we check %nr for being initialized.
+; CHECK-CONS: call { ptr, ptr } @__msan_metadata_ptr_for_load_8(ptr %nr)
+
+; In the conservative mode, call the store hooks for %bit and %addr:
+; CHECK-CONS: call void @__msan_instrument_asm_store(ptr %bit, i64 1)
+; CHECK-CONS: call void @__msan_instrument_asm_store(ptr %addr, i64 8)
+
+; Landing pad for the %nr check above.
+; CHECK-CONS: call void @__msan_warning
+
+; CHECK: call void asm "btsq $2, $1; setc $0"
+
+; CHECK: [[META:%.*]] = call {{.*}} @__msan_metadata_ptr_for_load_1(ptr %bit)
+; CHECK: [[SHADOW:%.*]] = extractvalue { ptr, ptr } [[META]], 0
+
+; Now load the shadow value for the boolean.
+; CHECK: [[MSLD:%.*]] = load {{.*}} [[SHADOW]]
+; CHECK: [[MSPROP:%.*]] = trunc i8 [[MSLD]] to i1
+
+; Is the shadow poisoned?
+; CHECK: br i1 [[MSPROP]], label %[[IFTRUE:.*]], label {{.*}}
+
+; If yes, raise a warning.
+; CHECK: [[IFTRUE]]:
+; CHECK: call void @__msan_warning
+
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/msan_x86intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/msan_x86intrinsics.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a83a94a06b98f75f0a9668ca2d030dd47168a162
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/msan_x86intrinsics.ll
@@ -0,0 +1,72 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck -check-prefix=CHECK                       \
+; RUN: -check-prefix=CHECK-ORIGINS %s
+; REQUIRES: x86-registered-target
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Store intrinsic.
+
+define void @StoreIntrinsic(ptr %p, <4 x float> %x) nounwind uwtable sanitize_memory {
+  call void @llvm.x86.sse.storeu.ps(ptr %p, <4 x float> %x)
+  ret void
+}
+
+declare void @llvm.x86.sse.storeu.ps(ptr, <4 x float>) nounwind
+
+; CHECK-LABEL: @StoreIntrinsic
+; CHECK-NOT: br
+; CHECK-NOT: = or
+; CHECK: store <4 x i32> {{.*}} align 1
+; CHECK: store <4 x float> %{{.*}}, ptr %{{.*}}, align 1{{$}}
+; CHECK: ret void
+
+
+; Load intrinsic.
+
+define <16 x i8> @LoadIntrinsic(ptr %p) nounwind uwtable sanitize_memory {
+  %call = call <16 x i8> @llvm.x86.sse3.ldu.dq(ptr %p)
+  ret <16 x i8> %call
+}
+
+declare <16 x i8> @llvm.x86.sse3.ldu.dq(ptr %p) nounwind
+
+; CHECK-LABEL: @LoadIntrinsic
+; CHECK: load <16 x i8>, ptr {{.*}} align 1
+; CHECK-ORIGINS: [[ORIGIN:%[01-9a-z]+]] = load i32, ptr {{.*}}
+; CHECK-NOT: br
+; CHECK-NOT: = or
+; CHECK: call <16 x i8> @llvm.x86.sse3.ldu.dq
+; CHECK: store <16 x i8> {{.*}} @__msan_retval_tls
+; CHECK-ORIGINS: store i32 {{.*}}[[ORIGIN]], ptr @__msan_retval_origin_tls
+; CHECK: ret <16 x i8>
+
+
+; Simple NoMem intrinsic
+; Check that shadow is OR'ed, and origin is Select'ed
+; And no shadow checks!
+
+define <8 x i16> @Pmulhuw128(<8 x i16> %a, <8 x i16> %b) nounwind uwtable sanitize_memory {
+  %call = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %call
+}
+
+declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a, <8 x i16> %b) nounwind
+
+; CHECK-LABEL: @Pmulhuw128
+; CHECK-NEXT: load <8 x i16>, ptr @__msan_param_tls
+; CHECK-ORIGINS: load i32, ptr @__msan_param_origin_tls
+; CHECK-NEXT: load <8 x i16>, ptr {{.*}} @__msan_param_tls
+; CHECK-ORIGINS: load i32, ptr {{.*}} @__msan_param_origin_tls
+; CHECK-NEXT: call void @llvm.donothing
+; CHECK-NEXT: = or <8 x i16>
+; CHECK-ORIGINS: = bitcast <8 x i16> {{.*}} to i128
+; CHECK-ORIGINS-NEXT: = icmp ne i128 {{.*}}, 0
+; CHECK-ORIGINS-NEXT: = select i1 {{.*}}, i32 {{.*}}, i32
+; CHECK-NEXT: call <8 x i16> @llvm.x86.sse2.pmulhu.w
+; CHECK-NEXT: store <8 x i16> {{.*}} @__msan_retval_tls
+; CHECK-ORIGINS: store i32 {{.*}} @__msan_retval_origin_tls
+; CHECK-NEXT: ret <8 x i16>
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/sse-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/sse-intrinsics-x86.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9d7763a6ef589dcd922fbd3cd147477bda7430e3
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/sse-intrinsics-x86.ll
@@ -0,0 +1,519 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <4 x float> @test_x86_sse_cmp_ps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_cmp_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> %a0, <4 x float> %a1, i8 7) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_cmp_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_cmp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a0, <4 x float> %a1, i8 7) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.cmp.ss(<4 x float>, <4 x float>, i8) nounwind readnone
+
+
+define i32 @test_x86_sse_comieq_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_comieq_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.comieq.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.comieq.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.comieq.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_comige_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_comige_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.comige.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.comige.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.comige.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_comigt_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_comigt_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.comigt.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.comigt.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.comigt.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_comile_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_comile_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.comile.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.comile.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.comile.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_comilt_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_comilt_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.comilt.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.comilt.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.comilt.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_comineq_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_comineq_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.comineq.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.comineq.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.comineq.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_cvtss2si(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_cvtss2si(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.cvtss2si(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.cvtss2si(<4 x float> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.cvtss2si(<4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_cvttss2si(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_cvttss2si(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.cvttss2si(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.cvttss2si(<4 x float> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.cvttss2si(<4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_ldmxcsr(ptr %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_ldmxcsr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    [[_LDMXCSR:%.*]] = load i32, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[_LDMXCSR]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.sse.ldmxcsr(ptr [[A0]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.sse.ldmxcsr(ptr %a0)
+  ret void
+}
+declare void @llvm.x86.sse.ldmxcsr(ptr) nounwind
+
+
+
+define <4 x float> @test_x86_sse_max_ps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_max_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_max_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_max_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_min_ps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_min_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_min_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_min_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_movmsk_ps(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_movmsk_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.movmsk.ps(<4 x float>) nounwind readnone
+
+
+
+define <4 x float> @test_x86_sse_rcp_ps(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_rcp_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.rcp.ps(<4 x float>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_rcp_ss(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_rcp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_rsqrt_ps(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_rsqrt_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_rsqrt_ss(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_rsqrt_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_stmxcsr(ptr %a0) #0 {
+; CHECK-LABEL: @test_x86_sse_stmxcsr(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[A0:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    store i32 0, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.sse.stmxcsr(ptr [[A0]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.sse.stmxcsr(ptr %a0)
+  ret void
+}
+declare void @llvm.x86.sse.stmxcsr(ptr) nounwind
+
+
+define i32 @test_x86_sse_ucomieq_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_ucomieq_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.ucomieq.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_ucomige_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_ucomige_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.ucomige.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.ucomige.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.ucomige.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_ucomigt_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_ucomigt_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.ucomigt.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_ucomile_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_ucomile_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.ucomile.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.ucomile.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.ucomile.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_ucomilt_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_ucomilt_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.ucomilt.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse_ucomineq_ss(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse_ucomineq_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> %a0, <4 x float> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse.ucomineq.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define void @sfence() nounwind {
+; CHECK-LABEL: @sfence(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    tail call void @llvm.x86.sse.sfence()
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.sse.sfence()
+  ret void
+}
+declare void @llvm.x86.sse.sfence() nounwind
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/sse2-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/sse2-intrinsics-x86.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e9323f6dd330833de262f9e489c79870d361da95
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/sse2-intrinsics-x86.ll
@@ -0,0 +1,1381 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <2 x double> @test_x86_sse2_cmp_pd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_cmp_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <2 x i1> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <2 x i64> [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double> %a0, <2 x double> %a1, i8 7) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8) nounwind readnone
+
+
+define <2 x double> @test_x86_sse2_cmp_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_cmp_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <2 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a0, <2 x double> %a1, i8 7) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8) nounwind readnone
+
+
+define i32 @test_x86_sse2_comieq_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_comieq_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.comieq.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.comieq.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_comige_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_comige_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.comige.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.comige.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.comige.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_comigt_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_comigt_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.comigt.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.comigt.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.comigt.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_comile_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_comile_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.comile.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.comile.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.comile.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_comilt_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_comilt_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.comilt.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.comilt.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.comilt.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_comineq_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_comineq_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.comineq.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.comineq.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.comineq.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define <4 x i32> @test_x86_sse2_cvtpd2dq(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtpd2dq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %a0) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>) nounwind readnone
+
+
+define <2 x i64> @test_mm_cvtpd_epi32_zext(<2 x double> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm_cvtpd_epi32_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[CVT:%.*]] = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i32> [[CVT]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[BC]]
+;
+  %cvt = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %a0)
+  %res = shufflevector <4 x i32> %cvt, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  %bc = bitcast <4 x i32> %res to <2 x i64>
+  ret <2 x i64> %bc
+}
+
+
+define <2 x i64> @test_mm_cvtpd_epi32_zext_load(ptr %p0) nounwind #0 {
+; CHECK-LABEL: @test_mm_cvtpd_epi32_zext_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[A0:%.*]] = load <2 x double>, ptr [[P0:%.*]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P0]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[CVT:%.*]] = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> [[A0]])
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i32> [[CVT]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[BC]]
+;
+  %a0 = load <2 x double>, ptr %p0
+  %cvt = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %a0)
+  %res = shufflevector <4 x i32> %cvt, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  %bc = bitcast <4 x i32> %res to <2 x i64>
+  ret <2 x i64> %bc
+}
+
+
+define <4 x float> @test_x86_sse2_cvtpd2ps(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtpd2ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %a0) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double>) nounwind readnone
+
+define <4 x float> @test_x86_sse2_cvtpd2ps_zext(<2 x double> %a0) nounwind #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtpd2ps_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[CVT:%.*]] = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[CVT]], <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %cvt = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %a0)
+  %res = shufflevector <4 x float> %cvt, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_x86_sse2_cvtpd2ps_zext_load(ptr %p0) nounwind #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtpd2ps_zext_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[A0:%.*]] = load <2 x double>, ptr [[P0:%.*]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P0]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[CVT:%.*]] = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> [[A0]])
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[CVT]], <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a0 = load <2 x double>, ptr %p0
+  %cvt = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %a0)
+  %res = shufflevector <4 x float> %cvt, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x float> %res
+}
+
+define <4 x i32> @test_x86_sse2_cvtps2dq(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtps2dq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float> %a0) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse2_cvtsd2si(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtsd2si(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse2_cvtsd2ss(<4 x float> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtsd2ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> %a0, <2 x double> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float>, <2 x double>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse2_cvtsd2ss_load(<4 x float> %a0, ptr %p1) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtsd2ss_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A1:%.*]] = load <2 x double>, ptr [[P1:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P1]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[_MSLD]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> [[A0:%.*]], <2 x double> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a1 = load <2 x double>, ptr %p1
+  %res = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> %a0, <2 x double> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+
+define <4 x float> @test_x86_sse2_cvtsd2ss_load_optsize(<4 x float> %a0, ptr %p1) optsize #0 {
+; CHECK-LABEL: @test_x86_sse2_cvtsd2ss_load_optsize(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A1:%.*]] = load <2 x double>, ptr [[P1:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P1]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[_MSLD]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP2]], i32 0, i32 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> [[A0:%.*]], <2 x double> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a1 = load <2 x double>, ptr %p1
+  %res = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> %a0, <2 x double> %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+
+define <4 x i32> @test_x86_sse2_cvttpd2dq(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvttpd2dq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> %a0) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double>) nounwind readnone
+
+
+define <2 x i64> @test_mm_cvttpd_epi32_zext(<2 x double> %a0) nounwind #0 {
+; CHECK-LABEL: @test_mm_cvttpd_epi32_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[CVT:%.*]] = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i32> [[CVT]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[BC]]
+;
+  %cvt = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> %a0)
+  %res = shufflevector <4 x i32> %cvt, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  %bc = bitcast <4 x i32> %res to <2 x i64>
+  ret <2 x i64> %bc
+}
+
+
+define <2 x i64> @test_mm_cvttpd_epi32_zext_load(ptr %p0) nounwind #0 {
+; CHECK-LABEL: @test_mm_cvttpd_epi32_zext_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[A0:%.*]] = load <2 x double>, ptr [[P0:%.*]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P0]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF0]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    [[CVT:%.*]] = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> [[A0]])
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i32> [[CVT]], <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[BC]]
+;
+  %a0 = load <2 x double>, ptr %p0
+  %cvt = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> %a0)
+  %res = shufflevector <4 x i32> %cvt, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  %bc = bitcast <4 x i32> %res to <2 x i64>
+  ret <2 x i64> %bc
+}
+
+
+define <4 x i32> @test_x86_sse2_cvttps2dq(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvttps2dq(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> [[A0:%.*]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> %a0) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float>) nounwind readnone
+
+
+define i32 @test_x86_sse2_cvttsd2si(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_cvttsd2si(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>) nounwind readnone
+
+
+define <2 x double> @test_x86_sse2_max_pd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_max_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse2.max.pd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define <2 x double> @test_x86_sse2_max_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_max_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP3]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse2.max.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define <2 x double> @test_x86_sse2_min_pd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_min_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse2.min.pd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define <2 x double> @test_x86_sse2_min_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_min_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP3]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse2.min.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_movmsk_pd(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_movmsk_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.movmsk.pd(<2 x double>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_packssdw_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[TMP4]], <4 x i32> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %a0, <4 x i32> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_packssdw_128_fold() #0 {
+; CHECK-LABEL: @test_x86_sse2_packssdw_128_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> zeroinitializer, <4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> zeroinitializer, <4 x i32> <i32 65535, i32 65536, i32 -1, i32 -131072>)
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> zeroinitializer, <4 x i32> <i32 65535, i32 65536, i32 -1, i32 -131072>)
+  ret <8 x i16> %res
+}
+
+
+define <16 x i8> @test_x86_sse2_packsswb_128(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_packsswb_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> [[TMP4]], <8 x i16> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %res = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a1) ; <<16 x i8>> [#uses=1]
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <16 x i8> @test_x86_sse2_packsswb_128_fold() #0 {
+; CHECK-LABEL: @test_x86_sse2_packsswb_128_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> zeroinitializer, <8 x i16> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> <i16 0, i16 255, i16 256, i16 -1, i16 -1, i16 -255, i16 -256, i16 -32678>, <8 x i16> zeroinitializer)
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %res = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <8 x i16> zeroinitializer)
+  ret <16 x i8> %res
+}
+
+
+define <16 x i8> @test_x86_sse2_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_packuswb_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i1> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> [[TMP4]], <8 x i16> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %res = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %a0, <8 x i16> %a1) ; <<16 x i8>> [#uses=1]
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <16 x i8> @test_x86_sse2_packuswb_128_fold() #0 {
+; CHECK-LABEL: @test_x86_sse2_packuswb_128_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> zeroinitializer, <8 x i16> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> <i16 0, i16 255, i16 256, i16 -1, i16 -1, i16 -255, i16 -256, i16 -32678>, <8 x i16> zeroinitializer)
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %res = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <8 x i16> zeroinitializer)
+  ret <16 x i8> %res
+}
+
+
+define <16 x i8> @test_x86_sse2_pavg_b(<16 x i8> %a0, <16 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_pavg_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8> [[A0:%.*]], <16 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %res = call <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1]
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8>, <16 x i8>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_pavg_w(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_pavg_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <4 x i32> @test_x86_sse2_pmadd_wd(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_pmadd_wd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define i32 @test_x86_sse2_pmovmskb_128(<16 x i8> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_pmovmskb_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> [[A0:%.*]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %a0) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_pmulh_w(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_pmulh_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_pmulhu_w(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_pmulhu_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <2 x i64> @test_x86_sse2_psad_bw(<16 x i8> %a0, <16 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psad_bw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <2 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <2 x i1> [[TMP5]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = lshr <2 x i64> [[TMP6]], <i64 48, i64 48>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[A0:%.*]], <16 x i8> [[A1:%.*]])
+; CHECK-NEXT:    store <2 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a0, <16 x i8> %a1) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>) nounwind readnone
+
+
+define <4 x i32> @test_x86_sse2_psll_d(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psll_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) nounwind readnone
+
+
+define <2 x i64> @test_x86_sse2_psll_q(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psll_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_psll_w(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psll_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <4 x i32> @test_x86_sse2_pslli_d(<4 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_pslli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %a0, i32 7) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32>, i32) nounwind readnone
+
+
+define <2 x i64> @test_x86_sse2_pslli_q(<2 x i64> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_pslli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <2 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64>, i32) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_pslli_w(<8 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_pslli_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %a0, i32 7) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16>, i32) nounwind readnone
+
+
+define <4 x i32> @test_x86_sse2_psra_d(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psra_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_psra_w(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psra_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <4 x i32> @test_x86_sse2_psrai_d(<4 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrai_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %a0, i32 7) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32>, i32) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_psrai_w(<8 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrai_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %a0, i32 7) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16>, i32) nounwind readnone
+
+
+define <4 x i32> @test_x86_sse2_psrl_d(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrl_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> [[TMP1]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> [[A0:%.*]], <4 x i32> [[A1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) nounwind readnone
+
+
+define <2 x i64> @test_x86_sse2_psrl_q(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrl_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> [[TMP1]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> [[A0:%.*]], <2 x i64> [[A1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_psrl_w(<8 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrl_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i128
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i128 [[TMP6]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_psrl_w_load(<8 x i16> %a0, ptr %p) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrl_w_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i128 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i1 [[TMP10]] to i128
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i128 [[TMP11]] to <8 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> [[TMP2]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i16> [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP14]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %a1 = load <8 x i16>, ptr %p
+  %res = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+
+
+define <4 x i32> @test_x86_sse2_psrli_d(<4 x i32> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrli_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %a0, i32 7) ; <<4 x i32>> [#uses=1]
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32>, i32) nounwind readnone
+
+
+define <2 x i64> @test_x86_sse2_psrli_q(<2 x i64> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrli_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <2 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %res = call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64>, i32) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_psrli_w(<8 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse2_psrli_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <8 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %a0, i32 7) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16>, i32) nounwind readnone
+
+
+define i32 @test_x86_sse2_ucomieq_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_ucomieq_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.ucomieq.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_ucomige_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_ucomige_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.ucomige.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_ucomigt_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_ucomigt_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.ucomigt.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_ucomile_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_ucomile_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.ucomile.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_ucomilt_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_ucomilt_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.ucomilt.sd(<2 x double>, <2 x double>) nounwind readnone
+
+
+define i32 @test_x86_sse2_ucomineq_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse2_ucomineq_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> %a0, <2 x double> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse2.ucomineq.sd(<2 x double>, <2 x double>) nounwind readnone
+
+define void @test_x86_sse2_pause() #0 {
+; CHECK-LABEL: @test_x86_sse2_pause(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    tail call void @llvm.x86.sse2.pause()
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.sse2.pause()
+  ret void
+}
+declare void @llvm.x86.sse2.pause() nounwind
+
+define void @lfence() nounwind #0 {
+; CHECK-LABEL: @lfence(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    tail call void @llvm.x86.sse2.lfence()
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.sse2.lfence()
+  ret void
+}
+declare void @llvm.x86.sse2.lfence() nounwind
+
+define void @mfence() nounwind #0 {
+; CHECK-LABEL: @mfence(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    tail call void @llvm.x86.sse2.mfence()
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.sse2.mfence()
+  ret void
+}
+declare void @llvm.x86.sse2.mfence() nounwind
+
+define void @clflush(ptr %p) nounwind #0 {
+; CHECK-LABEL: @clflush(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF0]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    tail call void @llvm.x86.sse2.clflush(ptr [[P:%.*]])
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.sse2.clflush(ptr %p)
+  ret void
+}
+declare void @llvm.x86.sse2.clflush(ptr) nounwind
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/sse41-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/sse41-intrinsics-x86.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a71455821bd6bd57b395b2687777a1dceef6b2b8
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/sse41-intrinsics-x86.ll
@@ -0,0 +1,431 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes
+; RUN: opt %s -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <2 x double> @test_x86_sse41_blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 {
+; CHECK-LABEL: @test_x86_sse41_blendvpd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x double> [[A2:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr <2 x i64> [[TMP4]], <i64 63, i64 63>
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc <2 x i64> [[TMP5]] to <2 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ashr <2 x i64> [[TMP1]], <i64 63, i64 63>
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <2 x i64> [[TMP7]] to <2 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[TMP6]], <2 x i64> [[TMP2]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <2 x double> [[A1:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <2 x double> [[A0:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <2 x i64> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i64> [[TMP12]], [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i64> [[TMP13]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[TMP8]], <2 x i64> [[TMP14]], <2 x i64> [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> [[A0]], <2 x double> [[A1]], <2 x double> [[A2]])
+; CHECK-NEXT:    store <2 x i64> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse41_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+; CHECK-LABEL: @test_x86_sse41_blendvps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[A2:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr <4 x i32> [[TMP4]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc <4 x i32> [[TMP5]] to <4 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ashr <4 x i32> [[TMP1]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x float> [[A1:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x float> [[A0:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[TMP12]], [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP13]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[TMP14]], <4 x i32> [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
+
+
+define <2 x double> @test_x86_sse41_dppd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_dppd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <2 x i1> <i1 false, i1 true>, <2 x i64> [[TMP3]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP4]])
+; CHECK-NEXT:    [[_MSDPP:%.*]] = icmp eq i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[_MSDPP]], <2 x i1> zeroinitializer, <2 x i1> <i1 false, i1 true>
+; CHECK-NEXT:    [[_MSDPP1:%.*]] = sext <2 x i1> [[TMP6]] to <2 x i64>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse41.dppd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i8 -18)
+; CHECK-NEXT:    store <2 x i64> [[_MSDPP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse41.dppd(<2 x double> %a0, <2 x double> %a1, i8 -18) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse41.dppd(<2 x double>, <2 x double>, i8) nounwind readnone
+
+
+define <4 x float> @test_x86_sse41_dpps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_dpps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSDPP:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[_MSDPP]], <4 x i1> zeroinitializer, <4 x i1> <i1 false, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[_MSDPP1:%.*]] = sext <4 x i1> [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.dpps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 -18)
+; CHECK-NEXT:    store <4 x i32> [[_MSDPP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.dpps(<4 x float> %a0, <4 x float> %a1, i8 -18) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse41.dpps(<4 x float>, <4 x float>, i8) nounwind readnone
+
+
+define <4 x float> @test_x86_sse41_insertps(<4 x float> %a0, <4 x float> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_insertps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], i8 17)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 17) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
+
+
+
+define <8 x i16> @test_x86_sse41_mpsadbw(<16 x i8> %a0, <16 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_mpsadbw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF0]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> [[A0:%.*]], <16 x i8> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %a0, <16 x i8> %a1, i8 7) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8>, <16 x i8>, i8) nounwind readnone
+
+define <8 x i16> @test_x86_sse41_mpsadbw_load_op0(ptr %ptr, <16 x i8> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_mpsadbw_load_op0(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A0:%.*]] = load <16 x i8>, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i8>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i8> [[_MSLD]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF0]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> [[A0]], <16 x i8> [[A1:%.*]], i8 7)
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %a0 = load <16 x i8>, ptr %ptr
+  %res = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %a0, <16 x i8> %a1, i8 7) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_x86_sse41_packusdw(<4 x i32> %a0, <4 x i32> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_packusdw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[TMP4]], <4 x i32> [[TMP6]])
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %a0, <4 x i32> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse41_packusdw_fold() #0 {
+; CHECK-LABEL: @test_x86_sse41_packusdw_fold(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP_VECTOR_PACK:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> zeroinitializer, <4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> zeroinitializer, <4 x i32> <i32 65535, i32 65536, i32 -1, i32 -131072>)
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP_VECTOR_PACK]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> zeroinitializer, <4 x i32> <i32 65535, i32 65536, i32 -1, i32 -131072>)
+  ret <8 x i16> %res
+}
+
+
+define <16 x i8> @test_x86_sse41_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) #0 {
+; CHECK-LABEL: @test_x86_sse41_pblendvb(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr <16 x i8> [[A2:%.*]], <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc <16 x i8> [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = ashr <16 x i8> [[TMP1]], <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc <16 x i8> [[TMP6]] to <16 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP5]], <16 x i8> [[TMP2]], <16 x i8> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <16 x i8> [[A1:%.*]], [[A0:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <16 x i8> [[TMP9]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <16 x i8> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP7]], <16 x i8> [[TMP11]], <16 x i8> [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[A0]], <16 x i8> [[A1]], <16 x i8> [[A2]])
+; CHECK-NEXT:    store <16 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %res = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) ; <<16 x i8>> [#uses=1]
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse41_phminposuw(<8 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse41_phminposuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16> [[A0:%.*]])
+; CHECK-NEXT:    store <8 x i16> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
+;
+  %res = call <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16> %a0) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16>) nounwind readnone
+
+
+define i32 @test_x86_sse41_ptestc(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_ptestc(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i1> [[TMP4]] to i2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i2 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse41.ptestc(<2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse41.ptestc(<2 x i64> %a0, <2 x i64> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse41.ptestc(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define i32 @test_x86_sse41_ptestnzc(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_ptestnzc(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i1> [[TMP4]] to i2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i2 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse41.ptestnzc(<2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse41.ptestnzc(<2 x i64> %a0, <2 x i64> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse41.ptestnzc(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define i32 @test_x86_sse41_ptestz(<2 x i64> %a0, <2 x i64> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_ptestz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i1> [[TMP4]] to i2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i2 [[TMP5]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.sse41.ptestz(<2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = call i32 @llvm.x86.sse41.ptestz(<2 x i64> %a0, <2 x i64> %a1) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.sse41.ptestz(<2 x i64>, <2 x i64>) nounwind readnone
+
+
+define <2 x double> @test_x86_sse41_round_pd(<2 x double> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse41_round_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %a0, i32 7) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse41.round.pd(<2 x double>, i32) nounwind readnone
+
+
+define <4 x float> @test_x86_sse41_round_ps(<4 x float> %a0) #0 {
+; CHECK-LABEL: @test_x86_sse41_round_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %a0, i32 7) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse41.round.ps(<4 x float>, i32) nounwind readnone
+
+
+define <2 x double> @test_x86_sse41_round_sd(<2 x double> %a0, <2 x double> %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_round_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1:%.*]], i32 7)
+; CHECK-NEXT:    store <2 x i64> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a0, <2 x double> %a1, i32 7) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
+
+
+define <2 x double> @test_x86_sse41_round_sd_load(<2 x double> %a0, ptr %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_round_sd_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A1B:%.*]] = load <2 x double>, ptr [[A1:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <2 x i64>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[_MSLD]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> [[A0:%.*]], <2 x double> [[A1B]], i32 7)
+; CHECK-NEXT:    store <2 x i64> [[TMP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %a1b = load <2 x double>, ptr %a1
+  %res = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a0, <2 x double> %a1b, i32 7) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+
+
+define <4 x float> @test_x86_sse41_round_ss_load(<4 x float> %a0, ptr %a1) #0 {
+; CHECK-LABEL: @test_x86_sse41_round_ss_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn()
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A1B:%.*]] = load <4 x float>, ptr [[A1:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[A1]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[_MSLD]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> [[A0:%.*]], <4 x float> [[A1B]], i32 7)
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %a1b = load <4 x float>, ptr %a1
+  %res = call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a0, <4 x float> %a1b, i32 7) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) nounwind readnone
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/vararg-too-large.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg-too-large.ll
new file mode 100644
index 0000000000000000000000000000000000000000..adb3e208d8553ec6749473c79ee52e5421ac27fe
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg-too-large.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -msan-check-access-address=0 -S 2>&1 -passes=msan | FileCheck  \
+; RUN: %s
+
+; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
+; passed to a variadic function.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local i64 @many_args() {
+entry:
+  %ret = call i64 (i64, ...) @sum(i64 120,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1,
+    i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1
+  )
+  ret i64 %ret
+}
+
+; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed.
+; CHECK-LABEL: @many_args
+; CHECK: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 792)
+; CHECK-NOT: i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 800)
+declare i64 @sum(i64 %n, ...)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/vararg.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg.ll
similarity index 100%
rename from llvm/test/Instrumentation/MemorySanitizer/X86/vararg.ll
rename to llvm/test/Instrumentation/MemorySanitizer/i386/vararg.ll
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_call.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_call.ll
new file mode 100644
index 0000000000000000000000000000000000000000..32d43e11fbd9b8a5c40ba79de85e45bd52ef6928
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_call.ll
@@ -0,0 +1,117 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,CHECK-ORIGIN"
+; RUN: opt < %s -msan-check-access-address=0 -S          \
+; RUN: -passes="msan<track-origins=1>" 2>&1 | FileCheck %s "--check-prefixes=CHECK,CHECK-ORIGIN"
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,CHECK-ORIGIN"
+
+; Test that shadow and origin are stored for variadic function params.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.__va_list_tag = type { i32, i32, ptr, ptr }
+
+define dso_local i32 @test(i32 %a, i32 %b, i32 %c) local_unnamed_addr {
+entry:
+  %call = tail call i32 (i32, ...) @sum(i32 3, i32 %a, i32 %b, i32 %c)
+  ret i32 %call
+}
+
+; CHECK: store i32 0, {{.*}} @__msan_param_tls {{.*}} i64 8
+; CHECK: store i32 0, {{.*}} @__msan_param_tls {{.*}} i64 16
+; CHECK: store i32 0, {{.*}} @__msan_param_tls {{.*}} i64 24
+; CHECK: store i32 0, {{.*}} @__msan_va_arg_tls {{.*}} i64 8
+; CHECK-ORIGIN: store i32 0, {{.*}} @__msan_va_arg_origin_tls {{.*}} i64 8
+; CHECK: store i32 0, {{.*}} @__msan_va_arg_tls {{.*}} i64 16
+; CHECK-ORIGIN: store i32 0, {{.*}} @__msan_va_arg_origin_tls {{.*}} i64 16
+; CHECK: store i32 0, {{.*}} @__msan_va_arg_tls {{.*}} i64 24
+; CHECK-ORIGIN: store i32 0, {{.*}} @__msan_va_arg_origin_tls {{.*}} i64 24
+
+define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #2
+  call void @llvm.va_start(ptr nonnull %args)
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body.lr.ph, label %for.end
+
+; CHECK: call void @llvm.memcpy.{{.*}} [[SHADOW_COPY:%[_0-9a-z]+]], {{.*}} @__msan_va_arg_tls
+; CHECK-ORIGIN: call void @llvm.memcpy{{.*}} [[ORIGIN_COPY:%[_0-9a-z]+]], {{.*}} @__msan_va_arg_origin_tls
+
+; CHECK: call void @llvm.va_start
+; CHECK: call void @llvm.memcpy.{{.*}}, {{.*}} [[SHADOW_COPY]], i{{.*}} [[REGSAVE:[0-9]+]]
+; CHECK-ORIGIN: call void @llvm.memcpy.{{.*}}, {{.*}} [[ORIGIN_COPY]], i{{.*}} [[REGSAVE]]
+
+; CHECK: [[OVERFLOW_SHADOW:%[_0-9a-z]+]] = getelementptr i8, ptr [[SHADOW_COPY]], i{{.*}} [[REGSAVE]]
+; CHECK: call void @llvm.memcpy.{{.*}}[[OVERFLOW_SHADOW]]
+; CHECK-ORIGIN: [[OVERFLOW_ORIGIN:%[_0-9a-z]+]] = getelementptr i8, ptr [[ORIGIN_COPY]], i{{.*}} [[REGSAVE]]
+; CHECK-ORIGIN: call void @llvm.memcpy.{{.*}}[[OVERFLOW_ORIGIN]]
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %args, i64 0, i64 0, i32 3
+  %overflow_arg_area_p = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %args, i64 0, i64 0, i32 2
+  %gp_offset.pre = load i32, ptr %args, align 16
+  br label %for.body
+
+for.body:                                         ; preds = %vaarg.end, %for.body.lr.ph
+  %gp_offset = phi i32 [ %gp_offset.pre, %for.body.lr.ph ], [ %gp_offset12, %vaarg.end ]
+  %sum.011 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %vaarg.end ]
+  %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %vaarg.end ]
+  %fits_in_gp = icmp ult i32 %gp_offset, 41
+  br i1 %fits_in_gp, label %vaarg.in_reg, label %vaarg.in_mem
+
+vaarg.in_reg:                                     ; preds = %for.body
+  %reg_save_area = load ptr, ptr %0, align 16
+  %1 = sext i32 %gp_offset to i64
+  %2 = getelementptr i8, ptr %reg_save_area, i64 %1
+  %3 = add i32 %gp_offset, 8
+  store i32 %3, ptr %args, align 16
+  br label %vaarg.end
+
+vaarg.in_mem:                                     ; preds = %for.body
+  %overflow_arg_area = load ptr, ptr %overflow_arg_area_p, align 8
+  %overflow_arg_area.next = getelementptr i8, ptr %overflow_arg_area, i64 8
+  store ptr %overflow_arg_area.next, ptr %overflow_arg_area_p, align 8
+  br label %vaarg.end
+
+vaarg.end:                                        ; preds = %vaarg.in_mem, %vaarg.in_reg
+  %gp_offset12 = phi i32 [ %3, %vaarg.in_reg ], [ %gp_offset, %vaarg.in_mem ]
+  %vaarg.addr.in = phi ptr [ %2, %vaarg.in_reg ], [ %overflow_arg_area, %vaarg.in_mem ]
+  %4 = load i32, ptr %vaarg.addr.in, align 4
+  %add = add nsw i32 %4, %sum.011
+  %inc = add nuw nsw i32 %i.010, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %vaarg.end, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %vaarg.end ]
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #2
+  ret i32 %sum.0.lcssa
+}
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+
+; Function Attrs: nounwind
+declare void @llvm.va_start(ptr) #2
+
+; Function Attrs: nounwind
+declare void @llvm.va_end(ptr) #2
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+
+declare dso_local i80 @sum_i80(i32, ...) local_unnamed_addr
+
+; Unaligned types like i80 should also work.
+define dso_local i80 @test_i80(i80 %a, i80 %b, i80 %c) local_unnamed_addr {
+entry:
+  %call = tail call i80 (i32, ...) @sum_i80(i32 3, i80 %a, i80 %b, i80 %c)
+  ret i80 %call
+}
+
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_shadow.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_shadow.ll
new file mode 100644
index 0000000000000000000000000000000000000000..205101564dfe0993be5386931becda455618fede
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_shadow.ll
@@ -0,0 +1,1315 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.Double4 = type { [4 x double] }
+%struct.LongDouble2 = type { [2 x x86_fp80] }
+%struct.LongDouble4 = type { [4 x x86_fp80] }
+%struct.IntInt = type { i32, i32 }
+%struct.Int64Int64 = type { i64, i64 }
+%struct.DoubleDouble = type { double, double }
+%struct.DoubleFloat = type { double, float }
+%struct.__va_list_tag = type { i32, i32, ptr, ptr }
+
+define linkonce_odr dso_local void @_Z4testIcEvT_(i8 noundef signext %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testIcEvT_(
+; CHECK-SAME: i8 noundef signext [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG_ADDR:%.*]] = alloca i8, align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 87960930222080
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP3]], i8 -1, i64 1, i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    store i8 [[TMP0]], ptr [[TMP6]], align 1
+; CHECK-NEXT:    store i8 [[ARG]], ptr [[ARG_ADDR]], align 1
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG_ADDR]])
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARG_ADDR]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext i8 [[_MSLD]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP7]] to i32
+; CHECK-NEXT:    store i8 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (i8, i32, ...) @_Z5test2IcEvT_iz(i8 noundef signext [[TMP7]], i32 noundef 1, i32 noundef [[CONV]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg.addr = alloca i8, align 1
+  store i8 %arg, ptr %arg.addr, align 1
+  call void @_Z3usePv(ptr noundef nonnull %arg.addr)
+  %0 = load i8, ptr %arg.addr, align 1
+  %conv = sext i8 %0 to i32
+  call void (i8, i32, ...) @_Z5test2IcEvT_iz(i8 noundef signext %0, i32 noundef 1, i32 noundef %conv)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testIiEvT_(i32 noundef %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testIiEvT_(
+; CHECK-SAME: i32 noundef [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 87960930222080
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[TMP3]], i8 -1, i64 4, i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP6]], align 4
+; CHECK-NEXT:    store i32 [[ARG]], ptr [[ARG_ADDR]], align 4
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG_ADDR]])
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARG_ADDR]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    store i32 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i32 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i32 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (i32, i32, ...) @_Z5test2IiEvT_iz(i32 noundef [[TMP7]], i32 noundef 1, i32 noundef [[TMP7]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg.addr = alloca i32, align 4
+  store i32 %arg, ptr %arg.addr, align 4
+  call void @_Z3usePv(ptr noundef nonnull %arg.addr)
+  %0 = load i32, ptr %arg.addr, align 4
+  call void (i32, i32, ...) @_Z5test2IiEvT_iz(i32 noundef %0, i32 noundef 1, i32 noundef %0)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testIfEvT_(float noundef %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testIfEvT_(
+; CHECK-SAME: float noundef [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG_ADDR:%.*]] = alloca float, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 87960930222080
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[TMP3]], i8 -1, i64 4, i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP6]], align 4
+; CHECK-NEXT:    store float [[ARG]], ptr [[ARG_ADDR]], align 4
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG_ADDR]])
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARG_ADDR]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[_MSLD]] to i64
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP7]] to double
+; CHECK-NEXT:    store i32 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i64 [[TMP11]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 [[TMP11]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (float, i32, ...) @_Z5test2IfEvT_iz(float noundef [[TMP7]], i32 noundef 1, double noundef [[CONV]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg.addr = alloca float, align 4
+  store float %arg, ptr %arg.addr, align 4
+  call void @_Z3usePv(ptr noundef nonnull %arg.addr)
+  %0 = load float, ptr %arg.addr, align 4
+  %conv = fpext float %0 to double
+  call void (float, i32, ...) @_Z5test2IfEvT_iz(float noundef %0, i32 noundef 1, double noundef %conv)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testIdEvT_(double noundef %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testIdEvT_(
+; CHECK-SAME: double noundef [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG_ADDR:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 87960930222080
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP3]], i8 -1, i64 8, i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    store i64 [[TMP0]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    store double [[ARG]], ptr [[ARG_ADDR]], align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG_ADDR]])
+; CHECK-NEXT:    [[TMP7:%.*]] = load double, ptr [[ARG_ADDR]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (double, i32, ...) @_Z5test2IdEvT_iz(double noundef [[TMP7]], i32 noundef 1, double noundef [[TMP7]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg.addr = alloca double, align 8
+  store double %arg, ptr %arg.addr, align 8
+  call void @_Z3usePv(ptr noundef nonnull %arg.addr)
+  %0 = load double, ptr %arg.addr, align 8
+  call void (double, i32, ...) @_Z5test2IdEvT_iz(double noundef %0, i32 noundef 1, double noundef %0)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testIeEvT_(x86_fp80 noundef %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testIeEvT_(
+; CHECK-SAME: x86_fp80 noundef [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i80, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG_ADDR:%.*]] = alloca x86_fp80, align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 87960930222080
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP3]], i8 -1, i64 16, i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    store i80 [[TMP0]], ptr [[TMP6]], align 16
+; CHECK-NEXT:    store x86_fp80 [[ARG]], ptr [[ARG_ADDR]], align 16
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG_ADDR]])
+; CHECK-NEXT:    [[TMP7:%.*]] = load x86_fp80, ptr [[ARG_ADDR]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[ARG_ADDR]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i80, ptr [[TMP10]], align 16
+; CHECK-NEXT:    store i80 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i80 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    store i80 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 176) to ptr), align 8
+; CHECK-NEXT:    store i64 16, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (x86_fp80, i32, ...) @_Z5test2IeEvT_iz(x86_fp80 noundef [[TMP7]], i32 noundef 1, x86_fp80 noundef [[TMP7]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg.addr = alloca x86_fp80, align 16
+  store x86_fp80 %arg, ptr %arg.addr, align 16
+  call void @_Z3usePv(ptr noundef nonnull %arg.addr)
+  %0 = load x86_fp80, ptr %arg.addr, align 16
+  call void (x86_fp80, i32, ...) @_Z5test2IeEvT_iz(x86_fp80 noundef %0, i32 noundef 1, x86_fp80 noundef %0)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testI6IntIntEvT_(i64 %arg.coerce) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testI6IntIntEvT_(
+; CHECK-SAME: i64 [[ARG_COERCE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG:%.*]] = alloca [[STRUCT_INTINT:%.*]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP1]], 87960930222080
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP3]], i8 -1, i64 8, i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    store i64 [[TMP0]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    store i64 [[ARG_COERCE]], ptr [[ARG]], align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[AGG_TMP_SROA_0_0_COPYLOAD:%.*]] = load i64, ptr [[ARG]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (i64, i32, ...) @_Z5test2I6IntIntEvT_iz(i64 [[AGG_TMP_SROA_0_0_COPYLOAD]], i32 noundef 1, i64 [[AGG_TMP_SROA_0_0_COPYLOAD]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg = alloca %struct.IntInt, align 8
+  store i64 %arg.coerce, ptr %arg, align 8
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  %agg.tmp.sroa.0.0.copyload = load i64, ptr %arg, align 8
+  call void (i64, i32, ...) @_Z5test2I6IntIntEvT_iz(i64 %agg.tmp.sroa.0.0.copyload, i32 noundef 1, i64 %agg.tmp.sroa.0.0.copyload)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testI10Int64Int64EvT_(i64 %arg.coerce0, i64 %arg.coerce1) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testI10Int64Int64EvT_(
+; CHECK-SAME: i64 [[ARG_COERCE0:%.*]], i64 [[ARG_COERCE1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG:%.*]] = alloca [[STRUCT_INT64INT64:%.*]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP4]], i8 -1, i64 16, i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 [[TMP0]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    store i64 [[ARG_COERCE0]], ptr [[ARG]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds { i64, i64 }, ptr [[ARG]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 [[TMP1]], ptr [[TMP11]], align 8
+; CHECK-NEXT:    store i64 [[ARG_COERCE1]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[AGG_TMP_SROA_0_0_COPYLOAD:%.*]] = load i64, ptr [[ARG]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[AGG_TMP_SROA_2_0_COPYLOAD:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load i64, ptr [[TMP17]], align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i64 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (i64, i64, i32, ...) @_Z5test2I10Int64Int64EvT_iz(i64 [[AGG_TMP_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP_SROA_2_0_COPYLOAD]], i32 noundef 1, i64 [[AGG_TMP_SROA_0_0_COPYLOAD]], i64 [[AGG_TMP_SROA_2_0_COPYLOAD]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg = alloca %struct.Int64Int64, align 8
+  store i64 %arg.coerce0, ptr %arg, align 8
+  %0 = getelementptr inbounds { i64, i64 }, ptr %arg, i64 0, i32 1
+  store i64 %arg.coerce1, ptr %0, align 8
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  %agg.tmp.sroa.0.0.copyload = load i64, ptr %arg, align 8
+  %agg.tmp.sroa.2.0.copyload = load i64, ptr %0, align 8
+  call void (i64, i64, i32, ...) @_Z5test2I10Int64Int64EvT_iz(i64 %agg.tmp.sroa.0.0.copyload, i64 %agg.tmp.sroa.2.0.copyload, i32 noundef 1, i64 %agg.tmp.sroa.0.0.copyload, i64 %agg.tmp.sroa.2.0.copyload)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testI12DoubleDoubleEvT_(double %arg.coerce0, double %arg.coerce1) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testI12DoubleDoubleEvT_(
+; CHECK-SAME: double [[ARG_COERCE0:%.*]], double [[ARG_COERCE1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG:%.*]] = alloca [[STRUCT_DOUBLEDOUBLE:%.*]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP4]], i8 -1, i64 16, i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 [[TMP0]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    store double [[ARG_COERCE0]], ptr [[ARG]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds { double, double }, ptr [[ARG]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i64 [[TMP1]], ptr [[TMP11]], align 8
+; CHECK-NEXT:    store double [[ARG_COERCE1]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[AGG_TMP_SROA_0_0_COPYLOAD:%.*]] = load double, ptr [[ARG]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[AGG_TMP_SROA_2_0_COPYLOAD:%.*]] = load double, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load i64, ptr [[TMP17]], align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i64 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (double, double, i32, ...) @_Z5test2I12DoubleDoubleEvT_iz(double [[AGG_TMP_SROA_0_0_COPYLOAD]], double [[AGG_TMP_SROA_2_0_COPYLOAD]], i32 noundef 1, double [[AGG_TMP_SROA_0_0_COPYLOAD]], double [[AGG_TMP_SROA_2_0_COPYLOAD]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg = alloca %struct.DoubleDouble, align 8
+  store double %arg.coerce0, ptr %arg, align 8
+  %0 = getelementptr inbounds { double, double }, ptr %arg, i64 0, i32 1
+  store double %arg.coerce1, ptr %0, align 8
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  %agg.tmp.sroa.0.0.copyload = load double, ptr %arg, align 8
+  %agg.tmp.sroa.2.0.copyload = load double, ptr %0, align 8
+  call void (double, double, i32, ...) @_Z5test2I12DoubleDoubleEvT_iz(double %agg.tmp.sroa.0.0.copyload, double %agg.tmp.sroa.2.0.copyload, i32 noundef 1, double %agg.tmp.sroa.0.0.copyload, double %agg.tmp.sroa.2.0.copyload)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testI7Double4EvT_(ptr noundef byval(%struct.Double4) align 8 %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testI7Double4EvT_(
+; CHECK-SAME: ptr noundef byval([[STRUCT_DOUBLE4:%.*]]) align 8 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i64 [[TMP0]], 87960930222080
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_param_tls, i64 32, i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 @__msan_param_tls, ptr align 8 [[TMP5]], i64 32, i1 false)
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), ptr align 8 [[TMP8]], i64 32, i1 false)
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 176) to ptr), ptr align 8 [[TMP11]], i64 32, i1 false)
+; CHECK-NEXT:    store i64 32, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (ptr, i32, ...) @_Z5test2I7Double4EvT_iz(ptr noundef nonnull byval([[STRUCT_DOUBLE4]]) align 8 [[ARG]], i32 noundef 1, ptr noundef nonnull byval([[STRUCT_DOUBLE4]]) align 8 [[ARG]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  call void (ptr, i32, ...) @_Z5test2I7Double4EvT_iz(ptr noundef nonnull byval(%struct.Double4) align 8 %arg, i32 noundef 1, ptr noundef nonnull byval(%struct.Double4) align 8 %arg)
+  ret void
+}
+
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
+
+define linkonce_odr dso_local void @_Z4testI11DoubleFloatEvT_(double %arg.coerce0, float %arg.coerce1) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testI11DoubleFloatEvT_(
+; CHECK-SAME: double [[ARG_COERCE0:%.*]], float [[ARG_COERCE1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARG:%.*]] = alloca [[STRUCT_DOUBLEFLOAT:%.*]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP4]], i8 -1, i64 16, i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    store i64 [[TMP0]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    store double [[ARG_COERCE0]], ptr [[ARG]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds { double, float }, ptr [[ARG]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[TMP11]], align 8
+; CHECK-NEXT:    store float [[ARG_COERCE1]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[AGG_TMP_SROA_0_0_COPYLOAD:%.*]] = load double, ptr [[ARG]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i64, ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[AGG_TMP_SROA_2_0_COPYLOAD:%.*]] = load float, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    [[_MSLD1:%.*]] = load i32, ptr [[TMP17]], align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    store i32 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    store i32 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    store i64 [[_MSLD]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    store i32 [[_MSLD1]], ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    store i64 0, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (double, float, i32, ...) @_Z5test2I11DoubleFloatEvT_iz(double [[AGG_TMP_SROA_0_0_COPYLOAD]], float [[AGG_TMP_SROA_2_0_COPYLOAD]], i32 noundef 1, double [[AGG_TMP_SROA_0_0_COPYLOAD]], float [[AGG_TMP_SROA_2_0_COPYLOAD]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arg = alloca %struct.DoubleFloat, align 8
+  store double %arg.coerce0, ptr %arg, align 8
+  %0 = getelementptr inbounds { double, float }, ptr %arg, i64 0, i32 1
+  store float %arg.coerce1, ptr %0, align 8
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  %agg.tmp.sroa.0.0.copyload = load double, ptr %arg, align 8
+  %agg.tmp.sroa.2.0.copyload = load float, ptr %0, align 8
+  call void (double, float, i32, ...) @_Z5test2I11DoubleFloatEvT_iz(double %agg.tmp.sroa.0.0.copyload, float %agg.tmp.sroa.2.0.copyload, i32 noundef 1, double %agg.tmp.sroa.0.0.copyload, float %agg.tmp.sroa.2.0.copyload)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testI11LongDouble2EvT_(ptr noundef byval(%struct.LongDouble2) align 16 %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testI11LongDouble2EvT_(
+; CHECK-SAME: ptr noundef byval([[STRUCT_LONGDOUBLE2:%.*]]) align 16 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i64 [[TMP0]], 87960930222080
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_param_tls, i64 32, i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 @__msan_param_tls, ptr align 8 [[TMP5]], i64 32, i1 false)
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), ptr align 8 [[TMP8]], i64 32, i1 false)
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 176) to ptr), ptr align 8 [[TMP11]], i64 32, i1 false)
+; CHECK-NEXT:    store i64 32, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (ptr, i32, ...) @_Z5test2I11LongDouble2EvT_iz(ptr noundef nonnull byval([[STRUCT_LONGDOUBLE2]]) align 16 [[ARG]], i32 noundef 1, ptr noundef nonnull byval([[STRUCT_LONGDOUBLE2]]) align 16 [[ARG]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  call void (ptr, i32, ...) @_Z5test2I11LongDouble2EvT_iz(ptr noundef nonnull byval(%struct.LongDouble2) align 16 %arg, i32 noundef 1, ptr noundef nonnull byval(%struct.LongDouble2) align 16 %arg)
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4testI11LongDouble4EvT_(ptr noundef byval(%struct.LongDouble4) align 16 %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4testI11LongDouble4EvT_(
+; CHECK-SAME: ptr noundef byval([[STRUCT_LONGDOUBLE4:%.*]]) align 16 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i64 [[TMP0]], 87960930222080
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_param_tls, i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 @__msan_param_tls, ptr align 8 [[TMP5]], i64 64, i1 false)
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), ptr align 8 [[TMP8]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 176) to ptr), ptr align 8 [[TMP11]], i64 64, i1 false)
+; CHECK-NEXT:    store i64 64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (ptr, i32, ...) @_Z5test2I11LongDouble4EvT_iz(ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], i32 noundef 1, ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  call void (ptr, i32, ...) @_Z5test2I11LongDouble4EvT_iz(ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, i32 noundef 1, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg)
+  ret void
+}
+
+declare void @_Z3usePv(ptr noundef) local_unnamed_addr #3
+
+define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef signext %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IcEvT_iz(
+; CHECK-SAME: i8 noundef signext [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #4
+
+declare void @llvm.va_start(ptr) #5
+
+declare void @llvm.va_end(ptr) #5
+
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #4
+
+define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IiEvT_iz(
+; CHECK-SAME: i32 noundef [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IfEvT_iz(
+; CHECK-SAME: float noundef [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IdEvT_iz(
+; CHECK-SAME: double noundef [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2IeEvT_iz(x86_fp80 noundef %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IeEvT_iz(
+; CHECK-SAME: x86_fp80 noundef [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(
+; CHECK-SAME: i64 [[T_COERCE:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(i64 %t.coerce0, i64 %t.coerce1, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(
+; CHECK-SAME: i64 [[T_COERCE0:%.*]], i64 [[T_COERCE1:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(double %t.coerce0, double %t.coerce1, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(
+; CHECK-SAME: double [[T_COERCE0:%.*]], double [[T_COERCE1:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(ptr noundef byval(%struct.Double4) align 8 %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(
+; CHECK-SAME: ptr noundef byval([[STRUCT_DOUBLE4:%.*]]) align 8 [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(double %t.coerce0, float %t.coerce1, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(
+; CHECK-SAME: double [[T_COERCE0:%.*]], float [[T_COERCE1:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(ptr noundef byval(%struct.LongDouble2) align 16 %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(
+; CHECK-SAME: ptr noundef byval([[STRUCT_LONGDOUBLE2:%.*]]) align 16 [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(ptr noundef byval(%struct.LongDouble4) align 16 %t, i32 noundef %n, ...) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(
+; CHECK-SAME: ptr noundef byval([[STRUCT_LONGDOUBLE4:%.*]]) align 16 [[T:%.*]], i32 noundef [[N:%.*]], ...) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 176, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8, i64 [[TMP1]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP2]], i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 800)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 -1, i64 24, i1 false)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP14]], 87960930222080
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP16]], ptr align 16 [[TMP2]], i64 176, i1 false)
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[ARGS]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP2]], i32 176
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %args = alloca [1 x %struct.__va_list_tag], align 16
+  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.va_start(ptr nonnull %args)
+  call void @_Z3usePv(ptr noundef nonnull %args)
+  call void @llvm.va_end(ptr nonnull %args)
+  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  ret void
+}
+
+define linkonce_odr dso_local void @_Z4test3I11LongDouble4EvT_(ptr noundef byval(%struct.LongDouble4) align 16 %arg) sanitize_memory {
+; CHECK-LABEL: define linkonce_odr dso_local void @_Z4test3I11LongDouble4EvT_(
+; CHECK-SAME: ptr noundef byval([[STRUCT_LONGDOUBLE4:%.*]]) align 16 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i64 [[TMP0]], 87960930222080
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_param_tls, i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARG]])
+; CHECK-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 @__msan_param_tls, ptr align 8 [[TMP5]], i64 64, i1 false)
+; CHECK-NEXT:    store i32 0, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), ptr align 8 [[TMP8]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), ptr align 8 [[TMP11]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), ptr align 8 [[TMP14]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 264) to ptr), ptr align 8 [[TMP17]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i64 [[TMP18]], 87960930222080
+; CHECK-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 328) to ptr), ptr align 8 [[TMP20]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 392) to ptr), ptr align 8 [[TMP23]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP24:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = xor i64 [[TMP24]], 87960930222080
+; CHECK-NEXT:    [[TMP26:%.*]] = inttoptr i64 [[TMP25]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 456) to ptr), ptr align 8 [[TMP26]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP28:%.*]] = xor i64 [[TMP27]], 87960930222080
+; CHECK-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 520) to ptr), ptr align 8 [[TMP29]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], 87960930222080
+; CHECK-NEXT:    [[TMP32:%.*]] = inttoptr i64 [[TMP31]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 584) to ptr), ptr align 8 [[TMP32]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP34:%.*]] = xor i64 [[TMP33]], 87960930222080
+; CHECK-NEXT:    [[TMP35:%.*]] = inttoptr i64 [[TMP34]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 648) to ptr), ptr align 8 [[TMP35]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP36:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP37:%.*]] = xor i64 [[TMP36]], 87960930222080
+; CHECK-NEXT:    [[TMP38:%.*]] = inttoptr i64 [[TMP37]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 712) to ptr), ptr align 8 [[TMP38]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP39:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP40:%.*]] = xor i64 [[TMP39]], 87960930222080
+; CHECK-NEXT:    [[TMP41:%.*]] = inttoptr i64 [[TMP40]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 176) to ptr), ptr align 8 [[TMP41]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP42:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP43:%.*]] = xor i64 [[TMP42]], 87960930222080
+; CHECK-NEXT:    [[TMP44:%.*]] = inttoptr i64 [[TMP43]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 240) to ptr), ptr align 8 [[TMP44]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP45:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i64 [[TMP45]], 87960930222080
+; CHECK-NEXT:    [[TMP47:%.*]] = inttoptr i64 [[TMP46]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 304) to ptr), ptr align 8 [[TMP47]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP48:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP49:%.*]] = xor i64 [[TMP48]], 87960930222080
+; CHECK-NEXT:    [[TMP50:%.*]] = inttoptr i64 [[TMP49]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 368) to ptr), ptr align 8 [[TMP50]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP51:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP52:%.*]] = xor i64 [[TMP51]], 87960930222080
+; CHECK-NEXT:    [[TMP53:%.*]] = inttoptr i64 [[TMP52]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 432) to ptr), ptr align 8 [[TMP53]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP54:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP55:%.*]] = xor i64 [[TMP54]], 87960930222080
+; CHECK-NEXT:    [[TMP56:%.*]] = inttoptr i64 [[TMP55]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 496) to ptr), ptr align 8 [[TMP56]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP57:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = xor i64 [[TMP57]], 87960930222080
+; CHECK-NEXT:    [[TMP59:%.*]] = inttoptr i64 [[TMP58]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 560) to ptr), ptr align 8 [[TMP59]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP60:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP61:%.*]] = xor i64 [[TMP60]], 87960930222080
+; CHECK-NEXT:    [[TMP62:%.*]] = inttoptr i64 [[TMP61]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 624) to ptr), ptr align 8 [[TMP62]], i64 64, i1 false)
+; CHECK-NEXT:    [[TMP63:%.*]] = ptrtoint ptr [[ARG]] to i64
+; CHECK-NEXT:    [[TMP64:%.*]] = xor i64 [[TMP63]], 87960930222080
+; CHECK-NEXT:    [[TMP65:%.*]] = inttoptr i64 [[TMP64]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 688) to ptr), ptr align 8 [[TMP65]], i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr align 8 inttoptr (i64 add (i64 ptrtoint (ptr @__msan_va_arg_tls to i64), i64 752) to ptr), i8 0, i32 48, i1 false)
+; CHECK-NEXT:    store i64 1280, ptr @__msan_va_arg_overflow_size_tls, align 8
+; CHECK-NEXT:    call void (ptr, i32, ...) @_Z5test2I11LongDouble4EvT_iz(ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], i32 noundef 20, ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]], ptr noundef nonnull byval([[STRUCT_LONGDOUBLE4]]) align 16 [[ARG]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @_Z3usePv(ptr noundef nonnull %arg)
+  call void (ptr, i32, ...) @_Z5test2I11LongDouble4EvT_iz(ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, i32 noundef 20, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg, ptr noundef nonnull byval(%struct.LongDouble4) align 16 %arg)
+  ret void
+}
diff --git a/llvm/test/MC/AArch64/armv9.6a-mpam.s b/llvm/test/MC/AArch64/armv9.6a-mpam.s
new file mode 100644
index 0000000000000000000000000000000000000000..c0696efd3ccebba4a9acf3079be25eea4385f58e
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.6a-mpam.s
@@ -0,0 +1,45 @@
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding < %s 2> %t | FileCheck %s --check-prefix=CHECK
+// RUN: FileCheck --check-prefix=CHECK-RO < %t %s
+
+//------------------------------------------------------------------------------
+// Armv9.6-A FEAT_MPAM Extensions
+//------------------------------------------------------------------------------
+
+msr MPAMBWIDR_EL1, x0
+msr MPAMBW3_EL3, x0
+msr MPAMBW2_EL2, x0
+msr MPAMBW1_EL1, x0
+msr MPAMBW1_EL12, x0
+msr MPAMBW0_EL1, x0
+msr MPAMBWCAP_EL2, x0
+msr MPAMBWSM_EL1, x0
+
+mrs x0, MPAMBWIDR_EL1
+mrs x0, MPAMBW3_EL3
+mrs x0, MPAMBW2_EL2
+mrs x0, MPAMBW1_EL1
+mrs x0, MPAMBW1_EL12
+mrs x0, MPAMBW0_EL1
+mrs x0, MPAMBWCAP_EL2
+mrs x0, MPAMBWSM_EL1
+
+//CHECK: msr     MPAMBW3_EL3, x0                 // encoding: [0x80,0xa5,0x1e,0xd5]
+//CHECK: msr     MPAMBW2_EL2, x0                 // encoding: [0x80,0xa5,0x1c,0xd5]
+//CHECK: msr     MPAMBW1_EL1, x0                 // encoding: [0x80,0xa5,0x18,0xd5]
+//CHECK: msr     MPAMBW1_EL12, x0                // encoding: [0x80,0xa5,0x1d,0xd5]
+//CHECK: msr     MPAMBW0_EL1, x0                 // encoding: [0xa0,0xa5,0x18,0xd5]
+//CHECK: msr     MPAMBWCAP_EL2, x0               // encoding: [0xc0,0xa5,0x1c,0xd5]
+//CHECK: msr     MPAMBWSM_EL1, x0                // encoding: [0xe0,0xa5,0x18,0xd5]
+
+//CHECK-RO: error: expected writable system register or pstate
+//CHECK-RO: msr MPAMBWIDR_EL1, x0
+//CHECK-RO:     ^
+
+//CHECK: mrs     x0, MPAMBWIDR_EL1               // encoding: [0xa0,0xa4,0x38,0xd5]
+//CHECK: mrs     x0, MPAMBW3_EL3                 // encoding: [0x80,0xa5,0x3e,0xd5]
+//CHECK: mrs     x0, MPAMBW2_EL2                 // encoding: [0x80,0xa5,0x3c,0xd5]
+//CHECK: mrs     x0, MPAMBW1_EL1                 // encoding: [0x80,0xa5,0x38,0xd5]
+//CHECK: mrs     x0, MPAMBW1_EL12                // encoding: [0x80,0xa5,0x3d,0xd5]
+//CHECK: mrs     x0, MPAMBW0_EL1                 // encoding: [0xa0,0xa5,0x38,0xd5]
+//CHECK: mrs     x0, MPAMBWCAP_EL2               // encoding: [0xc0,0xa5,0x3c,0xd5]
+//CHECK: mrs     x0, MPAMBWSM_EL1                // encoding: [0xe0,0xa5,0x38,0xd5]
diff --git a/llvm/test/MC/AArch64/global-tagging.ll b/llvm/test/MC/AArch64/global-tagging.ll
index fcaadead1159756f18390ffeea46c1cfc562608b..4961ec04c5fba4b5ae958a9dd3581554475cc50f 100644
--- a/llvm/test/MC/AArch64/global-tagging.ll
+++ b/llvm/test/MC/AArch64/global-tagging.ll
@@ -43,9 +43,17 @@
 ; CHECK-YAML-NEXT: -
 
 ; CHECK-ASM: .memtag internal_four
+; CHECK-ASM .p2align        4
+; CHECK-ASM: .size   internal_four, 16
 ; CHECK-ASM: .memtag four
+; CHECK-ASM .p2align        4
+; CHECK-ASM: .size   four, 16
 ; CHECK-ASM: .memtag sixteen
+; CHECK-ASM .p2align        4
+; CHECK-ASM: .size   sixteen, 16
 ; CHECK-ASM: .memtag huge
+; CHECK-ASM .p2align        4
+; CHECK-ASM: .size   huge, 16777232
 ; CHECK-ASM-NOT: .memtag specialcaselisted
 
 @internal_four = internal global i32 1, sanitize_memtag
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s
index 090973428066f1f6253207f5d4876e2b8c0c0837..4ae91340386b6501b7ada7cde1dc9bcc96a175d1 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_err.s
@@ -1,812 +1,813 @@
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16,+wavefrontsize32 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16,+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX11 --implicit-check-not=error: %s
 
+v_ceil_f16_e32 v128.h, 0xfe0b
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
+
 v_ceil_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_ceil_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_ceil_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
+
+v_ceil_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
+
+v_ceil_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
 v_ceil_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_ceil_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_ceil_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_ceil_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_ceil_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
 v_ceil_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+
+v_ceil_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+
+v_ceil_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+
+v_ceil_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+
+v_ceil_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
+
+v_ceil_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
 v_cos_f16_e32 v128, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cos_f16_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_cos_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cos_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cos_f16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v128, 0xaf123456
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cos_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cos_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255, v255
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_f16_f32_e32 v128, 0xaf123456
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_i16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v128, 0xaf123456 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v128, 0xaf123456 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:36: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255, v1
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_i16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255, v255
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_i16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v255, v255 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
 v_cvt_f16_i16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_i16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_i16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_i16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_f16_i16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
 v_cvt_f16_u16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_u16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_u16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_u16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_f16_u16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f32_f16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f32_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
+
+v_cvt_f32_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:28: error: invalid operand for instruction
 
 v_cvt_i16_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_i16_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_i16_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i32_i16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i32_i16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i32_i16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
+
+v_cvt_i32_i16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
 v_cvt_norm_i16_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cvt_norm_i16_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_i16_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_norm_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
 v_cvt_norm_u16_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cvt_norm_u16_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_u16_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_u16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_norm_u16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
 v_cvt_u16_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_u16_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_u16_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_u16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_u32_u16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_exp_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_exp_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_exp_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_floor_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_floor_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_floor_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u32_u16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_fract_f16_e32 v128, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_u32_u16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_fract_f16_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_u32_u16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_fract_f16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_exp_f16_e32 v128.h, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v128.h, 0xfe0b
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v128, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_log_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_log_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_log_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_not_b16_e32 v128, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:17: error: invalid operand for instruction
 
-v_not_b16_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_not_b16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_rcp_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_rcp_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_rcp_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_rndne_f16_e32 v128, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_rndne_f16_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_rndne_f16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_rsq_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_rsq_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_floor_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_rsq_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fract_f16_e32 v128, 0xfe0b
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sat_pk_u8_i16_e32 v199, v5
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v255, v1
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sin_f16_e32 v128, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sin_f16_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sin_f16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sqrt_f16_e32 v128.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fract_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v255.l, v1.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fract_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v5.l, v199.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v128.h, 0xfe0b
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_swap_b16_e32 v128.l, v0.l
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_swap_b16_e32 v0.l, s0
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_swap_b16_e32 v0.l, 0
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_swap_b16_e32 v0.l, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_trunc_f16_e32 v128, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_frexp_exp_i16_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_trunc_f16_e32 v255, v1
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_frexp_exp_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_trunc_f16_e32 v5, v199
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_frexp_exp_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_ceil_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_ceil_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_ceil_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_ceil_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cos_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cos_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v128, 0xaf123456 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v128, 0xfe0b
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v255, v1
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255, v255 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_i32_i16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_u32_u16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_exp_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_exp_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v128.h, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_floor_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_floor_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_fract_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_fract_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
 v_log_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_log_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_not_b16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_not_b16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_rcp_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_rcp_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_rndne_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_rndne_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_rsq_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_rsq_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_sat_pk_u8_i16_e32 v199, v5 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_sin_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v128, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sin_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v255, v1
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_trunc_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_trunc_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_ceil_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_ceil_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v128.h, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_ceil_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_ceil_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_cos_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_cos_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v128, 0xaf123456 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_cvt_i32_i16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_cvt_u32_u16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_exp_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_exp_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_floor_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_floor_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v128, 0xfe0b
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fract_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v255, v1
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fract_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_log_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_log_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_not_b16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v128.h, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_not_b16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_rcp_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_rcp_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_rndne_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_rndne_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
 v_rsq_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_rsq_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_sat_pk_u8_i16_e32 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_sin_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_sin_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_sqrt_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_sqrt_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_trunc_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_trunc_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_floor_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_floor_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_floor_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_floor_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:21: error: invalid operand for instruction
 
-v_floor_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sat_pk_u8_i16_e32 v199, v5
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_floor_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sat_pk_u8_i16_e32 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_floor_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sat_pk_u8_i16_e32 v199, v5 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_rcp_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v128, 0xfe0b
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rcp_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v255, v1
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rcp_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_rcp_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_rcp_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rcp_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_rcp_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_sqrt_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_sqrt_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_sqrt_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v128.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255.h, v1.h
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
 v_sqrt_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_sqrt_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_rsq_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_rsq_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255.l, v1.l
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_rsq_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_rsq_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_rsq_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5.h, v199.h
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_rsq_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_rsq_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_log_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5.l, v199.l
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_log_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_log_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_log_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v0.l, 0
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_log_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v0.l, 0xfe0b
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_log_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v0.l, s0
+// GFX11: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_log_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v128.l, v0.l
+// GFX11: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_exp_f16_e32 v128.h, 0xfe0b
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v128, 0xfe0b
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_exp_f16_e32 v255.h, v1.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v255, v1
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_exp_f16_e32 v5.h, v199.h
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_exp_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_exp_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v5, v199
+// GFX11: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_exp_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_exp_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: :[[@LINE-1]]:26: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_promote.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_promote.s
index f49939ac96168111f76297ff48784df59f33ce65..1d441720280ca263fae67b3a377dd46c712140cb 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop1_t16_promote.s
@@ -1,2010 +1,2011 @@
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX11 --implicit-check-not=_e32 %s
 
 v_ceil_f16 v128, 0xfe0b
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v128, 0xfe0b             ; encoding: [0x80,0x00,0xdc,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_ceil_f16 v255, -1
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, -1                 ; encoding: [0xff,0x00,0xdc,0xd5,0xc1,0x00,0x00,0x00]
 
 v_ceil_f16 v255, 0.5
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, 0.5                ; encoding: [0xff,0x00,0xdc,0xd5,0xf0,0x00,0x00,0x00]
 
 v_ceil_f16 v255, exec_hi
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, exec_hi            ; encoding: [0xff,0x00,0xdc,0xd5,0x7f,0x00,0x00,0x00]
 
 v_ceil_f16 v255, exec_lo
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, exec_lo            ; encoding: [0xff,0x00,0xdc,0xd5,0x7e,0x00,0x00,0x00]
 
 v_ceil_f16 v255, m0
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, m0                 ; encoding: [0xff,0x00,0xdc,0xd5,0x7d,0x00,0x00,0x00]
 
 v_ceil_f16 v255, null
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, null               ; encoding: [0xff,0x00,0xdc,0xd5,0x7c,0x00,0x00,0x00]
 
 v_ceil_f16 v255, s1
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, s1                 ; encoding: [0xff,0x00,0xdc,0xd5,0x01,0x00,0x00,0x00]
 
 v_ceil_f16 v255, s105
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, s105               ; encoding: [0xff,0x00,0xdc,0xd5,0x69,0x00,0x00,0x00]
 
 v_ceil_f16 v255, src_scc
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, src_scc            ; encoding: [0xff,0x00,0xdc,0xd5,0xfd,0x00,0x00,0x00]
 
 v_ceil_f16 v255, ttmp15
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, ttmp15             ; encoding: [0xff,0x00,0xdc,0xd5,0x7b,0x00,0x00,0x00]
 
 v_ceil_f16 v255, v1
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, v1                 ; encoding: [0xff,0x00,0xdc,0xd5,0x01,0x01,0x00,0x00]
+
+v_ceil_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ceil_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdc,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_ceil_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_ceil_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdc,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_ceil_f16 v255, v127
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, v127               ; encoding: [0xff,0x00,0xdc,0xd5,0x7f,0x01,0x00,0x00]
+
+v_ceil_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ceil_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdc,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_ceil_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_ceil_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdc,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_ceil_f16 v255, vcc_hi
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, vcc_hi             ; encoding: [0xff,0x00,0xdc,0xd5,0x6b,0x00,0x00,0x00]
 
 v_ceil_f16 v255, vcc_lo
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v255, vcc_lo             ; encoding: [0xff,0x00,0xdc,0xd5,0x6a,0x00,0x00,0x00]
 
 v_ceil_f16 v5, v199
-// GFX11: v_ceil_f16_e64
+// GFX11: v_ceil_f16_e64 v5, v199                 ; encoding: [0x05,0x00,0xdc,0xd5,0xc7,0x01,0x00,0x00]
+
+v_ceil_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_ceil_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdc,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_ceil_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_ceil_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdc,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cos_f16 v128, 0xfe0b
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xe1,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cos_f16 v255, -1
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xe1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cos_f16 v255, 0.5
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xe1,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cos_f16 v255, exec_hi
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xe1,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cos_f16 v255, exec_lo
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xe1,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cos_f16 v255, m0
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xe1,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cos_f16 v255, null
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, null                ; encoding: [0xff,0x00,0xe1,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cos_f16 v255, s1
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xe1,0xd5,0x01,0x00,0x00,0x00]
 
 v_cos_f16 v255, s105
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xe1,0xd5,0x69,0x00,0x00,0x00]
 
 v_cos_f16 v255, src_scc
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xe1,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cos_f16 v255, ttmp15
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xe1,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cos_f16 v255, v1
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xe1,0xd5,0x01,0x01,0x00,0x00]
+
+v_cos_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cos_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe1,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cos_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_cos_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe1,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cos_f16 v255, v127
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xe1,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cos_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cos_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe1,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cos_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_cos_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe1,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cos_f16 v255, vcc_hi
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xe1,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cos_f16 v255, vcc_lo
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xe1,0xd5,0x6a,0x00,0x00,0x00]
 
 v_cos_f16 v5, v199
-// GFX11: v_cos_f16_e64
+// GFX11: v_cos_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xe1,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_f16_f32 v128.l, 0xaf123456
-// GFX11: v_cvt_f16_f32_e64
+v_cos_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cos_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe1,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_cos_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_cos_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe1,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_f16_f32 v128.h, 0xaf123456
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v128.h, 0xaf123456 op_sel:[0,1] ; encoding: [0x80,0x40,0x8a,0xd5,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cvt_f16_f32 v255.l, -1
-// GFX11: v_cvt_f16_f32_e64
+v_cvt_f16_f32 v128.l, 0xaf123456
+// GFX11: v_cvt_f16_f32_e64 v128.l, 0xaf123456    ; encoding: [0x80,0x00,0x8a,0xd5,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
-v_cvt_f16_f32 v255.h, -1
-// GFX11: v_cvt_f16_f32_e64
+v_cvt_f16_f32 v255.h v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.h, v1 op_sel:[0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x40,0x8a,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_f32 v255.l, 0.5
-// GFX11: v_cvt_f16_f32_e64
+v_cvt_f16_f32 v255.h, -1
+// GFX11: v_cvt_f16_f32_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, 0.5
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, exec_hi
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, exec_hi
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, exec_lo
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, exec_lo
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, m0
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, m0
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, null
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, null
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, s1
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, s1
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, s105
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, s105
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, src_scc
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, src_scc
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, ttmp15
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, ttmp15
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, v1
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, v1
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, v1 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_f32 v255.l, v255
-// GFX11: v_cvt_f16_f32_e64
+v_cvt_f16_f32 v255.h, v1 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.h, v1 op_sel:[0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x40,0x8a,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cvt_f16_f32 v255.h, v255
-// GFX11: v_cvt_f16_f32_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, v255 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xff,0x01,0x00,0x00]
 
-v_cvt_f16_f32 v255.l, vcc_hi
-// GFX11: v_cvt_f16_f32_e64
+v_cvt_f16_f32 v255.h, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.h, v255 op_sel:[0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x40,0x8a,0xd5,0xe9,0x00,0x00,0x00,0xff,0x77,0x39,0x05]
 
-v_cvt_f16_f32 v255.h, vcc_hi
-// GFX11: v_cvt_f16_f32_e64
+v_cvt_f16_f32 v255.h, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.h, v255 op_sel:[0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x40,0x8a,0xd5,0xfa,0x00,0x00,0x00,0xff,0x1b,0x00,0xff]
 
-v_cvt_f16_f32 v255.l, vcc_lo
-// GFX11: v_cvt_f16_f32_e64
+v_cvt_f16_f32 v255.h, vcc_hi
+// GFX11: v_cvt_f16_f32_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, vcc_lo
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_i16 v128.l, 0xfe0b
-// GFX11: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, -1
-// GFX11: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, 0.5
-// GFX11: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, exec_hi
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_f32_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, exec_lo
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, -1
+// GFX11: v_cvt_f16_f32_e64 v255.l, -1            ; encoding: [0xff,0x00,0x8a,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, m0
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, 0.5
+// GFX11: v_cvt_f16_f32_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0x8a,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, null
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, exec_hi
+// GFX11: v_cvt_f16_f32_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0x8a,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, s1
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, exec_lo
+// GFX11: v_cvt_f16_f32_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0x8a,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, s105
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, m0
+// GFX11: v_cvt_f16_f32_e64 v255.l, m0            ; encoding: [0xff,0x00,0x8a,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, src_scc
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, null
+// GFX11: v_cvt_f16_f32_e64 v255.l, null          ; encoding: [0xff,0x00,0x8a,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, ttmp15
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, s1
+// GFX11: v_cvt_f16_f32_e64 v255.l, s1            ; encoding: [0xff,0x00,0x8a,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, v1.l
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, s105
+// GFX11: v_cvt_f16_f32_e64 v255.l, s105          ; encoding: [0xff,0x00,0x8a,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, v127.l
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, src_scc
+// GFX11: v_cvt_f16_f32_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0x8a,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, vcc_hi
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, ttmp15
+// GFX11: v_cvt_f16_f32_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0x8a,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, vcc_lo
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v1
+// GFX11: v_cvt_f16_f32_e64 v255.l, v1            ; encoding: [0xff,0x00,0x8a,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v5.l, v199.l
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.l, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x8a,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v1 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.l, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x8a,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v255
+// GFX11: v_cvt_f16_f32_e64 v255.l, v255          ; encoding: [0xff,0x00,0x8a,0xd5,0xff,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x8a,0xd5,0xe9,0x00,0x00,0x00,0xff,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v255 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_f32_e64_dpp v255.l, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x8a,0xd5,0xfa,0x00,0x00,0x00,0xff,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, vcc_hi
+// GFX11: v_cvt_f16_f32_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0x8a,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, vcc_lo
+// GFX11: v_cvt_f16_f32_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0x8a,0xd5,0x6a,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v128.h, 0xfe0b
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd1,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_f16_i16 v128.l, 0xfe0b
+// GFX11: v_cvt_f16_i16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd1,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, -1
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, 0.5
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, exec_hi
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, exec_lo
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, m0
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, null
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, s1
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, s105
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, src_scc
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, ttmp15
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, v1.h
-// GFX11: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.h, v127.h
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd1,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, vcc_hi
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v255.h, vcc_lo
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v5.h, v199.h
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v127.h
+// GFX11: v_cvt_f16_i16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd1,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
 v_cvt_f16_i16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+// GFX11: v_cvt_f16_i16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, vcc_hi
+// GFX11: v_cvt_f16_i16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, vcc_lo
+// GFX11: v_cvt_f16_i16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.l, -1
+// GFX11: v_cvt_f16_i16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.l, 0.5
+// GFX11: v_cvt_f16_i16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v128.l, 0xfe0b
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, exec_hi
+// GFX11: v_cvt_f16_i16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd1,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, -1
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, exec_lo
+// GFX11: v_cvt_f16_i16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd1,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, 0.5
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, m0
+// GFX11: v_cvt_f16_i16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd1,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, exec_hi
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, null
+// GFX11: v_cvt_f16_i16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd1,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, exec_lo
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, s1
+// GFX11: v_cvt_f16_i16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd1,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, m0
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, s105
+// GFX11: v_cvt_f16_i16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd1,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, null
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, src_scc
+// GFX11: v_cvt_f16_i16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd1,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, s1
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, ttmp15
+// GFX11: v_cvt_f16_i16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd1,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, s105
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v1.l
+// GFX11: v_cvt_f16_i16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd1,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, src_scc
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v255.l, ttmp15
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_f16_u16 v255.l, v1.l
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v127.l
+// GFX11: v_cvt_f16_i16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd1,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v127.l
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v255.l, vcc_hi
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_f16_u16 v255.l, vcc_lo
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, vcc_hi
+// GFX11: v_cvt_f16_i16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd1,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v5.l, v199.l
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, vcc_lo
+// GFX11: v_cvt_f16_i16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd1,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.h, v199.h
+// GFX11: v_cvt_f16_i16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd1,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd1,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd1,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_f16_u16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.l, v199.l
+// GFX11: v_cvt_f16_i16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd1,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd1,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_i16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd1,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_f16_u16 v128.h, 0xfe0b
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd0,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_f16_u16 v128.l, 0xfe0b
+// GFX11: v_cvt_f16_u16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd0,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, -1
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, 0.5
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, exec_hi
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, exec_lo
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, m0
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, null
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, s1
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, s105
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, src_scc
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, ttmp15
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, v1.h
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd0,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_f16_u16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_f16_u16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cvt_f16_u16 v255.h, v127.h
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd0,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_f16_u16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_f16_u16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cvt_f16_u16 v255.h, vcc_hi
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, vcc_lo
-// GFX11: v_cvt_f16_u16_e64
-
-v_cvt_f16_u16 v5.h, v199.h
-// GFX11: v_cvt_f16_u16_e64
+// GFX11: v_cvt_f16_u16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, -1
+// GFX11: v_cvt_f16_u16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, 0.5
+// GFX11: v_cvt_f16_u16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, exec_hi
+// GFX11: v_cvt_f16_u16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd0,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, exec_lo
+// GFX11: v_cvt_f16_u16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd0,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, m0
+// GFX11: v_cvt_f16_u16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd0,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, null
+// GFX11: v_cvt_f16_u16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd0,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_f32_f16 v5, v199.l
-// GFX11: v_cvt_f32_f16_e64
+v_cvt_f16_u16 v255.l, s1
+// GFX11: v_cvt_f16_u16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd0,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_f32_f16 v5, v199.h
-// GFX11: v_cvt_f32_f16_e64
+v_cvt_f16_u16 v255.l, s105
+// GFX11: v_cvt_f16_u16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd0,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v128.l, 0xfe0b
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, src_scc
+// GFX11: v_cvt_f16_u16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd0,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, -1
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, ttmp15
+// GFX11: v_cvt_f16_u16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd0,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, 0.5
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v1.l
+// GFX11: v_cvt_f16_u16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd0,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, exec_hi
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v255.l, exec_lo
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, m0
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v127.l
+// GFX11: v_cvt_f16_u16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd0,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, null
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v255.l, s1
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, s105
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, vcc_hi
+// GFX11: v_cvt_f16_u16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd0,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, src_scc
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, vcc_lo
+// GFX11: v_cvt_f16_u16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd0,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, ttmp15
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.h, v199.h
+// GFX11: v_cvt_f16_u16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd0,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, v1.l
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd0,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v255.l, v127.l
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd0,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, vcc_hi
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.l, v199.l
+// GFX11: v_cvt_f16_u16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd0,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, vcc_lo
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd0,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v5.l, v199.l
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f16_u16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd0,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.h
+// GFX11: v_cvt_f32_f16_e64 v5, v199.h op_sel:[1,0] ; encoding: [0x05,0x08,0x8b,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f32_f16_e64_dpp v5, v199.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x8b,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f32_f16_e64_dpp v5, v199.h op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x8b,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.l
+// GFX11: v_cvt_f32_f16_e64 v5, v199.l            ; encoding: [0x05,0x00,0x8b,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_f32_f16_e64_dpp v5, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x8b,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_f32_f16_e64_dpp v5, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x8b,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_i16_f16 v128.h, 0xfe0b
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_i16_f16 v128.l, 0xfe0b
+// GFX11: v_cvt_i16_f16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, -1
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, 0.5
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, exec_hi
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, exec_lo
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, m0
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, null
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, s1
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, s105
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, src_scc
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, ttmp15
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, v1.h
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd3,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cvt_i16_f16 v255.h, v127.h
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd3,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cvt_i16_f16 v255.h, vcc_hi
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, vcc_lo
-// GFX11: v_cvt_i16_f16_e64
+// GFX11: v_cvt_i16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v5.h, v199.h
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, -1
+// GFX11: v_cvt_i16_f16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd3,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, 0.5
+// GFX11: v_cvt_i16_f16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd3,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, exec_hi
+// GFX11: v_cvt_i16_f16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd3,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, exec_lo
+// GFX11: v_cvt_i16_f16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd3,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, m0
+// GFX11: v_cvt_i16_f16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd3,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
-
-v_cvt_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_i16_f16_e64
-
-v_cvt_i32_i16 v5, v199
-// GFX11: v_cvt_i32_i16_e64
-
-v_cvt_norm_i16_f16 v128.l, 0xfe0b
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, null
+// GFX11: v_cvt_i16_f16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd3,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, -1
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, s1
+// GFX11: v_cvt_i16_f16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd3,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, 0.5
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, s105
+// GFX11: v_cvt_i16_f16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd3,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, exec_hi
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, src_scc
+// GFX11: v_cvt_i16_f16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd3,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, exec_lo
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, ttmp15
+// GFX11: v_cvt_i16_f16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd3,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, m0
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v1.l
+// GFX11: v_cvt_i16_f16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd3,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, null
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v255.l, s1
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, s105
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v127.l
+// GFX11: v_cvt_i16_f16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd3,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, src_scc
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v255.l, ttmp15
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, v1.l
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, vcc_hi
+// GFX11: v_cvt_i16_f16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd3,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, v127.l
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, vcc_lo
+// GFX11: v_cvt_i16_f16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, vcc_hi
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.h, v199.h
+// GFX11: v_cvt_i16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, vcc_lo
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v5.l, v199.l
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.l, v199.l
+// GFX11: v_cvt_i16_f16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_i16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i32_i16 v5, v199
+// GFX11: v_cvt_i32_i16_e64 v5, v199              ; encoding: [0x05,0x00,0xea,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i32_i16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_i32_i16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xea,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_i32_i16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_i32_i16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xea,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_norm_i16_f16 v128.h, 0xfe0b
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xe3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_norm_i16_f16 v128.l, 0xfe0b
+// GFX11: v_cvt_norm_i16_f16_e64 v128.l, 0xfe0b   ; encoding: [0x80,0x00,0xe3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, -1
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, 0.5
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, exec_hi
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, exec_lo
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, m0
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, null
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, s1
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, s105
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, src_scc
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, ttmp15
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, v1.h
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe3,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, v127.h
-// GFX11: v_cvt_norm_i16_f16_e64
-
-v_cvt_norm_i16_f16 v255.h, vcc_hi
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v255.h, vcc_lo
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v5.h, v199.h
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v127.h
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe3,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
 v_cvt_norm_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, vcc_hi
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, vcc_lo
+// GFX11: v_cvt_norm_i16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.l, -1
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, -1       ; encoding: [0xff,0x00,0xe3,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.l, 0.5
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, 0.5      ; encoding: [0xff,0x00,0xe3,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v128.l, 0xfe0b
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, exec_hi
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, exec_hi  ; encoding: [0xff,0x00,0xe3,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, -1
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, exec_lo
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, exec_lo  ; encoding: [0xff,0x00,0xe3,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, 0.5
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, m0
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, m0       ; encoding: [0xff,0x00,0xe3,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, exec_hi
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, null
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, null     ; encoding: [0xff,0x00,0xe3,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, exec_lo
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, s1
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, s1       ; encoding: [0xff,0x00,0xe3,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, m0
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, s105
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, s105     ; encoding: [0xff,0x00,0xe3,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, null
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, src_scc
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, src_scc  ; encoding: [0xff,0x00,0xe3,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, s1
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, ttmp15
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, ttmp15   ; encoding: [0xff,0x00,0xe3,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, s105
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v1.l
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, v1.l     ; encoding: [0xff,0x00,0xe3,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, src_scc
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v255.l, ttmp15
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v255.l, v1.l
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v127.l
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, v127.l   ; encoding: [0xff,0x00,0xe3,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v127.l
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v255.l, vcc_hi
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v255.l, vcc_lo
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, vcc_hi
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, vcc_hi   ; encoding: [0xff,0x00,0xe3,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v5.l, v199.l
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, vcc_lo
+// GFX11: v_cvt_norm_i16_f16_e64 v255.l, vcc_lo   ; encoding: [0xff,0x00,0xe3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.h, v199.h
+// GFX11: v_cvt_norm_i16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xe3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xe3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xe3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.l, v199.l
+// GFX11: v_cvt_norm_i16_f16_e64 v5.l, v199.l     ; encoding: [0x05,0x00,0xe3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_i16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_norm_u16_f16 v128.h, 0xfe0b
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xe4,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_norm_u16_f16 v128.l, 0xfe0b
+// GFX11: v_cvt_norm_u16_f16_e64 v128.l, 0xfe0b   ; encoding: [0x80,0x00,0xe4,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, -1
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, 0.5
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, exec_hi
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, exec_lo
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, m0
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, null
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, s1
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, s105
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, src_scc
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, ttmp15
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, v1.h
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe4,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, v127.h
-// GFX11: v_cvt_norm_u16_f16_e64
-
-v_cvt_norm_u16_f16 v255.h, vcc_hi
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v255.h, vcc_lo
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v5.h, v199.h
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v127.h
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe4,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
 v_cvt_norm_u16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, vcc_hi
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, vcc_lo
+// GFX11: v_cvt_norm_u16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, -1
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, -1       ; encoding: [0xff,0x00,0xe4,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, 0.5
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, 0.5      ; encoding: [0xff,0x00,0xe4,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v128.l, 0xfe0b
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, exec_hi
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, exec_hi  ; encoding: [0xff,0x00,0xe4,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, -1
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, exec_lo
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, exec_lo  ; encoding: [0xff,0x00,0xe4,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, 0.5
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, m0
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, m0       ; encoding: [0xff,0x00,0xe4,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, exec_hi
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, null
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, null     ; encoding: [0xff,0x00,0xe4,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, exec_lo
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, s1
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, s1       ; encoding: [0xff,0x00,0xe4,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, m0
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, s105
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, s105     ; encoding: [0xff,0x00,0xe4,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, null
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, src_scc
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, src_scc  ; encoding: [0xff,0x00,0xe4,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, s1
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, ttmp15
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, ttmp15   ; encoding: [0xff,0x00,0xe4,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, s105
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v1.l
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, v1.l     ; encoding: [0xff,0x00,0xe4,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, src_scc
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v255.l, ttmp15
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_u16_f16 v255.l, v1.l
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v127.l
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, v127.l   ; encoding: [0xff,0x00,0xe4,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v127.l
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v255.l, vcc_hi
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_u16_f16 v255.l, vcc_lo
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, vcc_hi
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, vcc_hi   ; encoding: [0xff,0x00,0xe4,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v5.l, v199.l
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, vcc_lo
+// GFX11: v_cvt_norm_u16_f16_e64 v255.l, vcc_lo   ; encoding: [0xff,0x00,0xe4,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.h, v199.h
+// GFX11: v_cvt_norm_u16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xe4,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xe4,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xe4,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.l, v199.l
+// GFX11: v_cvt_norm_u16_f16_e64 v5.l, v199.l     ; encoding: [0x05,0x00,0xe4,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe4,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_norm_u16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe4,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_u16_f16 v128.h, 0xfe0b
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd2,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_u16_f16 v128.l, 0xfe0b
+// GFX11: v_cvt_u16_f16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd2,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, -1
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, 0.5
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, exec_hi
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, exec_lo
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, m0
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, null
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, s1
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, s105
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, src_scc
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, ttmp15
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, v1.h
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cvt_u16_f16 v255.h, v127.h
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd2,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cvt_u16_f16 v255.h, vcc_hi
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, vcc_lo
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v5.h, v199.h
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, -1
+// GFX11: v_cvt_u16_f16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd2,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, 0.5
+// GFX11: v_cvt_u16_f16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd2,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, exec_hi
+// GFX11: v_cvt_u16_f16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd2,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, exec_lo
+// GFX11: v_cvt_u16_f16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd2,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, m0
+// GFX11: v_cvt_u16_f16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd2,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, null
+// GFX11: v_cvt_u16_f16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd2,0xd5,0x7c,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, s1
+// GFX11: v_cvt_u16_f16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd2,0xd5,0x01,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, s105
+// GFX11: v_cvt_u16_f16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd2,0xd5,0x69,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, src_scc
+// GFX11: v_cvt_u16_f16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd2,0xd5,0xfd,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, ttmp15
+// GFX11: v_cvt_u16_f16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd2,0xd5,0x7b,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, v1.l
+// GFX11: v_cvt_u16_f16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cvt_u16_f16 v255.l, v127.l
+// GFX11: v_cvt_u16_f16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd2,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
+
+v_cvt_u16_f16 v255.l, vcc_hi
+// GFX11: v_cvt_u16_f16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd2,0xd5,0x6b,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, vcc_lo
+// GFX11: v_cvt_u16_f16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd2,0xd5,0x6a,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v5.h, v199.h
+// GFX11: v_cvt_u16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd2,0xd5,0xc7,0x01,0x00,0x00]
 
 v_cvt_u16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_u16_f16_e64
+// GFX11: v_cvt_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd2,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd2,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
+
+v_cvt_u16_f16 v5.l, v199.l
+// GFX11: v_cvt_u16_f16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd2,0xd5,0xc7,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd2,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_cvt_u16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd2,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_u32_u16 v5, v199
-// GFX11: v_cvt_u32_u16_e64
+// GFX11: v_cvt_u32_u16_e64 v5, v199              ; encoding: [0x05,0x00,0xeb,0xd5,0xc7,0x01,0x00,0x00]
+
+v_cvt_u32_u16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_cvt_u32_u16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xeb,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_cvt_u32_u16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_cvt_u32_u16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xeb,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_exp_f16 v128, 0xfe0b
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd8,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_exp_f16 v255, -1
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd8,0xd5,0xc1,0x00,0x00,0x00]
 
 v_exp_f16 v255, 0.5
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd8,0xd5,0xf0,0x00,0x00,0x00]
 
 v_exp_f16 v255, exec_hi
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd8,0xd5,0x7f,0x00,0x00,0x00]
 
 v_exp_f16 v255, exec_lo
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd8,0xd5,0x7e,0x00,0x00,0x00]
 
 v_exp_f16 v255, m0
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd8,0xd5,0x7d,0x00,0x00,0x00]
 
 v_exp_f16 v255, null
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd8,0xd5,0x7c,0x00,0x00,0x00]
 
 v_exp_f16 v255, s1
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd8,0xd5,0x01,0x00,0x00,0x00]
 
 v_exp_f16 v255, s105
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd8,0xd5,0x69,0x00,0x00,0x00]
 
 v_exp_f16 v255, src_scc
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd8,0xd5,0xfd,0x00,0x00,0x00]
 
 v_exp_f16 v255, ttmp15
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd8,0xd5,0x7b,0x00,0x00,0x00]
 
 v_exp_f16 v255, v1
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd8,0xd5,0x01,0x01,0x00,0x00]
+
+v_exp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_exp_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_exp_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_exp_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_exp_f16 v255, v127
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd8,0xd5,0x7f,0x01,0x00,0x00]
+
+v_exp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_exp_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd8,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_exp_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_exp_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd8,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_exp_f16 v255, vcc_hi
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd8,0xd5,0x6b,0x00,0x00,0x00]
 
 v_exp_f16 v255, vcc_lo
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd8,0xd5,0x6a,0x00,0x00,0x00]
 
 v_exp_f16 v5, v199
-// GFX11: v_exp_f16_e64
+// GFX11: v_exp_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd8,0xd5,0xc7,0x01,0x00,0x00]
+
+v_exp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_exp_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd8,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_exp_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_exp_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd8,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_floor_f16 v128, 0xfe0b
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xdb,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_floor_f16 v255, -1
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xdb,0xd5,0xc1,0x00,0x00,0x00]
 
 v_floor_f16 v255, 0.5
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xdb,0xd5,0xf0,0x00,0x00,0x00]
 
 v_floor_f16 v255, exec_hi
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xdb,0xd5,0x7f,0x00,0x00,0x00]
 
 v_floor_f16 v255, exec_lo
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xdb,0xd5,0x7e,0x00,0x00,0x00]
 
 v_floor_f16 v255, m0
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xdb,0xd5,0x7d,0x00,0x00,0x00]
 
 v_floor_f16 v255, null
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, null              ; encoding: [0xff,0x00,0xdb,0xd5,0x7c,0x00,0x00,0x00]
 
 v_floor_f16 v255, s1
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xdb,0xd5,0x01,0x00,0x00,0x00]
 
 v_floor_f16 v255, s105
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xdb,0xd5,0x69,0x00,0x00,0x00]
 
 v_floor_f16 v255, src_scc
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xdb,0xd5,0xfd,0x00,0x00,0x00]
 
 v_floor_f16 v255, ttmp15
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xdb,0xd5,0x7b,0x00,0x00,0x00]
 
 v_floor_f16 v255, v1
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xdb,0xd5,0x01,0x01,0x00,0x00]
 
-v_floor_f16 v255, v127
-// GFX11: v_floor_f16_e64
+v_floor_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_floor_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_floor_f16 v255, vcc_hi
-// GFX11: v_floor_f16_e64
+v_floor_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_floor_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_floor_f16 v255, vcc_lo
-// GFX11: v_floor_f16_e64
+v_floor_f16 v255, v127
+// GFX11: v_floor_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xdb,0xd5,0x7f,0x01,0x00,0x00]
+
+v_floor_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_floor_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdb,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_floor_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_floor_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdb,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
+
+v_floor_f16 v255, vcc_hi
+// GFX11: v_floor_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xdb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_floor_f16 v255, vcc_lo
+// GFX11: v_floor_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xdb,0xd5,0x6a,0x00,0x00,0x00]
 
 v_floor_f16 v5, v199
-// GFX11: v_floor_f16_e64
+// GFX11: v_floor_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xdb,0xd5,0xc7,0x01,0x00,0x00]
+
+v_floor_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_floor_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdb,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_floor_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_floor_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdb,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_fract_f16 v128, 0xfe0b
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xdf,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_fract_f16 v255, -1
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xdf,0xd5,0xc1,0x00,0x00,0x00]
 
 v_fract_f16 v255, 0.5
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xdf,0xd5,0xf0,0x00,0x00,0x00]
 
 v_fract_f16 v255, exec_hi
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xdf,0xd5,0x7f,0x00,0x00,0x00]
 
 v_fract_f16 v255, exec_lo
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xdf,0xd5,0x7e,0x00,0x00,0x00]
 
 v_fract_f16 v255, m0
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xdf,0xd5,0x7d,0x00,0x00,0x00]
 
 v_fract_f16 v255, null
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, null              ; encoding: [0xff,0x00,0xdf,0xd5,0x7c,0x00,0x00,0x00]
 
 v_fract_f16 v255, s1
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xdf,0xd5,0x01,0x00,0x00,0x00]
 
 v_fract_f16 v255, s105
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xdf,0xd5,0x69,0x00,0x00,0x00]
 
 v_fract_f16 v255, src_scc
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xdf,0xd5,0xfd,0x00,0x00,0x00]
 
 v_fract_f16 v255, ttmp15
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xdf,0xd5,0x7b,0x00,0x00,0x00]
 
 v_fract_f16 v255, v1
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xdf,0xd5,0x01,0x01,0x00,0x00]
+
+v_fract_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fract_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdf,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_fract_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_fract_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdf,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_fract_f16 v255, v127
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xdf,0xd5,0x7f,0x01,0x00,0x00]
+
+v_fract_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fract_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdf,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_fract_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_fract_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdf,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_fract_f16 v255, vcc_hi
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xdf,0xd5,0x6b,0x00,0x00,0x00]
 
 v_fract_f16 v255, vcc_lo
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xdf,0xd5,0x6a,0x00,0x00,0x00]
 
 v_fract_f16 v5, v199
-// GFX11: v_fract_f16_e64
+// GFX11: v_fract_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xdf,0xd5,0xc7,0x01,0x00,0x00]
+
+v_fract_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_fract_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdf,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_fract_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_fract_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdf,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
+
+v_frexp_exp_i16_f16 v128.h, 0xfe0b
+// GFX11: v_frexp_exp_i16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xda,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_frexp_exp_i16_f16 v128.l, 0xfe0b
-// GFX11: v_frexp_exp_i16_f16_e64
+// GFX11: v_frexp_exp_i16_f16_e64 v128.l, 0xfe0b  ; encoding: [0x80,0x00,0xda,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, -1
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, -1
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0xc1,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, 0.5
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, 0.5
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0xf0,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, exec_hi
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, exec_hi
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7f,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, exec_lo
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, exec_lo
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7e,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, m0
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, m0
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7d,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, null
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, null
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7c,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, s1
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, s1
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x01,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, s105
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, s105
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x69,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, src_scc
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, src_scc
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0xfd,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, ttmp15
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, ttmp15
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, v1.l
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v1.h
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xda,0xd5,0x01,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, v127.l
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xda,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.l, vcc_hi
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xda,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.l, vcc_lo
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v127.h
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xda,0xd5,0x7f,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v5.l, v199.l
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xda,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xda,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, vcc_hi
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x6b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, vcc_lo
+// GFX11: v_frexp_exp_i16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x6a,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, -1
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, -1      ; encoding: [0xff,0x00,0xda,0xd5,0xc1,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, 0.5
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, 0.5     ; encoding: [0xff,0x00,0xda,0xd5,0xf0,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, exec_hi
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, exec_hi ; encoding: [0xff,0x00,0xda,0xd5,0x7f,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v128.h, 0xfe0b
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, exec_lo
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, exec_lo ; encoding: [0xff,0x00,0xda,0xd5,0x7e,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, -1
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, m0
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, m0      ; encoding: [0xff,0x00,0xda,0xd5,0x7d,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, 0.5
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, null
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, null    ; encoding: [0xff,0x00,0xda,0xd5,0x7c,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, exec_hi
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, s1
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, s1      ; encoding: [0xff,0x00,0xda,0xd5,0x01,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, exec_lo
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, s105
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, s105    ; encoding: [0xff,0x00,0xda,0xd5,0x69,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, m0
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, src_scc
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, src_scc ; encoding: [0xff,0x00,0xda,0xd5,0xfd,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, null
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, ttmp15
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, ttmp15  ; encoding: [0xff,0x00,0xda,0xd5,0x7b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, s1
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v1.l
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, v1.l    ; encoding: [0xff,0x00,0xda,0xd5,0x01,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, s105
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xda,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.h, src_scc
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xda,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.h, ttmp15
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v127.l
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, v127.l  ; encoding: [0xff,0x00,0xda,0xd5,0x7f,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, v1.h
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xda,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.h, v127.h
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xda,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.h, vcc_hi
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, vcc_hi
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, vcc_hi  ; encoding: [0xff,0x00,0xda,0xd5,0x6b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, vcc_lo
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, vcc_lo
+// GFX11: v_frexp_exp_i16_f16_e64 v255.l, vcc_lo  ; encoding: [0xff,0x00,0xda,0xd5,0x6a,0x00,0x00,0x00]
 
 v_frexp_exp_i16_f16 v5.h, v199.h
-// GFX11: v_frexp_exp_i16_f16_e64
+// GFX11: v_frexp_exp_i16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xda,0xd5,0xc7,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
-
-v_frexp_exp_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xda,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
 v_frexp_exp_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xda,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v5.l, v199.l
+// GFX11: v_frexp_exp_i16_f16_e64 v5.l, v199.l    ; encoding: [0x05,0x00,0xda,0xd5,0xc7,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xda,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX11: v_frexp_exp_i16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xda,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_frexp_mant_f16 v128, 0xfe0b
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v128, 0xfe0b       ; encoding: [0x80,0x00,0xd9,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_frexp_mant_f16 v255, -1
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, -1           ; encoding: [0xff,0x00,0xd9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, 0.5
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, 0.5          ; encoding: [0xff,0x00,0xd9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, exec_hi
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, exec_hi      ; encoding: [0xff,0x00,0xd9,0xd5,0x7f,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, exec_lo
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, exec_lo      ; encoding: [0xff,0x00,0xd9,0xd5,0x7e,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, m0
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, m0           ; encoding: [0xff,0x00,0xd9,0xd5,0x7d,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, null
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, null         ; encoding: [0xff,0x00,0xd9,0xd5,0x7c,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, s1
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, s1           ; encoding: [0xff,0x00,0xd9,0xd5,0x01,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, s105
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, s105         ; encoding: [0xff,0x00,0xd9,0xd5,0x69,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, src_scc
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, src_scc      ; encoding: [0xff,0x00,0xd9,0xd5,0xfd,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, ttmp15
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, ttmp15       ; encoding: [0xff,0x00,0xd9,0xd5,0x7b,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, v1
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, v1           ; encoding: [0xff,0x00,0xd9,0xd5,0x01,0x01,0x00,0x00]
+
+v_frexp_mant_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_mant_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_frexp_mant_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_frexp_mant_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_frexp_mant_f16 v255, v127
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, v127         ; encoding: [0xff,0x00,0xd9,0xd5,0x7f,0x01,0x00,0x00]
+
+v_frexp_mant_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_mant_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd9,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_frexp_mant_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_frexp_mant_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd9,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_frexp_mant_f16 v255, vcc_hi
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, vcc_hi       ; encoding: [0xff,0x00,0xd9,0xd5,0x6b,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, vcc_lo
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v255, vcc_lo       ; encoding: [0xff,0x00,0xd9,0xd5,0x6a,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v5, v199
-// GFX11: v_frexp_mant_f16_e64
+// GFX11: v_frexp_mant_f16_e64 v5, v199           ; encoding: [0x05,0x00,0xd9,0xd5,0xc7,0x01,0x00,0x00]
+
+v_frexp_mant_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_frexp_mant_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd9,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_frexp_mant_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_frexp_mant_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd9,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_log_f16 v128, 0xfe0b
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd7,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_log_f16 v255, -1
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd7,0xd5,0xc1,0x00,0x00,0x00]
 
 v_log_f16 v255, 0.5
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd7,0xd5,0xf0,0x00,0x00,0x00]
 
 v_log_f16 v255, exec_hi
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd7,0xd5,0x7f,0x00,0x00,0x00]
 
 v_log_f16 v255, exec_lo
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd7,0xd5,0x7e,0x00,0x00,0x00]
 
 v_log_f16 v255, m0
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd7,0xd5,0x7d,0x00,0x00,0x00]
 
 v_log_f16 v255, null
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd7,0xd5,0x7c,0x00,0x00,0x00]
 
 v_log_f16 v255, s1
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd7,0xd5,0x01,0x00,0x00,0x00]
 
 v_log_f16 v255, s105
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd7,0xd5,0x69,0x00,0x00,0x00]
 
 v_log_f16 v255, src_scc
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd7,0xd5,0xfd,0x00,0x00,0x00]
 
 v_log_f16 v255, ttmp15
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd7,0xd5,0x7b,0x00,0x00,0x00]
 
 v_log_f16 v255, v1
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd7,0xd5,0x01,0x01,0x00,0x00]
+
+v_log_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_log_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd7,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_log_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_log_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd7,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_log_f16 v255, v127
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd7,0xd5,0x7f,0x01,0x00,0x00]
+
+v_log_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_log_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd7,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_log_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_log_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd7,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_log_f16 v255, vcc_hi
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd7,0xd5,0x6b,0x00,0x00,0x00]
 
 v_log_f16 v255, vcc_lo
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd7,0xd5,0x6a,0x00,0x00,0x00]
 
 v_log_f16 v5, v199
-// GFX11: v_log_f16_e64
+// GFX11: v_log_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd7,0xd5,0xc7,0x01,0x00,0x00]
+
+v_log_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_log_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd7,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_log_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_log_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd7,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_not_b16 v128, 0xfe0b
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xe9,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_not_b16 v255, -1
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, -1                  ; encoding: [0xff,0x00,0xe9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_not_b16 v255, 0.5
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xe9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_not_b16 v255, exec_hi
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xe9,0xd5,0x7f,0x00,0x00,0x00]
 
 v_not_b16 v255, exec_lo
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xe9,0xd5,0x7e,0x00,0x00,0x00]
 
 v_not_b16 v255, m0
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, m0                  ; encoding: [0xff,0x00,0xe9,0xd5,0x7d,0x00,0x00,0x00]
 
 v_not_b16 v255, null
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, null                ; encoding: [0xff,0x00,0xe9,0xd5,0x7c,0x00,0x00,0x00]
 
 v_not_b16 v255, s1
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, s1                  ; encoding: [0xff,0x00,0xe9,0xd5,0x01,0x00,0x00,0x00]
 
 v_not_b16 v255, s105
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, s105                ; encoding: [0xff,0x00,0xe9,0xd5,0x69,0x00,0x00,0x00]
 
 v_not_b16 v255, src_scc
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xe9,0xd5,0xfd,0x00,0x00,0x00]
 
 v_not_b16 v255, ttmp15
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xe9,0xd5,0x7b,0x00,0x00,0x00]
 
 v_not_b16 v255, v1
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, v1                  ; encoding: [0xff,0x00,0xe9,0xd5,0x01,0x01,0x00,0x00]
+
+v_not_b16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_not_b16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_not_b16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_not_b16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_not_b16 v255, v127
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, v127                ; encoding: [0xff,0x00,0xe9,0xd5,0x7f,0x01,0x00,0x00]
+
+v_not_b16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_not_b16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe9,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_not_b16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_not_b16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe9,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_not_b16 v255, vcc_hi
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xe9,0xd5,0x6b,0x00,0x00,0x00]
 
 v_not_b16 v255, vcc_lo
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xe9,0xd5,0x6a,0x00,0x00,0x00]
 
 v_not_b16 v5, v199
-// GFX11: v_not_b16_e64
+// GFX11: v_not_b16_e64 v5, v199                  ; encoding: [0x05,0x00,0xe9,0xd5,0xc7,0x01,0x00,0x00]
+
+v_not_b16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_not_b16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe9,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_not_b16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_not_b16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe9,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_rcp_f16 v128, 0xfe0b
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd4,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_rcp_f16 v255, -1
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd4,0xd5,0xc1,0x00,0x00,0x00]
 
 v_rcp_f16 v255, 0.5
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd4,0xd5,0xf0,0x00,0x00,0x00]
 
 v_rcp_f16 v255, exec_hi
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd4,0xd5,0x7f,0x00,0x00,0x00]
 
 v_rcp_f16 v255, exec_lo
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd4,0xd5,0x7e,0x00,0x00,0x00]
 
 v_rcp_f16 v255, m0
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd4,0xd5,0x7d,0x00,0x00,0x00]
 
 v_rcp_f16 v255, null
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd4,0xd5,0x7c,0x00,0x00,0x00]
 
 v_rcp_f16 v255, s1
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd4,0xd5,0x01,0x00,0x00,0x00]
 
 v_rcp_f16 v255, s105
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd4,0xd5,0x69,0x00,0x00,0x00]
 
 v_rcp_f16 v255, src_scc
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd4,0xd5,0xfd,0x00,0x00,0x00]
 
 v_rcp_f16 v255, ttmp15
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd4,0xd5,0x7b,0x00,0x00,0x00]
 
 v_rcp_f16 v255, v1
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd4,0xd5,0x01,0x01,0x00,0x00]
+
+v_rcp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rcp_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd4,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_rcp_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_rcp_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd4,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_rcp_f16 v255, v127
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd4,0xd5,0x7f,0x01,0x00,0x00]
+
+v_rcp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rcp_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd4,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_rcp_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_rcp_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd4,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_rcp_f16 v255, vcc_hi
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd4,0xd5,0x6b,0x00,0x00,0x00]
 
 v_rcp_f16 v255, vcc_lo
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd4,0xd5,0x6a,0x00,0x00,0x00]
 
 v_rcp_f16 v5, v199
-// GFX11: v_rcp_f16_e64
+// GFX11: v_rcp_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd4,0xd5,0xc7,0x01,0x00,0x00]
+
+v_rcp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rcp_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd4,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_rcp_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_rcp_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd4,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_rndne_f16 v128, 0xfe0b
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xde,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_rndne_f16 v255, -1
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xde,0xd5,0xc1,0x00,0x00,0x00]
 
 v_rndne_f16 v255, 0.5
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xde,0xd5,0xf0,0x00,0x00,0x00]
 
 v_rndne_f16 v255, exec_hi
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xde,0xd5,0x7f,0x00,0x00,0x00]
 
 v_rndne_f16 v255, exec_lo
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xde,0xd5,0x7e,0x00,0x00,0x00]
 
 v_rndne_f16 v255, m0
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xde,0xd5,0x7d,0x00,0x00,0x00]
 
 v_rndne_f16 v255, null
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, null              ; encoding: [0xff,0x00,0xde,0xd5,0x7c,0x00,0x00,0x00]
 
 v_rndne_f16 v255, s1
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xde,0xd5,0x01,0x00,0x00,0x00]
 
 v_rndne_f16 v255, s105
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xde,0xd5,0x69,0x00,0x00,0x00]
 
 v_rndne_f16 v255, src_scc
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xde,0xd5,0xfd,0x00,0x00,0x00]
 
 v_rndne_f16 v255, ttmp15
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xde,0xd5,0x7b,0x00,0x00,0x00]
 
 v_rndne_f16 v255, v1
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xde,0xd5,0x01,0x01,0x00,0x00]
+
+v_rndne_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rndne_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xde,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_rndne_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_rndne_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xde,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_rndne_f16 v255, v127
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xde,0xd5,0x7f,0x01,0x00,0x00]
+
+v_rndne_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rndne_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xde,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_rndne_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_rndne_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xde,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_rndne_f16 v255, vcc_hi
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xde,0xd5,0x6b,0x00,0x00,0x00]
 
 v_rndne_f16 v255, vcc_lo
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xde,0xd5,0x6a,0x00,0x00,0x00]
 
 v_rndne_f16 v5, v199
-// GFX11: v_rndne_f16_e64
+// GFX11: v_rndne_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xde,0xd5,0xc7,0x01,0x00,0x00]
+
+v_rndne_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rndne_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xde,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_rndne_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_rndne_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xde,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_rsq_f16 v128, 0xfe0b
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd6,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_rsq_f16 v255, -1
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd6,0xd5,0xc1,0x00,0x00,0x00]
 
 v_rsq_f16 v255, 0.5
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd6,0xd5,0xf0,0x00,0x00,0x00]
 
 v_rsq_f16 v255, exec_hi
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd6,0xd5,0x7f,0x00,0x00,0x00]
 
 v_rsq_f16 v255, exec_lo
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd6,0xd5,0x7e,0x00,0x00,0x00]
 
 v_rsq_f16 v255, m0
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd6,0xd5,0x7d,0x00,0x00,0x00]
 
 v_rsq_f16 v255, null
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd6,0xd5,0x7c,0x00,0x00,0x00]
 
 v_rsq_f16 v255, s1
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd6,0xd5,0x01,0x00,0x00,0x00]
 
 v_rsq_f16 v255, s105
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd6,0xd5,0x69,0x00,0x00,0x00]
 
 v_rsq_f16 v255, src_scc
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd6,0xd5,0xfd,0x00,0x00,0x00]
 
 v_rsq_f16 v255, ttmp15
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd6,0xd5,0x7b,0x00,0x00,0x00]
 
 v_rsq_f16 v255, v1
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd6,0xd5,0x01,0x01,0x00,0x00]
+
+v_rsq_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rsq_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd6,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_rsq_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_rsq_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd6,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_rsq_f16 v255, v127
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd6,0xd5,0x7f,0x01,0x00,0x00]
+
+v_rsq_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rsq_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd6,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_rsq_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_rsq_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd6,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_rsq_f16 v255, vcc_hi
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd6,0xd5,0x6b,0x00,0x00,0x00]
 
 v_rsq_f16 v255, vcc_lo
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd6,0xd5,0x6a,0x00,0x00,0x00]
 
 v_rsq_f16 v5, v199
-// GFX11: v_rsq_f16_e64
+// GFX11: v_rsq_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd6,0xd5,0xc7,0x01,0x00,0x00]
+
+v_rsq_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_rsq_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd6,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_rsq_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_rsq_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd6,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_sat_pk_u8_i16 v199, v5
-// GFX11: v_sat_pk_u8_i16_e64
+// GFX11: v_sat_pk_u8_i16_e64 v199, v5            ; encoding: [0xc7,0x00,0xe2,0xd5,0x05,0x01,0x00,0x00]
+
+v_sat_pk_u8_i16 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sat_pk_u8_i16_e64_dpp v199, v5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xc7,0x00,0xe2,0xd5,0xe9,0x00,0x00,0x00,0x05,0x77,0x39,0x05]
+
+v_sat_pk_u8_i16 v199, v5 quad_perm:[3,2,1,0]
+// GFX11: v_sat_pk_u8_i16_e64_dpp v199, v5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xc7,0x00,0xe2,0xd5,0xfa,0x00,0x00,0x00,0x05,0x1b,0x00,0xff]
 
 v_sin_f16 v128, 0xfe0b
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xe0,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_sin_f16 v255, -1
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xe0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_sin_f16 v255, 0.5
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xe0,0xd5,0xf0,0x00,0x00,0x00]
 
 v_sin_f16 v255, exec_hi
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xe0,0xd5,0x7f,0x00,0x00,0x00]
 
 v_sin_f16 v255, exec_lo
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xe0,0xd5,0x7e,0x00,0x00,0x00]
 
 v_sin_f16 v255, m0
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xe0,0xd5,0x7d,0x00,0x00,0x00]
 
 v_sin_f16 v255, null
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, null                ; encoding: [0xff,0x00,0xe0,0xd5,0x7c,0x00,0x00,0x00]
 
 v_sin_f16 v255, s1
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xe0,0xd5,0x01,0x00,0x00,0x00]
 
 v_sin_f16 v255, s105
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xe0,0xd5,0x69,0x00,0x00,0x00]
 
 v_sin_f16 v255, src_scc
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xe0,0xd5,0xfd,0x00,0x00,0x00]
 
 v_sin_f16 v255, ttmp15
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xe0,0xd5,0x7b,0x00,0x00,0x00]
 
 v_sin_f16 v255, v1
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xe0,0xd5,0x01,0x01,0x00,0x00]
+
+v_sin_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sin_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe0,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sin_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_sin_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe0,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_sin_f16 v255, v127
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xe0,0xd5,0x7f,0x01,0x00,0x00]
+
+v_sin_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sin_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe0,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_sin_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_sin_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe0,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_sin_f16 v255, vcc_hi
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xe0,0xd5,0x6b,0x00,0x00,0x00]
 
 v_sin_f16 v255, vcc_lo
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xe0,0xd5,0x6a,0x00,0x00,0x00]
 
 v_sin_f16 v5, v199
-// GFX11: v_sin_f16_e64
+// GFX11: v_sin_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xe0,0xd5,0xc7,0x01,0x00,0x00]
+
+v_sin_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sin_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe0,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_sin_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_sin_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe0,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_sqrt_f16 v128, 0xfe0b
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v128, 0xfe0b             ; encoding: [0x80,0x00,0xd5,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_sqrt_f16 v255, -1
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, -1                 ; encoding: [0xff,0x00,0xd5,0xd5,0xc1,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, 0.5
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, 0.5                ; encoding: [0xff,0x00,0xd5,0xd5,0xf0,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, exec_hi
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, exec_hi            ; encoding: [0xff,0x00,0xd5,0xd5,0x7f,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, exec_lo
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, exec_lo            ; encoding: [0xff,0x00,0xd5,0xd5,0x7e,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, m0
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, m0                 ; encoding: [0xff,0x00,0xd5,0xd5,0x7d,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, null
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, null               ; encoding: [0xff,0x00,0xd5,0xd5,0x7c,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, s1
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, s1                 ; encoding: [0xff,0x00,0xd5,0xd5,0x01,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, s105
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, s105               ; encoding: [0xff,0x00,0xd5,0xd5,0x69,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, src_scc
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, src_scc            ; encoding: [0xff,0x00,0xd5,0xd5,0xfd,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, ttmp15
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, ttmp15             ; encoding: [0xff,0x00,0xd5,0xd5,0x7b,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, v1
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, v1                 ; encoding: [0xff,0x00,0xd5,0xd5,0x01,0x01,0x00,0x00]
+
+v_sqrt_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sqrt_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd5,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sqrt_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX11: v_sqrt_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd5,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_sqrt_f16 v255, v127
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, v127               ; encoding: [0xff,0x00,0xd5,0xd5,0x7f,0x01,0x00,0x00]
+
+v_sqrt_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sqrt_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd5,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_sqrt_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_sqrt_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd5,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_sqrt_f16 v255, vcc_hi
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, vcc_hi             ; encoding: [0xff,0x00,0xd5,0xd5,0x6b,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, vcc_lo
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v255, vcc_lo             ; encoding: [0xff,0x00,0xd5,0xd5,0x6a,0x00,0x00,0x00]
 
 v_sqrt_f16 v5, v199
-// GFX11: v_sqrt_f16_e64
+// GFX11: v_sqrt_f16_e64 v5, v199                 ; encoding: [0x05,0x00,0xd5,0xd5,0xc7,0x01,0x00,0x00]
+
+v_sqrt_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_sqrt_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd5,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_sqrt_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_sqrt_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd5,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_trunc_f16 v128, 0xfe0b
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xdd,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_trunc_f16 v255, -1
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xdd,0xd5,0xc1,0x00,0x00,0x00]
 
 v_trunc_f16 v255, 0.5
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xdd,0xd5,0xf0,0x00,0x00,0x00]
 
 v_trunc_f16 v255, exec_hi
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xdd,0xd5,0x7f,0x00,0x00,0x00]
 
 v_trunc_f16 v255, exec_lo
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xdd,0xd5,0x7e,0x00,0x00,0x00]
 
 v_trunc_f16 v255, m0
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xdd,0xd5,0x7d,0x00,0x00,0x00]
 
 v_trunc_f16 v255, null
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, null              ; encoding: [0xff,0x00,0xdd,0xd5,0x7c,0x00,0x00,0x00]
 
 v_trunc_f16 v255, s1
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xdd,0xd5,0x01,0x00,0x00,0x00]
 
 v_trunc_f16 v255, s105
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xdd,0xd5,0x69,0x00,0x00,0x00]
 
 v_trunc_f16 v255, src_scc
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xdd,0xd5,0xfd,0x00,0x00,0x00]
 
 v_trunc_f16 v255, ttmp15
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xdd,0xd5,0x7b,0x00,0x00,0x00]
 
 v_trunc_f16 v255, v1
-// GFX11: v_trunc_f16_e64
-
-v_trunc_f16 v255, v127
-// GFX11: v_trunc_f16_e64
-
-v_trunc_f16 v255, vcc_hi
-// GFX11: v_trunc_f16_e64
-
-v_trunc_f16 v255, vcc_lo
-// GFX11: v_trunc_f16_e64
-
-v_trunc_f16 v5, v199
-// GFX11: v_trunc_f16_e64
-
-v_ceil_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_ceil_f16_e64
-
-v_ceil_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_ceil_f16_e64
-
-v_ceil_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_ceil_f16_e64
-
-v_cos_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_cos_f16_e64
-
-v_cos_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_cos_f16_e64
-
-v_cos_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_cos_f16_e64
-
-v_cvt_f16_f32 v255.l, v1 quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v1 quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v255 quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f32_f16 v5, v199.l quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f32_f16_e64
-
-v_cvt_f32_f16 v5, v199.h quad_perm:[3,2,1,0]
-// GFX11: v_cvt_f32_f16_e64
-
-v_cvt_i32_i16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_cvt_i32_i16_e64
-
-v_cvt_u32_u16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_cvt_u32_u16_e64
-
-v_exp_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_exp_f16_e64
-
-v_exp_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_exp_f16_e64
-
-v_exp_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_exp_f16_e64
-
-v_floor_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_floor_f16_e64
-
-v_floor_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_floor_f16_e64
-
-v_floor_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_floor_f16_e64
-
-v_fract_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_fract_f16_e64
-
-v_fract_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_fract_f16_e64
-
-v_fract_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_fract_f16_e64
+// GFX11: v_trunc_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xdd,0xd5,0x01,0x01,0x00,0x00]
 
-v_frexp_mant_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_frexp_mant_f16_e64
-
-v_log_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_log_f16_e64
-
-v_log_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_log_f16_e64
-
-v_log_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_log_f16_e64
-
-v_not_b16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_not_b16_e64
-
-v_not_b16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_not_b16_e64
-
-v_not_b16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_not_b16_e64
-
-v_rcp_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_rcp_f16_e64
-
-v_rcp_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_rcp_f16_e64
-
-v_rcp_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_rcp_f16_e64
-
-v_rndne_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_rndne_f16_e64
-
-v_rndne_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_rndne_f16_e64
-
-v_rndne_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_rndne_f16_e64
-
-v_rsq_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_rsq_f16_e64
-
-v_rsq_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_rsq_f16_e64
-
-v_rsq_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_rsq_f16_e64
-
-v_sat_pk_u8_i16 v199, v5 quad_perm:[3,2,1,0]
-// GFX11: v_sat_pk_u8_i16_e64
-
-v_sin_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_sin_f16_e64
-
-v_sin_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_sin_f16_e64
-
-v_sin_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_sin_f16_e64
-
-v_sqrt_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_sqrt_f16_e64
-
-v_sqrt_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_sqrt_f16_e64
-
-v_sqrt_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_sqrt_f16_e64
+v_trunc_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_trunc_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdd,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
 v_trunc_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX11: v_trunc_f16_e64
-
-v_trunc_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdd,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_trunc_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX11: v_trunc_f16_e64
-
-v_ceil_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_ceil_f16_e64
-
-v_ceil_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_ceil_f16_e64
-
-v_ceil_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_ceil_f16_e64
-
-v_cos_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cos_f16_e64
-
-v_cos_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cos_f16_e64
-
-v_cos_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cos_f16_e64
-
-v_cvt_f16_f32 v255.l, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f16_f32_e64
-
-v_cvt_f32_f16 v5, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f32_f16_e64
-
-v_cvt_f32_f16 v5, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_f32_f16_e64
-
-v_cvt_i32_i16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_i32_i16_e64
-
-v_cvt_u32_u16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_cvt_u32_u16_e64
-
-v_exp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_exp_f16_e64
-
-v_exp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_exp_f16_e64
-
-v_exp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_exp_f16_e64
-
-v_floor_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_floor_f16_e64
-
-v_floor_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_floor_f16_e64
-
-v_floor_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_floor_f16_e64
-
-v_fract_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_fract_f16_e64
-
-v_fract_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_fract_f16_e64
-
-v_fract_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_fract_f16_e64
-
-v_frexp_mant_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_frexp_mant_f16_e64
-
-v_log_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_log_f16_e64
-
-v_log_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_log_f16_e64
-
-v_log_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_log_f16_e64
-
-v_not_b16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_not_b16_e64
-
-v_not_b16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_not_b16_e64
-
-v_not_b16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_not_b16_e64
-
-v_rcp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rcp_f16_e64
-
-v_rcp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rcp_f16_e64
-
-v_rcp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rcp_f16_e64
-
-v_rndne_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rndne_f16_e64
-
-v_rndne_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rndne_f16_e64
-
-v_rndne_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rndne_f16_e64
-
-v_rsq_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rsq_f16_e64
-
-v_rsq_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rsq_f16_e64
-
-v_rsq_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_rsq_f16_e64
-
-v_sat_pk_u8_i16 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sat_pk_u8_i16_e64
-
-v_sin_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sin_f16_e64
-
-v_sin_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sin_f16_e64
-
-v_sin_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sin_f16_e64
+v_trunc_f16 v255, v127
+// GFX11: v_trunc_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xdd,0xd5,0x7f,0x01,0x00,0x00]
 
-v_sqrt_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sqrt_f16_e64
+v_trunc_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_trunc_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdd,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_sqrt_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sqrt_f16_e64
+v_trunc_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX11: v_trunc_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdd,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_sqrt_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_sqrt_f16_e64
+v_trunc_f16 v255, vcc_hi
+// GFX11: v_trunc_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xdd,0xd5,0x6b,0x00,0x00,0x00]
 
-v_trunc_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_trunc_f16_e64
+v_trunc_f16 v255, vcc_lo
+// GFX11: v_trunc_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xdd,0xd5,0x6a,0x00,0x00,0x00]
 
-v_trunc_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_trunc_f16_e64
+v_trunc_f16 v5, v199
+// GFX11: v_trunc_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xdd,0xd5,0xc7,0x01,0x00,0x00]
 
 v_trunc_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_trunc_f16_e64
+// GFX11: v_trunc_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdd,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
 
+v_trunc_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX11: v_trunc_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdd,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s
index d4c2aa480dc73b80802dc4091455bcdcd213ebae..0ccad9c6730791a71cb01d7a7a8ca639121e8d83 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s
@@ -1,703 +1,704 @@
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s
 
 v_ceil_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_ceil_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_ceil_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_ceil_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
 v_ceil_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
+
+v_ceil_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
+
+v_ceil_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
 v_cos_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_cos_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cos_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cos_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v128.l, 0xaf123456
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cos_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255.l, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cos_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255.l, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cos_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cos_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_f16_f32_e32 v128.h, 0xaf123456
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v128.h, 0xaf123456 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v128.h, 0xaf123456 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v128.l, 0xaf123456
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v128.l, 0xaf123456 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v128.l, 0xaf123456 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_f32_e32 v255.h, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v255.h, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_f32_e32 v255.h, v255
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v128.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255.h, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v255.l, v1.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.l, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255.l, v1
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255.l, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255.l, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255.l, v255
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_f32_e32 v255.l, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_f32_e32 v255.l, v255 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_i16_e32 v128.h, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_i16_e32 v128.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_i16_e32 v255.h, v1.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.h, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_i16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_f16_i16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.l, v1.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_i16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v128.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.l, v1.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.h, v199.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.l, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.l, v199.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_i16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
 v_cvt_f16_u16_e32 v128.h, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_u16_e32 v128.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_u16_e32 v255.h, v1.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.h, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_f16_u16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v255.l, v1.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_f16_u16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_u16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_f16_u16_e32 v5.h, v199.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
 v_cvt_f16_u16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.l, v199.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v128.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.l, v1.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f16_u16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.l, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f32_f16_e32 v5, v199.h
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f32_f16_e32 v5, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f32_f16_e32 v5, v199.l
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f32_f16_e32 v5, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_f32_f16_e32 v5, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
+
+v_cvt_f32_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_i16_f16_e32 v128.h, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
+
+v_cvt_i16_f16_e32 v128.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_i16_f16_e32 v255.h, v1.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.h, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.l, v1.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_i32_i16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.h, v199.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v128.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v255.l, v1.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.l, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.l, v199.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i32_i16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_i32_i16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
+
+v_cvt_i32_i16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
 v_cvt_norm_i16_f16_e32 v128.h, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cvt_norm_i16_f16_e32 v128.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_i16_f16_e32 v255.h, v1.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.h, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_norm_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.l, v1.l
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v128.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v255.l, v1.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.h, v199.h
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.l, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.l, v199.l
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
 v_cvt_norm_u16_f16_e32 v128.h, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
+
+v_cvt_norm_u16_f16_e32 v128.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_u16_f16_e32 v255.h, v1.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.h, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
 v_cvt_norm_u16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_norm_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.l, v1.l
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_norm_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v128.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.l, v1.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.h, v199.h
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.l, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.l, v199.l
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_norm_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
 v_cvt_u16_f16_e32 v128.h, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v255.h, v1.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_u16_f16_e32 v5.h, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_u16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v128.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_cvt_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.h, v1.h
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
 v_cvt_u16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_cvt_u32_u16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_exp_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_exp_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_exp_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_floor_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_floor_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_floor_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_fract_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_fract_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_fract_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
-
-v_frexp_exp_i16_f16_e32 v128.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_frexp_exp_i16_f16_e32 v255.l, v1.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_frexp_exp_i16_f16_e32 v5.l, v199.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_frexp_exp_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_frexp_exp_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_frexp_exp_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_frexp_exp_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v128.h, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.l, v1.l
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.h, v1.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.h, v199.h
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.h, v199.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_frexp_exp_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_cvt_u16_f16_e32 v5.l, v199.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_u16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_u16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_u32_u16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_log_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_u32_u16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_log_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_cvt_u32_u16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:23: error: invalid operand for instruction
 
-v_log_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_exp_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_not_b16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_not_b16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_not_b16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_exp_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_rcp_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_exp_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rcp_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_exp_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_rcp_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_exp_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_rndne_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rndne_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rndne_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_rsq_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_rsq_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rsq_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sat_pk_u8_i16_e32 v199, v5
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_floor_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sin_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sin_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sin_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sqrt_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_fract_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_swap_b16_e32 v128.l, v0.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_fract_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_swap_b16_e32 v0.l, v255.l
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v128.h, 0xfe0b
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_swap_b16_e32 v0.l, s0
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v128.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_swap_b16_e32 v0.l, 0
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v255.h, v1.h
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_swap_b16_e32 v0.l, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_trunc_f16_e32 v128, 0xfe0b
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_frexp_exp_i16_f16_e32 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_trunc_f16_e32 v255, v1
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_frexp_exp_i16_f16_e32 v255.l, v1.l
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_trunc_f16_e32 v5, v199
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+v_frexp_exp_i16_f16_e32 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_ceil_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_ceil_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.h, v199.h
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cos_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cos_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v128.l, 0xaf123456 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.l, v199.l
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255.l, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255.l, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_exp_i16_f16_e32 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v128.h, 0xaf123456 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255.h, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255.h, v255 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199.l quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199.h quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_i32_i16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_cvt_u32_u16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_frexp_mant_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:31: error: invalid operand for instruction
 
-v_exp_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_exp_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_floor_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_floor_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_fract_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fract_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_log_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_log_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_log_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
 v_not_b16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:15: error: invalid operand for instruction
 
-v_not_b16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-
-v_rcp_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_rcp_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_rndne_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_not_b16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:19: error: invalid operand for instruction
 
-v_rndne_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rsq_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rsq_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_sat_pk_u8_i16_e32 v199, v5 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_sin_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sin_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rcp_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_trunc_f16_e32 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_trunc_f16_e32 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_ceil_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_ceil_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cos_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cos_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rndne_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v128.l, 0xaf123456 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255.l, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255.l, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v128.h, 0xaf123456 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f16_f32_e32 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_f16_f32_e32 v255.h, v255.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_rsq_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_cvt_f32_f16_e32 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sat_pk_u8_i16_e32 v199, v5
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_cvt_i32_i16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sat_pk_u8_i16_e32 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_cvt_u32_u16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sat_pk_u8_i16_e32 v199, v5 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_exp_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_exp_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_floor_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_floor_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_fract_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_fract_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sin_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:24: error: invalid operand for instruction
 
-v_frexp_mant_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_log_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_log_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_not_b16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_not_b16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_rcp_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_rcp_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_sqrt_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:25: error: invalid operand for instruction
 
-v_rndne_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v0.l, 0
+// GFX12: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_rndne_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v0.l, 0xfe0b
+// GFX12: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_rsq_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v0.l, s0
+// GFX12: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_rsq_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v0.l, v255.l
+// GFX12: :[[@LINE-1]]:22: error: invalid operand for instruction
 
-v_sat_pk_u8_i16_e32 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_swap_b16_e32 v128.l, v0.l
+// GFX12: :[[@LINE-1]]:16: error: invalid operand for instruction
 
-v_sin_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v128, 0xfe0b
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sin_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v255, v1
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
-v_sqrt_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_sqrt_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
 
-v_trunc_f16_e32 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+v_trunc_f16_e32 v5, v199
+// GFX12: :[[@LINE-1]]:1: error: operands are not valid for this GPU or mode
 
 v_trunc_f16_e32 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
+
+v_trunc_f16_e32 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: :[[@LINE-1]]:26: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s
index c9aa0c67bacdf8c08ddedda31abca6667ea78f6b..f220ec2b7d1e588df28a8f04b27e8d9bf327412c 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s
@@ -1,1969 +1,1970 @@
+; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s
 
 v_ceil_f16 v128, 0xfe0b
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v128, 0xfe0b             ; encoding: [0x80,0x00,0xdc,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_ceil_f16 v255, -1
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, -1                 ; encoding: [0xff,0x00,0xdc,0xd5,0xc1,0x00,0x00,0x00]
 
 v_ceil_f16 v255, 0.5
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, 0.5                ; encoding: [0xff,0x00,0xdc,0xd5,0xf0,0x00,0x00,0x00]
 
 v_ceil_f16 v255, exec_hi
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, exec_hi            ; encoding: [0xff,0x00,0xdc,0xd5,0x7f,0x00,0x00,0x00]
 
 v_ceil_f16 v255, exec_lo
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, exec_lo            ; encoding: [0xff,0x00,0xdc,0xd5,0x7e,0x00,0x00,0x00]
 
 v_ceil_f16 v255, m0
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, m0                 ; encoding: [0xff,0x00,0xdc,0xd5,0x7d,0x00,0x00,0x00]
 
 v_ceil_f16 v255, null
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, null               ; encoding: [0xff,0x00,0xdc,0xd5,0x7c,0x00,0x00,0x00]
 
 v_ceil_f16 v255, s1
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, s1                 ; encoding: [0xff,0x00,0xdc,0xd5,0x01,0x00,0x00,0x00]
 
 v_ceil_f16 v255, s105
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, s105               ; encoding: [0xff,0x00,0xdc,0xd5,0x69,0x00,0x00,0x00]
 
 v_ceil_f16 v255, src_scc
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, src_scc            ; encoding: [0xff,0x00,0xdc,0xd5,0xfd,0x00,0x00,0x00]
 
 v_ceil_f16 v255, ttmp15
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, ttmp15             ; encoding: [0xff,0x00,0xdc,0xd5,0x7b,0x00,0x00,0x00]
 
 v_ceil_f16 v255, v1
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, v1                 ; encoding: [0xff,0x00,0xdc,0xd5,0x01,0x01,0x00,0x00]
+
+v_ceil_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ceil_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdc,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_ceil_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_ceil_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdc,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_ceil_f16 v255, v127
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, v127               ; encoding: [0xff,0x00,0xdc,0xd5,0x7f,0x01,0x00,0x00]
+
+v_ceil_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ceil_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdc,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_ceil_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_ceil_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdc,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_ceil_f16 v255, vcc_hi
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, vcc_hi             ; encoding: [0xff,0x00,0xdc,0xd5,0x6b,0x00,0x00,0x00]
 
 v_ceil_f16 v255, vcc_lo
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v255, vcc_lo             ; encoding: [0xff,0x00,0xdc,0xd5,0x6a,0x00,0x00,0x00]
 
 v_ceil_f16 v5, v199
-// GFX12: v_ceil_f16_e64
+// GFX12: v_ceil_f16_e64 v5, v199                 ; encoding: [0x05,0x00,0xdc,0xd5,0xc7,0x01,0x00,0x00]
+
+v_ceil_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_ceil_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdc,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_ceil_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_ceil_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdc,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cos_f16 v128, 0xfe0b
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xe1,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cos_f16 v255, -1
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xe1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cos_f16 v255, 0.5
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xe1,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cos_f16 v255, exec_hi
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xe1,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cos_f16 v255, exec_lo
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xe1,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cos_f16 v255, m0
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xe1,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cos_f16 v255, null
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, null                ; encoding: [0xff,0x00,0xe1,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cos_f16 v255, s1
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xe1,0xd5,0x01,0x00,0x00,0x00]
 
 v_cos_f16 v255, s105
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xe1,0xd5,0x69,0x00,0x00,0x00]
 
 v_cos_f16 v255, src_scc
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xe1,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cos_f16 v255, ttmp15
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xe1,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cos_f16 v255, v1
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xe1,0xd5,0x01,0x01,0x00,0x00]
+
+v_cos_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cos_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe1,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cos_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_cos_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe1,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cos_f16 v255, v127
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xe1,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cos_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cos_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe1,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cos_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_cos_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe1,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cos_f16 v255, vcc_hi
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xe1,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cos_f16 v255, vcc_lo
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xe1,0xd5,0x6a,0x00,0x00,0x00]
 
 v_cos_f16 v5, v199
-// GFX12: v_cos_f16_e64
+// GFX12: v_cos_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xe1,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_f16_f32 v255.l, v1
-// GFX12: v_cvt_f16_f32_e64
+v_cos_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cos_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe1,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_f16_f32 v255.l, v255
-// GFX12: v_cvt_f16_f32_e64
+v_cos_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_cos_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe1,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_f16_f32 v128.h, 0xaf123456
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v128.h, 0xaf123456 op_sel:[0,1] ; encoding: [0x80,0x40,0x8a,0xd5,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf]
 
 v_cvt_f16_f32 v255.h, -1
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, 0.5
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, exec_hi
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, exec_lo
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, m0
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, null
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, s1
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, s105
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, src_scc
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, ttmp15
-// GFX12: v_cvt_f16_f32_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_f16_f32 v255.h, v1
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v255
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, vcc_hi
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, vcc_lo
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_i16 v128.l, 0xfe0b
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_f32_e64 v255.h, v1 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, -1
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, 0.5
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, exec_hi
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, exec_lo
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, m0
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, null
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, s1
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, s105
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.l, src_scc
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.h, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.h, v1 op_sel:[0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x40,0x8a,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v255.l, ttmp15
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.h, v1 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.h, v1 op_sel:[0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x40,0x8a,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v255.l, v1.l
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.h, v255
+// GFX12: v_cvt_f16_f32_e64 v255.h, v255 op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0xff,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, v127.l
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.h, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.h, v255 op_sel:[0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x40,0x8a,0xd5,0xe9,0x00,0x00,0x00,0xff,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v255.l, vcc_hi
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.h, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.h, v255 op_sel:[0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x40,0x8a,0xd5,0xfa,0x00,0x00,0x00,0xff,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v255.l, vcc_lo
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.h, vcc_hi
+// GFX12: v_cvt_f16_f32_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v5.l, v199.l
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.h, vcc_lo
+// GFX12: v_cvt_f16_f32_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0x8a,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v1
+// GFX12: v_cvt_f16_f32_e64 v255.l, v1            ; encoding: [0xff,0x00,0x8a,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.l, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x8a,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v1 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.l, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x8a,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v255
+// GFX12: v_cvt_f16_f32_e64 v255.l, v255          ; encoding: [0xff,0x00,0x8a,0xd5,0xff,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.l, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0x8a,0xd5,0xe9,0x00,0x00,0x00,0xff,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_f32 v255.l, v255 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_f32_e64_dpp v255.l, v255 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0x8a,0xd5,0xfa,0x00,0x00,0x00,0xff,0x1b,0x00,0xff]
 
 v_cvt_f16_i16 v128.h, 0xfe0b
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd1,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_f16_i16 v128.l, 0xfe0b
+// GFX12: v_cvt_f16_i16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd1,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, -1
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, 0.5
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, exec_hi
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, exec_lo
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, m0
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, null
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, s1
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, s105
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, src_scc
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, ttmp15
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_f16_i16 v255.h, v1.h
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd1,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, v127.h
-// GFX12: v_cvt_f16_i16_e64
-
-v_cvt_f16_i16 v255.h, vcc_hi
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_i16 v255.h, vcc_lo
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v5.h, v199.h
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v127.h
+// GFX12: v_cvt_f16_i16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd1,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
 v_cvt_f16_i16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+// GFX12: v_cvt_f16_i16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_f16_i16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, vcc_hi
+// GFX12: v_cvt_f16_i16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.h, vcc_lo
+// GFX12: v_cvt_f16_i16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd1,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.l, -1
+// GFX12: v_cvt_f16_i16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_f16_i16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_i16_e64
+v_cvt_f16_i16 v255.l, 0.5
+// GFX12: v_cvt_f16_i16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v128.l, 0xfe0b
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, exec_hi
+// GFX12: v_cvt_f16_i16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd1,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, -1
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, exec_lo
+// GFX12: v_cvt_f16_i16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd1,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, 0.5
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, m0
+// GFX12: v_cvt_f16_i16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd1,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, exec_hi
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, null
+// GFX12: v_cvt_f16_i16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd1,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, exec_lo
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, s1
+// GFX12: v_cvt_f16_i16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd1,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, m0
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, s105
+// GFX12: v_cvt_f16_i16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd1,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, null
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, src_scc
+// GFX12: v_cvt_f16_i16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd1,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, s1
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, ttmp15
+// GFX12: v_cvt_f16_i16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd1,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, s105
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v1.l
+// GFX12: v_cvt_f16_i16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd1,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, src_scc
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v255.l, ttmp15
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_f16_u16 v255.l, v1.l
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v127.l
+// GFX12: v_cvt_f16_i16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd1,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v127.l
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd1,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v255.l, vcc_hi
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd1,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_f16_u16 v255.l, vcc_lo
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, vcc_hi
+// GFX12: v_cvt_f16_i16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd1,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v5.l, v199.l
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v255.l, vcc_lo
+// GFX12: v_cvt_f16_i16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd1,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.h, v199.h
+// GFX12: v_cvt_f16_i16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd1,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd1,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd1,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_f16_u16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.l, v199.l
+// GFX12: v_cvt_f16_i16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd1,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_f16_u16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd1,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_f16_u16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_i16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_i16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd1,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_f16_u16 v128.h, 0xfe0b
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd0,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_f16_u16 v128.l, 0xfe0b
+// GFX12: v_cvt_f16_u16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd0,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, -1
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, 0.5
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, exec_hi
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, exec_lo
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, m0
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, null
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, s1
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, s105
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, src_scc
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, ttmp15
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, v1.h
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd0,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_f16_u16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_f16_u16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cvt_f16_u16 v255.h, v127.h
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd0,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_f16_u16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_f16_u16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cvt_f16_u16 v255.h, vcc_hi
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v255.h, vcc_lo
-// GFX12: v_cvt_f16_u16_e64
+// GFX12: v_cvt_f16_u16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd0,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v5.h, v199.h
-// GFX12: v_cvt_f16_u16_e64
-
-v_cvt_f16_u16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, -1
+// GFX12: v_cvt_f16_u16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, 0.5
+// GFX12: v_cvt_f16_u16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, exec_hi
+// GFX12: v_cvt_f16_u16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd0,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, exec_lo
+// GFX12: v_cvt_f16_u16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd0,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, m0
+// GFX12: v_cvt_f16_u16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd0,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_f16_u16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_u16_e64
+v_cvt_f16_u16 v255.l, null
+// GFX12: v_cvt_f16_u16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd0,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_f32_f16 v5, v199.l
-// GFX12: v_cvt_f32_f16_e64
+v_cvt_f16_u16 v255.l, s1
+// GFX12: v_cvt_f16_u16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd0,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_f32_f16 v5, v199.h
-// GFX12: v_cvt_f32_f16_e64
+v_cvt_f16_u16 v255.l, s105
+// GFX12: v_cvt_f16_u16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd0,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v128.l, 0xfe0b
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, src_scc
+// GFX12: v_cvt_f16_u16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd0,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, -1
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, ttmp15
+// GFX12: v_cvt_f16_u16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd0,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, 0.5
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v1.l
+// GFX12: v_cvt_f16_u16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd0,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, exec_hi
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v255.l, exec_lo
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, m0
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v127.l
+// GFX12: v_cvt_f16_u16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd0,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, null
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd0,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v255.l, s1
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd0,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, s105
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, vcc_hi
+// GFX12: v_cvt_f16_u16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd0,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, src_scc
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v255.l, vcc_lo
+// GFX12: v_cvt_f16_u16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd0,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, ttmp15
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.h, v199.h
+// GFX12: v_cvt_f16_u16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd0,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, v1.l
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd0,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v255.l, v127.l
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd0,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, vcc_hi
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.l, v199.l
+// GFX12: v_cvt_f16_u16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd0,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, vcc_lo
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd0,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v5.l, v199.l
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f16_u16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f16_u16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd0,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.h
+// GFX12: v_cvt_f32_f16_e64 v5, v199.h op_sel:[1,0] ; encoding: [0x05,0x08,0x8b,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f32_f16_e64_dpp v5, v199.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0x8b,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f32_f16_e64_dpp v5, v199.h op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0x8b,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.l
+// GFX12: v_cvt_f32_f16_e64 v5, v199.l            ; encoding: [0x05,0x00,0x8b,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_f32_f16_e64_dpp v5, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x8b,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_f32_f16 v5, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_f32_f16_e64_dpp v5, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x8b,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_i16_f16 v128.h, 0xfe0b
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_i16_f16 v128.l, 0xfe0b
+// GFX12: v_cvt_i16_f16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, -1
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, 0.5
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, exec_hi
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, exec_lo
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, m0
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, null
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, s1
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, s105
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, src_scc
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, ttmp15
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, v1.h
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd3,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cvt_i16_f16 v255.h, v127.h
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd3,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cvt_i16_f16 v255.h, vcc_hi
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cvt_i16_f16 v255.h, vcc_lo
-// GFX12: v_cvt_i16_f16_e64
+// GFX12: v_cvt_i16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v5.h, v199.h
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, -1
+// GFX12: v_cvt_i16_f16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd3,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, 0.5
+// GFX12: v_cvt_i16_f16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd3,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, exec_hi
+// GFX12: v_cvt_i16_f16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd3,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, exec_lo
+// GFX12: v_cvt_i16_f16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd3,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, m0
+// GFX12: v_cvt_i16_f16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd3,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, null
+// GFX12: v_cvt_i16_f16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd3,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_i16_f16_e64
+v_cvt_i16_f16 v255.l, s1
+// GFX12: v_cvt_i16_f16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd3,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_i32_i16 v5, v199
-// GFX12: v_cvt_i32_i16_e64
+v_cvt_i16_f16 v255.l, s105
+// GFX12: v_cvt_i16_f16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd3,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v128.l, 0xfe0b
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, src_scc
+// GFX12: v_cvt_i16_f16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd3,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, -1
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, ttmp15
+// GFX12: v_cvt_i16_f16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd3,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, 0.5
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v1.l
+// GFX12: v_cvt_i16_f16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd3,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, exec_hi
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v255.l, exec_lo
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, m0
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v127.l
+// GFX12: v_cvt_i16_f16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd3,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, null
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v255.l, s1
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, s105
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, vcc_hi
+// GFX12: v_cvt_i16_f16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd3,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, src_scc
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v255.l, vcc_lo
+// GFX12: v_cvt_i16_f16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, ttmp15
-// GFX12: v_cvt_norm_i16_f16_e64
-
-v_cvt_norm_i16_f16 v255.l, v1.l
-// GFX12: v_cvt_norm_i16_f16_e64
-
-v_cvt_norm_i16_f16 v255.l, v127.l
-// GFX12: v_cvt_norm_i16_f16_e64
-
-v_cvt_norm_i16_f16 v255.l, vcc_hi
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.h, v199.h
+// GFX12: v_cvt_i16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, vcc_lo
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v5.l, v199.l
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.l, v199.l
+// GFX12: v_cvt_i16_f16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_i16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i32_i16 v5, v199
+// GFX12: v_cvt_i32_i16_e64 v5, v199              ; encoding: [0x05,0x00,0xea,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i32_i16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_i32_i16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xea,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_i32_i16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_i32_i16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xea,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_norm_i16_f16 v128.h, 0xfe0b
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xe3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_norm_i16_f16 v128.l, 0xfe0b
+// GFX12: v_cvt_norm_i16_f16_e64 v128.l, 0xfe0b   ; encoding: [0x80,0x00,0xe3,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, -1
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, 0.5
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, exec_hi
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, exec_lo
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, m0
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, null
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, s1
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, s105
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, src_scc
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, ttmp15
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_norm_i16_f16 v255.h, v1.h
-// GFX12: v_cvt_norm_i16_f16_e64
-
-v_cvt_norm_i16_f16 v255.h, v127.h
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe3,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, vcc_hi
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_i16_f16 v255.h, vcc_lo
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v5.h, v199.h
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v127.h
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe3,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
 v_cvt_norm_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, vcc_hi
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.h, vcc_lo
+// GFX12: v_cvt_norm_i16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.l, -1
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, -1       ; encoding: [0xff,0x00,0xe3,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_norm_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_i16_f16_e64
+v_cvt_norm_i16_f16 v255.l, 0.5
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, 0.5      ; encoding: [0xff,0x00,0xe3,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v128.l, 0xfe0b
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, exec_hi
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, exec_hi  ; encoding: [0xff,0x00,0xe3,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, -1
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, exec_lo
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, exec_lo  ; encoding: [0xff,0x00,0xe3,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, 0.5
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, m0
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, m0       ; encoding: [0xff,0x00,0xe3,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, exec_hi
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, null
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, null     ; encoding: [0xff,0x00,0xe3,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, exec_lo
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, s1
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, s1       ; encoding: [0xff,0x00,0xe3,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, m0
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, s105
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, s105     ; encoding: [0xff,0x00,0xe3,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, null
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, src_scc
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, src_scc  ; encoding: [0xff,0x00,0xe3,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, s1
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, ttmp15
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, ttmp15   ; encoding: [0xff,0x00,0xe3,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, s105
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v1.l
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, v1.l     ; encoding: [0xff,0x00,0xe3,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, src_scc
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v255.l, ttmp15
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v255.l, v1.l
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v127.l
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, v127.l   ; encoding: [0xff,0x00,0xe3,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v127.l
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe3,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v255.l, vcc_hi
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe3,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v255.l, vcc_lo
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, vcc_hi
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, vcc_hi   ; encoding: [0xff,0x00,0xe3,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v5.l, v199.l
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v255.l, vcc_lo
+// GFX12: v_cvt_norm_i16_f16_e64 v255.l, vcc_lo   ; encoding: [0xff,0x00,0xe3,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.h, v199.h
+// GFX12: v_cvt_norm_i16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xe3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xe3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xe3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.l, v199.l
+// GFX12: v_cvt_norm_i16_f16_e64 v5.l, v199.l     ; encoding: [0x05,0x00,0xe3,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe3,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_i16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe3,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_norm_u16_f16 v128.h, 0xfe0b
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xe4,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_norm_u16_f16 v128.l, 0xfe0b
+// GFX12: v_cvt_norm_u16_f16_e64 v128.l, 0xfe0b   ; encoding: [0x80,0x00,0xe4,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, -1
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, 0.5
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, exec_hi
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, exec_lo
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, m0
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, null
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, s1
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, s105
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, src_scc
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, ttmp15
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_norm_u16_f16 v255.h, v1.h
-// GFX12: v_cvt_norm_u16_f16_e64
-
-v_cvt_norm_u16_f16 v255.h, v127.h
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe4,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, vcc_hi
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_norm_u16_f16 v255.h, vcc_lo
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v5.h, v199.h
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v127.h
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xe4,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
 v_cvt_norm_u16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_norm_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, vcc_hi
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.h, vcc_lo
+// GFX12: v_cvt_norm_u16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xe4,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, -1
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, -1       ; encoding: [0xff,0x00,0xe4,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_norm_u16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_norm_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, 0.5
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, 0.5      ; encoding: [0xff,0x00,0xe4,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v128.l, 0xfe0b
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, exec_hi
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, exec_hi  ; encoding: [0xff,0x00,0xe4,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, -1
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, exec_lo
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, exec_lo  ; encoding: [0xff,0x00,0xe4,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, 0.5
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, m0
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, m0       ; encoding: [0xff,0x00,0xe4,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, exec_hi
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, null
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, null     ; encoding: [0xff,0x00,0xe4,0xd5,0x7c,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, exec_lo
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, s1
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, s1       ; encoding: [0xff,0x00,0xe4,0xd5,0x01,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, m0
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, s105
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, s105     ; encoding: [0xff,0x00,0xe4,0xd5,0x69,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, null
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, src_scc
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, src_scc  ; encoding: [0xff,0x00,0xe4,0xd5,0xfd,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, s1
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, ttmp15
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, ttmp15   ; encoding: [0xff,0x00,0xe4,0xd5,0x7b,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, s105
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v1.l
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, v1.l     ; encoding: [0xff,0x00,0xe4,0xd5,0x01,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, src_scc
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v255.l, ttmp15
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_cvt_u16_f16 v255.l, v1.l
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v127.l
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, v127.l   ; encoding: [0xff,0x00,0xe4,0xd5,0x7f,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v127.l
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe4,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v255.l, vcc_hi
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe4,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_cvt_u16_f16 v255.l, vcc_lo
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, vcc_hi
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, vcc_hi   ; encoding: [0xff,0x00,0xe4,0xd5,0x6b,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v5.l, v199.l
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v255.l, vcc_lo
+// GFX12: v_cvt_norm_u16_f16_e64 v255.l, vcc_lo   ; encoding: [0xff,0x00,0xe4,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.h, v199.h
+// GFX12: v_cvt_norm_u16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xe4,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xe4,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xe4,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_cvt_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.l, v199.l
+// GFX12: v_cvt_norm_u16_f16_e64 v5.l, v199.l     ; encoding: [0x05,0x00,0xe4,0xd5,0xc7,0x01,0x00,0x00]
 
-v_cvt_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe4,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_cvt_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_norm_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_norm_u16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe4,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_u16_f16 v128.h, 0xfe0b
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xd2,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cvt_u16_f16 v128.l, 0xfe0b
+// GFX12: v_cvt_u16_f16_e64 v128.l, 0xfe0b        ; encoding: [0x80,0x00,0xd2,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, -1
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, 0.5
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, exec_hi
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7f,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, exec_lo
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7e,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, m0
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7d,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, null
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7c,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, s1
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x01,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, s105
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x69,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, src_scc
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0xfd,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, ttmp15
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x7b,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, v1.h
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_cvt_u16_f16 v255.h, v127.h
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xd2,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_cvt_u16_f16 v255.h, vcc_hi
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x6b,0x00,0x00,0x00]
 
 v_cvt_u16_f16 v255.h, vcc_lo
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xd2,0xd5,0x6a,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v5.h, v199.h
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, -1
+// GFX12: v_cvt_u16_f16_e64 v255.l, -1            ; encoding: [0xff,0x00,0xd2,0xd5,0xc1,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, 0.5
+// GFX12: v_cvt_u16_f16_e64 v255.l, 0.5           ; encoding: [0xff,0x00,0xd2,0xd5,0xf0,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, exec_hi
+// GFX12: v_cvt_u16_f16_e64 v255.l, exec_hi       ; encoding: [0xff,0x00,0xd2,0xd5,0x7f,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, exec_lo
+// GFX12: v_cvt_u16_f16_e64 v255.l, exec_lo       ; encoding: [0xff,0x00,0xd2,0xd5,0x7e,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, m0
+// GFX12: v_cvt_u16_f16_e64 v255.l, m0            ; encoding: [0xff,0x00,0xd2,0xd5,0x7d,0x00,0x00,0x00]
 
-v_cvt_u16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+v_cvt_u16_f16 v255.l, null
+// GFX12: v_cvt_u16_f16_e64 v255.l, null          ; encoding: [0xff,0x00,0xd2,0xd5,0x7c,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, s1
+// GFX12: v_cvt_u16_f16_e64 v255.l, s1            ; encoding: [0xff,0x00,0xd2,0xd5,0x01,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, s105
+// GFX12: v_cvt_u16_f16_e64 v255.l, s105          ; encoding: [0xff,0x00,0xd2,0xd5,0x69,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, src_scc
+// GFX12: v_cvt_u16_f16_e64 v255.l, src_scc       ; encoding: [0xff,0x00,0xd2,0xd5,0xfd,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, ttmp15
+// GFX12: v_cvt_u16_f16_e64 v255.l, ttmp15        ; encoding: [0xff,0x00,0xd2,0xd5,0x7b,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, v1.l
+// GFX12: v_cvt_u16_f16_e64 v255.l, v1.l          ; encoding: [0xff,0x00,0xd2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+v_cvt_u16_f16 v255.l, v127.l
+// GFX12: v_cvt_u16_f16_e64 v255.l, v127.l        ; encoding: [0xff,0x00,0xd2,0xd5,0x7f,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd2,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd2,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
+
+v_cvt_u16_f16 v255.l, vcc_hi
+// GFX12: v_cvt_u16_f16_e64 v255.l, vcc_hi        ; encoding: [0xff,0x00,0xd2,0xd5,0x6b,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v255.l, vcc_lo
+// GFX12: v_cvt_u16_f16_e64 v255.l, vcc_lo        ; encoding: [0xff,0x00,0xd2,0xd5,0x6a,0x00,0x00,0x00]
+
+v_cvt_u16_f16 v5.h, v199.h
+// GFX12: v_cvt_u16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xd2,0xd5,0xc7,0x01,0x00,0x00]
 
 v_cvt_u16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_u16_f16_e64
+// GFX12: v_cvt_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xd2,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xd2,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
+
+v_cvt_u16_f16 v5.l, v199.l
+// GFX12: v_cvt_u16_f16_e64 v5.l, v199.l          ; encoding: [0x05,0x00,0xd2,0xd5,0xc7,0x01,0x00,0x00]
+
+v_cvt_u16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd2,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_cvt_u16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_cvt_u16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd2,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_cvt_u32_u16 v5, v199
-// GFX12: v_cvt_u32_u16_e64
+// GFX12: v_cvt_u32_u16_e64 v5, v199              ; encoding: [0x05,0x00,0xeb,0xd5,0xc7,0x01,0x00,0x00]
+
+v_cvt_u32_u16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_cvt_u32_u16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xeb,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_cvt_u32_u16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_cvt_u32_u16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xeb,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_exp_f16 v128, 0xfe0b
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd8,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_exp_f16 v255, -1
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd8,0xd5,0xc1,0x00,0x00,0x00]
 
 v_exp_f16 v255, 0.5
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd8,0xd5,0xf0,0x00,0x00,0x00]
 
 v_exp_f16 v255, exec_hi
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd8,0xd5,0x7f,0x00,0x00,0x00]
 
 v_exp_f16 v255, exec_lo
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd8,0xd5,0x7e,0x00,0x00,0x00]
 
 v_exp_f16 v255, m0
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd8,0xd5,0x7d,0x00,0x00,0x00]
 
 v_exp_f16 v255, null
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd8,0xd5,0x7c,0x00,0x00,0x00]
 
 v_exp_f16 v255, s1
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd8,0xd5,0x01,0x00,0x00,0x00]
 
 v_exp_f16 v255, s105
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd8,0xd5,0x69,0x00,0x00,0x00]
 
 v_exp_f16 v255, src_scc
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd8,0xd5,0xfd,0x00,0x00,0x00]
 
 v_exp_f16 v255, ttmp15
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd8,0xd5,0x7b,0x00,0x00,0x00]
 
 v_exp_f16 v255, v1
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd8,0xd5,0x01,0x01,0x00,0x00]
+
+v_exp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_exp_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_exp_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_exp_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_exp_f16 v255, v127
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd8,0xd5,0x7f,0x01,0x00,0x00]
+
+v_exp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_exp_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd8,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_exp_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_exp_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd8,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_exp_f16 v255, vcc_hi
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd8,0xd5,0x6b,0x00,0x00,0x00]
 
 v_exp_f16 v255, vcc_lo
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd8,0xd5,0x6a,0x00,0x00,0x00]
 
 v_exp_f16 v5, v199
-// GFX12: v_exp_f16_e64
+// GFX12: v_exp_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd8,0xd5,0xc7,0x01,0x00,0x00]
+
+v_exp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_exp_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd8,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_exp_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_exp_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd8,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_floor_f16 v128, 0xfe0b
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xdb,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_floor_f16 v255, -1
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xdb,0xd5,0xc1,0x00,0x00,0x00]
 
 v_floor_f16 v255, 0.5
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xdb,0xd5,0xf0,0x00,0x00,0x00]
 
 v_floor_f16 v255, exec_hi
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xdb,0xd5,0x7f,0x00,0x00,0x00]
 
 v_floor_f16 v255, exec_lo
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xdb,0xd5,0x7e,0x00,0x00,0x00]
 
 v_floor_f16 v255, m0
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xdb,0xd5,0x7d,0x00,0x00,0x00]
 
 v_floor_f16 v255, null
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, null              ; encoding: [0xff,0x00,0xdb,0xd5,0x7c,0x00,0x00,0x00]
 
 v_floor_f16 v255, s1
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xdb,0xd5,0x01,0x00,0x00,0x00]
 
 v_floor_f16 v255, s105
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xdb,0xd5,0x69,0x00,0x00,0x00]
 
 v_floor_f16 v255, src_scc
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xdb,0xd5,0xfd,0x00,0x00,0x00]
 
 v_floor_f16 v255, ttmp15
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xdb,0xd5,0x7b,0x00,0x00,0x00]
 
 v_floor_f16 v255, v1
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xdb,0xd5,0x01,0x01,0x00,0x00]
+
+v_floor_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_floor_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_floor_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_floor_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_floor_f16 v255, v127
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xdb,0xd5,0x7f,0x01,0x00,0x00]
+
+v_floor_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_floor_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdb,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_floor_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_floor_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdb,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_floor_f16 v255, vcc_hi
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xdb,0xd5,0x6b,0x00,0x00,0x00]
 
 v_floor_f16 v255, vcc_lo
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xdb,0xd5,0x6a,0x00,0x00,0x00]
 
 v_floor_f16 v5, v199
-// GFX12: v_floor_f16_e64
+// GFX12: v_floor_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xdb,0xd5,0xc7,0x01,0x00,0x00]
+
+v_floor_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_floor_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdb,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_floor_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_floor_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdb,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_fract_f16 v128, 0xfe0b
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xdf,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_fract_f16 v255, -1
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xdf,0xd5,0xc1,0x00,0x00,0x00]
 
 v_fract_f16 v255, 0.5
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xdf,0xd5,0xf0,0x00,0x00,0x00]
 
 v_fract_f16 v255, exec_hi
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xdf,0xd5,0x7f,0x00,0x00,0x00]
 
 v_fract_f16 v255, exec_lo
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xdf,0xd5,0x7e,0x00,0x00,0x00]
 
 v_fract_f16 v255, m0
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xdf,0xd5,0x7d,0x00,0x00,0x00]
 
 v_fract_f16 v255, null
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, null              ; encoding: [0xff,0x00,0xdf,0xd5,0x7c,0x00,0x00,0x00]
 
 v_fract_f16 v255, s1
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xdf,0xd5,0x01,0x00,0x00,0x00]
 
 v_fract_f16 v255, s105
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xdf,0xd5,0x69,0x00,0x00,0x00]
 
 v_fract_f16 v255, src_scc
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xdf,0xd5,0xfd,0x00,0x00,0x00]
 
 v_fract_f16 v255, ttmp15
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xdf,0xd5,0x7b,0x00,0x00,0x00]
 
 v_fract_f16 v255, v1
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xdf,0xd5,0x01,0x01,0x00,0x00]
+
+v_fract_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_fract_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdf,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_fract_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_fract_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdf,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_fract_f16 v255, v127
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xdf,0xd5,0x7f,0x01,0x00,0x00]
+
+v_fract_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_fract_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdf,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_fract_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_fract_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdf,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_fract_f16 v255, vcc_hi
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xdf,0xd5,0x6b,0x00,0x00,0x00]
 
 v_fract_f16 v255, vcc_lo
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xdf,0xd5,0x6a,0x00,0x00,0x00]
 
 v_fract_f16 v5, v199
-// GFX12: v_fract_f16_e64
+// GFX12: v_fract_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xdf,0xd5,0xc7,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v128.l, 0xfe0b
-// GFX12: v_frexp_exp_i16_f16_e64
+v_fract_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_fract_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdf,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.l, -1
-// GFX12: v_frexp_exp_i16_f16_e64
+v_fract_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_fract_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdf,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.l, 0.5
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v128.h, 0xfe0b
+// GFX12: v_frexp_exp_i16_f16_e64 v128.h, 0xfe0b op_sel:[0,1] ; encoding: [0x80,0x40,0xda,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, exec_hi
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v128.l, 0xfe0b
+// GFX12: v_frexp_exp_i16_f16_e64 v128.l, 0xfe0b  ; encoding: [0x80,0x00,0xda,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, exec_lo
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, -1
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, -1 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0xc1,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, m0
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, 0.5
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, 0.5 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0xf0,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, null
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, exec_hi
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, exec_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7f,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, s1
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, exec_lo
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, exec_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7e,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, s105
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, m0
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, m0 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7d,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, src_scc
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, null
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, null op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7c,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, ttmp15
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, s1
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, s1 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x01,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, v1.l
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, s105
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, s105 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x69,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, v127.l
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, src_scc
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, src_scc op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0xfd,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, vcc_hi
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, ttmp15
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, ttmp15 op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x7b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.l, vcc_lo
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v1.h
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, v1.h op_sel:[1,1] ; encoding: [0xff,0x48,0xda,0xd5,0x01,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v5.l, v199.l
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xda,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.h, v1.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xda,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v127.h
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, v127.h op_sel:[1,1] ; encoding: [0xff,0x48,0xda,0xd5,0x7f,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x48,0xda,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.h, v127.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x48,0xda,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, vcc_hi
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, vcc_hi op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x6b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.h, vcc_lo
+// GFX12: v_frexp_exp_i16_f16_e64 v255.h, vcc_lo op_sel:[0,1] ; encoding: [0xff,0x40,0xda,0xd5,0x6a,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v128.h, 0xfe0b
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, -1
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, -1      ; encoding: [0xff,0x00,0xda,0xd5,0xc1,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, -1
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, 0.5
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, 0.5     ; encoding: [0xff,0x00,0xda,0xd5,0xf0,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, 0.5
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, exec_hi
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, exec_hi ; encoding: [0xff,0x00,0xda,0xd5,0x7f,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, exec_hi
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, exec_lo
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, exec_lo ; encoding: [0xff,0x00,0xda,0xd5,0x7e,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, exec_lo
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, m0
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, m0      ; encoding: [0xff,0x00,0xda,0xd5,0x7d,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, m0
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, null
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, null    ; encoding: [0xff,0x00,0xda,0xd5,0x7c,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, null
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, s1
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, s1      ; encoding: [0xff,0x00,0xda,0xd5,0x01,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, s1
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, s105
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, s105    ; encoding: [0xff,0x00,0xda,0xd5,0x69,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, s105
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, src_scc
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, src_scc ; encoding: [0xff,0x00,0xda,0xd5,0xfd,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, src_scc
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, ttmp15
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, ttmp15  ; encoding: [0xff,0x00,0xda,0xd5,0x7b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, ttmp15
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v1.l
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, v1.l    ; encoding: [0xff,0x00,0xda,0xd5,0x01,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, v1.h
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.l, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xda,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v255.h, v127.h
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v1.l quad_perm:[3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.l, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xda,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.h, vcc_hi
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v127.l
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, v127.l  ; encoding: [0xff,0x00,0xda,0xd5,0x7f,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, vcc_lo
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.l, v127.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xda,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v5.h, v199.h
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, v127.l quad_perm:[3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v255.l, v127.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xda,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.h, v1.h quad_perm:[3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, vcc_hi
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, vcc_hi  ; encoding: [0xff,0x00,0xda,0xd5,0x6b,0x00,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, v127.h quad_perm:[3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v255.l, vcc_lo
+// GFX12: v_frexp_exp_i16_f16_e64 v255.l, vcc_lo  ; encoding: [0xff,0x00,0xda,0xd5,0x6a,0x00,0x00,0x00]
+
+v_frexp_exp_i16_f16 v5.h, v199.h
+// GFX12: v_frexp_exp_i16_f16_e64 v5.h, v199.h op_sel:[1,1] ; encoding: [0x05,0x48,0xda,0xd5,0xc7,0x01,0x00,0x00]
+
+v_frexp_exp_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xda,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
 v_frexp_exp_i16_f16 v5.h, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v5.h, v199.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xda,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
-v_frexp_exp_i16_f16 v255.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v5.l, v199.l
+// GFX12: v_frexp_exp_i16_f16_e64 v5.l, v199.l    ; encoding: [0x05,0x00,0xda,0xd5,0xc7,0x01,0x00,0x00]
 
-v_frexp_exp_i16_f16 v255.h, v127.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v5.l, v199.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xda,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
 
-v_frexp_exp_i16_f16 v5.h, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_exp_i16_f16_e64
+v_frexp_exp_i16_f16 v5.l, v199.l quad_perm:[3,2,1,0]
+// GFX12: v_frexp_exp_i16_f16_e64_dpp v5.l, v199.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xda,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_frexp_mant_f16 v128, 0xfe0b
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v128, 0xfe0b       ; encoding: [0x80,0x00,0xd9,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_frexp_mant_f16 v255, -1
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, -1           ; encoding: [0xff,0x00,0xd9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, 0.5
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, 0.5          ; encoding: [0xff,0x00,0xd9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, exec_hi
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, exec_hi      ; encoding: [0xff,0x00,0xd9,0xd5,0x7f,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, exec_lo
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, exec_lo      ; encoding: [0xff,0x00,0xd9,0xd5,0x7e,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, m0
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, m0           ; encoding: [0xff,0x00,0xd9,0xd5,0x7d,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, null
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, null         ; encoding: [0xff,0x00,0xd9,0xd5,0x7c,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, s1
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, s1           ; encoding: [0xff,0x00,0xd9,0xd5,0x01,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, s105
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, s105         ; encoding: [0xff,0x00,0xd9,0xd5,0x69,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, src_scc
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, src_scc      ; encoding: [0xff,0x00,0xd9,0xd5,0xfd,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, ttmp15
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, ttmp15       ; encoding: [0xff,0x00,0xd9,0xd5,0x7b,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, v1
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, v1           ; encoding: [0xff,0x00,0xd9,0xd5,0x01,0x01,0x00,0x00]
+
+v_frexp_mant_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_mant_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_frexp_mant_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_frexp_mant_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_frexp_mant_f16 v255, v127
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, v127         ; encoding: [0xff,0x00,0xd9,0xd5,0x7f,0x01,0x00,0x00]
+
+v_frexp_mant_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_mant_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd9,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_frexp_mant_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_frexp_mant_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd9,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_frexp_mant_f16 v255, vcc_hi
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, vcc_hi       ; encoding: [0xff,0x00,0xd9,0xd5,0x6b,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v255, vcc_lo
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v255, vcc_lo       ; encoding: [0xff,0x00,0xd9,0xd5,0x6a,0x00,0x00,0x00]
 
 v_frexp_mant_f16 v5, v199
-// GFX12: v_frexp_mant_f16_e64
+// GFX12: v_frexp_mant_f16_e64 v5, v199           ; encoding: [0x05,0x00,0xd9,0xd5,0xc7,0x01,0x00,0x00]
+
+v_frexp_mant_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_frexp_mant_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd9,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_frexp_mant_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_frexp_mant_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd9,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_log_f16 v128, 0xfe0b
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd7,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_log_f16 v255, -1
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd7,0xd5,0xc1,0x00,0x00,0x00]
 
 v_log_f16 v255, 0.5
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd7,0xd5,0xf0,0x00,0x00,0x00]
 
 v_log_f16 v255, exec_hi
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd7,0xd5,0x7f,0x00,0x00,0x00]
 
 v_log_f16 v255, exec_lo
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd7,0xd5,0x7e,0x00,0x00,0x00]
 
 v_log_f16 v255, m0
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd7,0xd5,0x7d,0x00,0x00,0x00]
 
 v_log_f16 v255, null
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd7,0xd5,0x7c,0x00,0x00,0x00]
 
 v_log_f16 v255, s1
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd7,0xd5,0x01,0x00,0x00,0x00]
 
 v_log_f16 v255, s105
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd7,0xd5,0x69,0x00,0x00,0x00]
 
 v_log_f16 v255, src_scc
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd7,0xd5,0xfd,0x00,0x00,0x00]
 
 v_log_f16 v255, ttmp15
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd7,0xd5,0x7b,0x00,0x00,0x00]
 
 v_log_f16 v255, v1
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd7,0xd5,0x01,0x01,0x00,0x00]
+
+v_log_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_log_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd7,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_log_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_log_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd7,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_log_f16 v255, v127
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd7,0xd5,0x7f,0x01,0x00,0x00]
+
+v_log_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_log_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd7,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_log_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_log_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd7,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_log_f16 v255, vcc_hi
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd7,0xd5,0x6b,0x00,0x00,0x00]
 
 v_log_f16 v255, vcc_lo
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd7,0xd5,0x6a,0x00,0x00,0x00]
 
 v_log_f16 v5, v199
-// GFX12: v_log_f16_e64
+// GFX12: v_log_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd7,0xd5,0xc7,0x01,0x00,0x00]
+
+v_log_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_log_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd7,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_log_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_log_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd7,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_not_b16 v128, 0xfe0b
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xe9,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_not_b16 v255, -1
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, -1                  ; encoding: [0xff,0x00,0xe9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_not_b16 v255, 0.5
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xe9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_not_b16 v255, exec_hi
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xe9,0xd5,0x7f,0x00,0x00,0x00]
 
 v_not_b16 v255, exec_lo
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xe9,0xd5,0x7e,0x00,0x00,0x00]
 
 v_not_b16 v255, m0
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, m0                  ; encoding: [0xff,0x00,0xe9,0xd5,0x7d,0x00,0x00,0x00]
 
 v_not_b16 v255, null
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, null                ; encoding: [0xff,0x00,0xe9,0xd5,0x7c,0x00,0x00,0x00]
 
 v_not_b16 v255, s1
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, s1                  ; encoding: [0xff,0x00,0xe9,0xd5,0x01,0x00,0x00,0x00]
 
 v_not_b16 v255, s105
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, s105                ; encoding: [0xff,0x00,0xe9,0xd5,0x69,0x00,0x00,0x00]
 
 v_not_b16 v255, src_scc
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xe9,0xd5,0xfd,0x00,0x00,0x00]
 
 v_not_b16 v255, ttmp15
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xe9,0xd5,0x7b,0x00,0x00,0x00]
 
 v_not_b16 v255, v1
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, v1                  ; encoding: [0xff,0x00,0xe9,0xd5,0x01,0x01,0x00,0x00]
+
+v_not_b16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_not_b16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_not_b16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_not_b16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_not_b16 v255, v127
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, v127                ; encoding: [0xff,0x00,0xe9,0xd5,0x7f,0x01,0x00,0x00]
+
+v_not_b16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_not_b16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe9,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_not_b16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_not_b16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe9,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_not_b16 v255, vcc_hi
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xe9,0xd5,0x6b,0x00,0x00,0x00]
 
 v_not_b16 v255, vcc_lo
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xe9,0xd5,0x6a,0x00,0x00,0x00]
 
 v_not_b16 v5, v199
-// GFX12: v_not_b16_e64
+// GFX12: v_not_b16_e64 v5, v199                  ; encoding: [0x05,0x00,0xe9,0xd5,0xc7,0x01,0x00,0x00]
+
+v_not_b16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_not_b16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe9,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_not_b16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_not_b16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe9,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_rcp_f16 v128, 0xfe0b
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd4,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_rcp_f16 v255, -1
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd4,0xd5,0xc1,0x00,0x00,0x00]
 
 v_rcp_f16 v255, 0.5
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd4,0xd5,0xf0,0x00,0x00,0x00]
 
 v_rcp_f16 v255, exec_hi
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd4,0xd5,0x7f,0x00,0x00,0x00]
 
 v_rcp_f16 v255, exec_lo
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd4,0xd5,0x7e,0x00,0x00,0x00]
 
 v_rcp_f16 v255, m0
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd4,0xd5,0x7d,0x00,0x00,0x00]
 
 v_rcp_f16 v255, null
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd4,0xd5,0x7c,0x00,0x00,0x00]
 
 v_rcp_f16 v255, s1
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd4,0xd5,0x01,0x00,0x00,0x00]
 
 v_rcp_f16 v255, s105
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd4,0xd5,0x69,0x00,0x00,0x00]
 
 v_rcp_f16 v255, src_scc
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd4,0xd5,0xfd,0x00,0x00,0x00]
 
 v_rcp_f16 v255, ttmp15
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd4,0xd5,0x7b,0x00,0x00,0x00]
 
 v_rcp_f16 v255, v1
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd4,0xd5,0x01,0x01,0x00,0x00]
+
+v_rcp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rcp_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd4,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_rcp_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_rcp_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd4,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_rcp_f16 v255, v127
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd4,0xd5,0x7f,0x01,0x00,0x00]
+
+v_rcp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rcp_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd4,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_rcp_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_rcp_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd4,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_rcp_f16 v255, vcc_hi
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd4,0xd5,0x6b,0x00,0x00,0x00]
 
 v_rcp_f16 v255, vcc_lo
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd4,0xd5,0x6a,0x00,0x00,0x00]
 
 v_rcp_f16 v5, v199
-// GFX12: v_rcp_f16_e64
+// GFX12: v_rcp_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd4,0xd5,0xc7,0x01,0x00,0x00]
+
+v_rcp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rcp_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd4,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_rcp_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_rcp_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd4,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_rndne_f16 v128, 0xfe0b
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xde,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_rndne_f16 v255, -1
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xde,0xd5,0xc1,0x00,0x00,0x00]
 
 v_rndne_f16 v255, 0.5
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xde,0xd5,0xf0,0x00,0x00,0x00]
 
 v_rndne_f16 v255, exec_hi
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xde,0xd5,0x7f,0x00,0x00,0x00]
 
 v_rndne_f16 v255, exec_lo
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xde,0xd5,0x7e,0x00,0x00,0x00]
 
 v_rndne_f16 v255, m0
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xde,0xd5,0x7d,0x00,0x00,0x00]
 
 v_rndne_f16 v255, null
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, null              ; encoding: [0xff,0x00,0xde,0xd5,0x7c,0x00,0x00,0x00]
 
 v_rndne_f16 v255, s1
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xde,0xd5,0x01,0x00,0x00,0x00]
 
 v_rndne_f16 v255, s105
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xde,0xd5,0x69,0x00,0x00,0x00]
 
 v_rndne_f16 v255, src_scc
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xde,0xd5,0xfd,0x00,0x00,0x00]
 
 v_rndne_f16 v255, ttmp15
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xde,0xd5,0x7b,0x00,0x00,0x00]
 
 v_rndne_f16 v255, v1
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xde,0xd5,0x01,0x01,0x00,0x00]
+
+v_rndne_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rndne_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xde,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_rndne_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_rndne_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xde,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_rndne_f16 v255, v127
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xde,0xd5,0x7f,0x01,0x00,0x00]
+
+v_rndne_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rndne_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xde,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_rndne_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_rndne_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xde,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_rndne_f16 v255, vcc_hi
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xde,0xd5,0x6b,0x00,0x00,0x00]
 
 v_rndne_f16 v255, vcc_lo
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xde,0xd5,0x6a,0x00,0x00,0x00]
 
 v_rndne_f16 v5, v199
-// GFX12: v_rndne_f16_e64
+// GFX12: v_rndne_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xde,0xd5,0xc7,0x01,0x00,0x00]
+
+v_rndne_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rndne_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xde,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_rndne_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_rndne_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xde,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_rsq_f16 v128, 0xfe0b
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xd6,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_rsq_f16 v255, -1
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xd6,0xd5,0xc1,0x00,0x00,0x00]
 
 v_rsq_f16 v255, 0.5
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xd6,0xd5,0xf0,0x00,0x00,0x00]
 
 v_rsq_f16 v255, exec_hi
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xd6,0xd5,0x7f,0x00,0x00,0x00]
 
 v_rsq_f16 v255, exec_lo
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xd6,0xd5,0x7e,0x00,0x00,0x00]
 
 v_rsq_f16 v255, m0
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xd6,0xd5,0x7d,0x00,0x00,0x00]
 
 v_rsq_f16 v255, null
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, null                ; encoding: [0xff,0x00,0xd6,0xd5,0x7c,0x00,0x00,0x00]
 
 v_rsq_f16 v255, s1
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xd6,0xd5,0x01,0x00,0x00,0x00]
 
 v_rsq_f16 v255, s105
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xd6,0xd5,0x69,0x00,0x00,0x00]
 
 v_rsq_f16 v255, src_scc
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xd6,0xd5,0xfd,0x00,0x00,0x00]
 
 v_rsq_f16 v255, ttmp15
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xd6,0xd5,0x7b,0x00,0x00,0x00]
 
 v_rsq_f16 v255, v1
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xd6,0xd5,0x01,0x01,0x00,0x00]
+
+v_rsq_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rsq_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd6,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_rsq_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_rsq_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd6,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_rsq_f16 v255, v127
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xd6,0xd5,0x7f,0x01,0x00,0x00]
+
+v_rsq_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rsq_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd6,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_rsq_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_rsq_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd6,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_rsq_f16 v255, vcc_hi
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xd6,0xd5,0x6b,0x00,0x00,0x00]
 
 v_rsq_f16 v255, vcc_lo
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xd6,0xd5,0x6a,0x00,0x00,0x00]
 
 v_rsq_f16 v5, v199
-// GFX12: v_rsq_f16_e64
+// GFX12: v_rsq_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xd6,0xd5,0xc7,0x01,0x00,0x00]
+
+v_rsq_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_rsq_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd6,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_rsq_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_rsq_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd6,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_sat_pk_u8_i16 v199, v5
-// GFX12: v_sat_pk_u8_i16_e64
+// GFX12: v_sat_pk_u8_i16_e64 v199, v5            ; encoding: [0xc7,0x00,0xe2,0xd5,0x05,0x01,0x00,0x00]
+
+v_sat_pk_u8_i16 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sat_pk_u8_i16_e64_dpp v199, v5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xc7,0x00,0xe2,0xd5,0xe9,0x00,0x00,0x00,0x05,0x77,0x39,0x05]
+
+v_sat_pk_u8_i16 v199, v5 quad_perm:[3,2,1,0]
+// GFX12: v_sat_pk_u8_i16_e64_dpp v199, v5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xc7,0x00,0xe2,0xd5,0xfa,0x00,0x00,0x00,0x05,0x1b,0x00,0xff]
 
 v_sin_f16 v128, 0xfe0b
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v128, 0xfe0b              ; encoding: [0x80,0x00,0xe0,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_sin_f16 v255, -1
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, -1                  ; encoding: [0xff,0x00,0xe0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_sin_f16 v255, 0.5
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, 0.5                 ; encoding: [0xff,0x00,0xe0,0xd5,0xf0,0x00,0x00,0x00]
 
 v_sin_f16 v255, exec_hi
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, exec_hi             ; encoding: [0xff,0x00,0xe0,0xd5,0x7f,0x00,0x00,0x00]
 
 v_sin_f16 v255, exec_lo
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, exec_lo             ; encoding: [0xff,0x00,0xe0,0xd5,0x7e,0x00,0x00,0x00]
 
 v_sin_f16 v255, m0
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, m0                  ; encoding: [0xff,0x00,0xe0,0xd5,0x7d,0x00,0x00,0x00]
 
 v_sin_f16 v255, null
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, null                ; encoding: [0xff,0x00,0xe0,0xd5,0x7c,0x00,0x00,0x00]
 
 v_sin_f16 v255, s1
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, s1                  ; encoding: [0xff,0x00,0xe0,0xd5,0x01,0x00,0x00,0x00]
 
 v_sin_f16 v255, s105
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, s105                ; encoding: [0xff,0x00,0xe0,0xd5,0x69,0x00,0x00,0x00]
 
 v_sin_f16 v255, src_scc
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, src_scc             ; encoding: [0xff,0x00,0xe0,0xd5,0xfd,0x00,0x00,0x00]
 
 v_sin_f16 v255, ttmp15
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, ttmp15              ; encoding: [0xff,0x00,0xe0,0xd5,0x7b,0x00,0x00,0x00]
 
 v_sin_f16 v255, v1
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, v1                  ; encoding: [0xff,0x00,0xe0,0xd5,0x01,0x01,0x00,0x00]
+
+v_sin_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sin_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe0,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sin_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_sin_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe0,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_sin_f16 v255, v127
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, v127                ; encoding: [0xff,0x00,0xe0,0xd5,0x7f,0x01,0x00,0x00]
+
+v_sin_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sin_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xe0,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_sin_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_sin_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xe0,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_sin_f16 v255, vcc_hi
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, vcc_hi              ; encoding: [0xff,0x00,0xe0,0xd5,0x6b,0x00,0x00,0x00]
 
 v_sin_f16 v255, vcc_lo
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v255, vcc_lo              ; encoding: [0xff,0x00,0xe0,0xd5,0x6a,0x00,0x00,0x00]
 
 v_sin_f16 v5, v199
-// GFX12: v_sin_f16_e64
+// GFX12: v_sin_f16_e64 v5, v199                  ; encoding: [0x05,0x00,0xe0,0xd5,0xc7,0x01,0x00,0x00]
+
+v_sin_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sin_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xe0,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_sin_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_sin_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xe0,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_sqrt_f16 v128, 0xfe0b
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v128, 0xfe0b             ; encoding: [0x80,0x00,0xd5,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_sqrt_f16 v255, -1
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, -1                 ; encoding: [0xff,0x00,0xd5,0xd5,0xc1,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, 0.5
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, 0.5                ; encoding: [0xff,0x00,0xd5,0xd5,0xf0,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, exec_hi
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, exec_hi            ; encoding: [0xff,0x00,0xd5,0xd5,0x7f,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, exec_lo
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, exec_lo            ; encoding: [0xff,0x00,0xd5,0xd5,0x7e,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, m0
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, m0                 ; encoding: [0xff,0x00,0xd5,0xd5,0x7d,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, null
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, null               ; encoding: [0xff,0x00,0xd5,0xd5,0x7c,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, s1
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, s1                 ; encoding: [0xff,0x00,0xd5,0xd5,0x01,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, s105
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, s105               ; encoding: [0xff,0x00,0xd5,0xd5,0x69,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, src_scc
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, src_scc            ; encoding: [0xff,0x00,0xd5,0xd5,0xfd,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, ttmp15
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, ttmp15             ; encoding: [0xff,0x00,0xd5,0xd5,0x7b,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, v1
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, v1                 ; encoding: [0xff,0x00,0xd5,0xd5,0x01,0x01,0x00,0x00]
+
+v_sqrt_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sqrt_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd5,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+v_sqrt_f16 v255, v1 quad_perm:[3,2,1,0]
+// GFX12: v_sqrt_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd5,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
 v_sqrt_f16 v255, v127
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, v127               ; encoding: [0xff,0x00,0xd5,0xd5,0x7f,0x01,0x00,0x00]
+
+v_sqrt_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sqrt_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xd5,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
+
+v_sqrt_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_sqrt_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xd5,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
 v_sqrt_f16 v255, vcc_hi
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, vcc_hi             ; encoding: [0xff,0x00,0xd5,0xd5,0x6b,0x00,0x00,0x00]
 
 v_sqrt_f16 v255, vcc_lo
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v255, vcc_lo             ; encoding: [0xff,0x00,0xd5,0xd5,0x6a,0x00,0x00,0x00]
 
 v_sqrt_f16 v5, v199
-// GFX12: v_sqrt_f16_e64
+// GFX12: v_sqrt_f16_e64 v5, v199                 ; encoding: [0x05,0x00,0xd5,0xd5,0xc7,0x01,0x00,0x00]
+
+v_sqrt_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_sqrt_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xd5,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_sqrt_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_sqrt_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xd5,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
 
 v_trunc_f16 v128, 0xfe0b
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v128, 0xfe0b            ; encoding: [0x80,0x00,0xdd,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00]
 
 v_trunc_f16 v255, -1
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, -1                ; encoding: [0xff,0x00,0xdd,0xd5,0xc1,0x00,0x00,0x00]
 
 v_trunc_f16 v255, 0.5
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, 0.5               ; encoding: [0xff,0x00,0xdd,0xd5,0xf0,0x00,0x00,0x00]
 
 v_trunc_f16 v255, exec_hi
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, exec_hi           ; encoding: [0xff,0x00,0xdd,0xd5,0x7f,0x00,0x00,0x00]
 
 v_trunc_f16 v255, exec_lo
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, exec_lo           ; encoding: [0xff,0x00,0xdd,0xd5,0x7e,0x00,0x00,0x00]
 
 v_trunc_f16 v255, m0
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, m0                ; encoding: [0xff,0x00,0xdd,0xd5,0x7d,0x00,0x00,0x00]
 
 v_trunc_f16 v255, null
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, null              ; encoding: [0xff,0x00,0xdd,0xd5,0x7c,0x00,0x00,0x00]
 
 v_trunc_f16 v255, s1
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, s1                ; encoding: [0xff,0x00,0xdd,0xd5,0x01,0x00,0x00,0x00]
 
 v_trunc_f16 v255, s105
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, s105              ; encoding: [0xff,0x00,0xdd,0xd5,0x69,0x00,0x00,0x00]
 
 v_trunc_f16 v255, src_scc
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, src_scc           ; encoding: [0xff,0x00,0xdd,0xd5,0xfd,0x00,0x00,0x00]
 
 v_trunc_f16 v255, ttmp15
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64 v255, ttmp15            ; encoding: [0xff,0x00,0xdd,0xd5,0x7b,0x00,0x00,0x00]
 
 v_trunc_f16 v255, v1
-// GFX12: v_trunc_f16_e64
-
-v_trunc_f16 v255, v127
-// GFX12: v_trunc_f16_e64
-
-v_trunc_f16 v255, vcc_hi
-// GFX12: v_trunc_f16_e64
-
-v_trunc_f16 v255, vcc_lo
-// GFX12: v_trunc_f16_e64
-
-v_trunc_f16 v5, v199
-// GFX12: v_trunc_f16_e64
-
-v_ceil_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_ceil_f16_e64
-
-v_ceil_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_ceil_f16_e64
-
-v_ceil_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_ceil_f16_e64
-
-v_cos_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_cos_f16_e64
+// GFX12: v_trunc_f16_e64 v255, v1                ; encoding: [0xff,0x00,0xdd,0xd5,0x01,0x01,0x00,0x00]
 
-v_cos_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_cos_f16_e64
-
-v_cos_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_cos_f16_e64
-
-v_cvt_f16_f32 v255.l, v1 quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v1 quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v255 quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f32_f16 v5, v199.l quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f32_f16_e64
-
-v_cvt_f32_f16 v5, v199.h quad_perm:[3,2,1,0]
-// GFX12: v_cvt_f32_f16_e64
-
-v_cvt_i32_i16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_cvt_i32_i16_e64
-
-v_cvt_u32_u16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_cvt_u32_u16_e64
-
-v_exp_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_exp_f16_e64
-
-v_exp_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_exp_f16_e64
-
-v_exp_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_exp_f16_e64
-
-v_floor_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_floor_f16_e64
-
-v_floor_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_floor_f16_e64
-
-v_floor_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_floor_f16_e64
-
-v_fract_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_fract_f16_e64
-
-v_fract_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_fract_f16_e64
-
-v_fract_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_fract_f16_e64
-
-v_frexp_mant_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_frexp_mant_f16_e64
-
-v_log_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_log_f16_e64
-
-v_log_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_log_f16_e64
-
-v_log_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_log_f16_e64
-
-v_not_b16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_not_b16_e64
-
-v_not_b16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_not_b16_e64
-
-v_not_b16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_not_b16_e64
-
-v_rcp_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_rcp_f16_e64
-
-v_rcp_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_rcp_f16_e64
-
-v_rcp_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_rcp_f16_e64
-
-v_rndne_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_rndne_f16_e64
-
-v_rndne_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_rndne_f16_e64
-
-v_rndne_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_rndne_f16_e64
-
-v_rsq_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_rsq_f16_e64
-
-v_rsq_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_rsq_f16_e64
-
-v_rsq_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_rsq_f16_e64
-
-v_sat_pk_u8_i16 v199, v5 quad_perm:[3,2,1,0]
-// GFX12: v_sat_pk_u8_i16_e64
-
-v_sin_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_sin_f16_e64
-
-v_sin_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_sin_f16_e64
-
-v_sin_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_sin_f16_e64
-
-v_sqrt_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_sqrt_f16_e64
-
-v_sqrt_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_sqrt_f16_e64
-
-v_sqrt_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_sqrt_f16_e64
+v_trunc_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_trunc_f16_e64_dpp v255, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdd,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 
 v_trunc_f16 v255, v1 quad_perm:[3,2,1,0]
-// GFX12: v_trunc_f16_e64
-
-v_trunc_f16 v255, v127 quad_perm:[3,2,1,0]
-// GFX12: v_trunc_f16_e64
-
-v_trunc_f16 v5, v199 quad_perm:[3,2,1,0]
-// GFX12: v_trunc_f16_e64
-
-v_ceil_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_ceil_f16_e64
-
-v_ceil_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_ceil_f16_e64
-
-v_ceil_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_ceil_f16_e64
-
-v_cos_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cos_f16_e64
-
-v_cos_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cos_f16_e64
-
-v_cos_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cos_f16_e64
-
-v_cvt_f16_f32 v255.l, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.l, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f16_f32 v255.h, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f16_f32_e64
-
-v_cvt_f32_f16 v5, v199.l dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f32_f16_e64
-
-v_cvt_f32_f16 v5, v199.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_f32_f16_e64
-
-v_cvt_i32_i16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_i32_i16_e64
-
-v_cvt_u32_u16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_u32_u16_e64
-
-v_exp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_exp_f16_e64
-
-v_exp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_exp_f16_e64
-
-v_exp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_exp_f16_e64
-
-v_floor_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_floor_f16_e64
-
-v_floor_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_floor_f16_e64
-
-v_floor_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_floor_f16_e64
-
-v_fract_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_fract_f16_e64
-
-v_fract_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_fract_f16_e64
-
-v_fract_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_fract_f16_e64
-
-v_frexp_mant_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_mant_f16_e64
-
-v_frexp_mant_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_frexp_mant_f16_e64
-
-v_log_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_log_f16_e64
+// GFX12: v_trunc_f16_e64_dpp v255, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdd,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
 
-v_log_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_log_f16_e64
-
-v_log_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_log_f16_e64
-
-v_not_b16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_not_b16_e64
-
-v_not_b16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_not_b16_e64
-
-v_not_b16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_not_b16_e64
-
-v_rcp_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rcp_f16_e64
-
-v_rcp_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rcp_f16_e64
-
-v_rcp_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rcp_f16_e64
-
-v_rndne_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rndne_f16_e64
-
-v_rndne_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rndne_f16_e64
-
-v_rndne_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rndne_f16_e64
-
-v_rsq_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rsq_f16_e64
-
-v_rsq_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rsq_f16_e64
-
-v_rsq_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_rsq_f16_e64
-
-v_sat_pk_u8_i16 v199, v5 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sat_pk_u8_i16_e64
-
-v_sin_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sin_f16_e64
-
-v_sin_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sin_f16_e64
-
-v_sin_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sin_f16_e64
+v_trunc_f16 v255, v127
+// GFX12: v_trunc_f16_e64 v255, v127              ; encoding: [0xff,0x00,0xdd,0xd5,0x7f,0x01,0x00,0x00]
 
-v_sqrt_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sqrt_f16_e64
+v_trunc_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_trunc_f16_e64_dpp v255, v127 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xff,0x00,0xdd,0xd5,0xe9,0x00,0x00,0x00,0x7f,0x77,0x39,0x05]
 
-v_sqrt_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sqrt_f16_e64
+v_trunc_f16 v255, v127 quad_perm:[3,2,1,0]
+// GFX12: v_trunc_f16_e64_dpp v255, v127 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x00,0xdd,0xd5,0xfa,0x00,0x00,0x00,0x7f,0x1b,0x00,0xff]
 
-v_sqrt_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_sqrt_f16_e64
+v_trunc_f16 v255, vcc_hi
+// GFX12: v_trunc_f16_e64 v255, vcc_hi            ; encoding: [0xff,0x00,0xdd,0xd5,0x6b,0x00,0x00,0x00]
 
-v_trunc_f16 v255, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_trunc_f16_e64
+v_trunc_f16 v255, vcc_lo
+// GFX12: v_trunc_f16_e64 v255, vcc_lo            ; encoding: [0xff,0x00,0xdd,0xd5,0x6a,0x00,0x00,0x00]
 
-v_trunc_f16 v255, v127 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_trunc_f16_e64
+v_trunc_f16 v5, v199
+// GFX12: v_trunc_f16_e64 v5, v199                ; encoding: [0x05,0x00,0xdd,0xd5,0xc7,0x01,0x00,0x00]
 
 v_trunc_f16 v5, v199 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_trunc_f16_e64
+// GFX12: v_trunc_f16_e64_dpp v5, v199 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xdd,0xd5,0xe9,0x00,0x00,0x00,0xc7,0x77,0x39,0x05]
+
+v_trunc_f16 v5, v199 quad_perm:[3,2,1,0]
+// GFX12: v_trunc_f16_e64_dpp v5, v199 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xdd,0xd5,0xfa,0x00,0x00,0x00,0xc7,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-mpam.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-mpam.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b9ff0a4d209b2f3165d60db1899b4aebf6d05787
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AArch64/armv9.6a-mpam.txt
@@ -0,0 +1,50 @@
+# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
+
+#------------------------------------------------------------------------------
+# Armv9.6-A FEAT_MPAM Extensions
+#------------------------------------------------------------------------------
+
+[0x80,0xa5,0x1e,0xd5]
+# CHECK: msr MPAMBW3_EL3, x0
+
+[0x80,0xa5,0x1c,0xd5]
+# CHECK: msr MPAMBW2_EL2, x0
+
+[0x80,0xa5,0x18,0xd5]
+# CHECK: msr MPAMBW1_EL1, x0
+
+[0x80,0xa5,0x1d,0xd5]
+# CHECK: msr MPAMBW1_EL12, x0
+
+[0xa0,0xa5,0x18,0xd5]
+# CHECK: msr MPAMBW0_EL1, x0
+
+[0xc0,0xa5,0x1c,0xd5]
+# CHECK: msr MPAMBWCAP_EL2, x0
+
+[0xe0,0xa5,0x18,0xd5]
+# CHECK: msr MPAMBWSM_EL1, x0
+
+[0xa0,0xa4,0x38,0xd5]
+# CHECK: mrs x0, MPAMBWIDR_EL1
+
+[0x80,0xa5,0x3e,0xd5]
+# CHECK: mrs x0, MPAMBW3_EL3
+
+[0x80,0xa5,0x3c,0xd5]
+# CHECK: mrs x0, MPAMBW2_EL2
+
+[0x80,0xa5,0x38,0xd5]
+# CHECK: mrs x0, MPAMBW1_EL1
+
+[0x80,0xa5,0x3d,0xd5]
+# CHECK: mrs x0, MPAMBW1_EL12
+
+[0xa0,0xa5,0x38,0xd5]
+# CHECK: mrs x0, MPAMBW0_EL1
+
+[0xc0,0xa5,0x3c,0xd5]
+# CHECK: mrs x0, MPAMBWCAP_EL2
+
+[0xe0,0xa5,0x38,0xd5]
+# CHECK: mrs x0, MPAMBWSM_EL1
diff --git a/llvm/test/MC/LoongArch/Directives/option-invalid.s b/llvm/test/MC/LoongArch/Directives/option-invalid.s
new file mode 100644
index 0000000000000000000000000000000000000000..d9064f03015452eeee0806774282a7e96f6a26dd
--- /dev/null
+++ b/llvm/test/MC/LoongArch/Directives/option-invalid.s
@@ -0,0 +1,23 @@
+# RUN: not llvm-mc --triple=loongarch64 %s 2>&1 \
+# RUN:   | FileCheck --implicit-check-not=error: %s
+
+# CHECK: :[[#@LINE+1]]:8: error: expected identifier
+.option
+
+# CHECK: :[[#@LINE+1]]:9: error: expected identifier
+.option 123
+
+# CHECK: :[[#@LINE+1]]:9: error: expected identifier
+.option "str"
+
+# CHECK: :[[#@LINE+1]]:12: warning: unknown option, expected 'push', 'pop', 'relax' or 'norelax'
+.option bar
+
+# CHECK: :[[#@LINE+1]]:12: error: .option pop with no .option push
+.option pop
+
+# CHECK: :[[#@LINE+1]]:14: error: expected newline
+.option push 123
+
+# CHECK: :[[#@LINE+1]]:13: error: expected newline
+.option pop 123
diff --git a/llvm/test/MC/LoongArch/Directives/option-pushpop.s b/llvm/test/MC/LoongArch/Directives/option-pushpop.s
new file mode 100644
index 0000000000000000000000000000000000000000..a33b7b8d1a09b3f250c8e3e9f31b3b412ac48c91
--- /dev/null
+++ b/llvm/test/MC/LoongArch/Directives/option-pushpop.s
@@ -0,0 +1,59 @@
+# RUN: llvm-mc --triple=loongarch64 --mattr=-relax %s \
+# RUN:     | FileCheck --check-prefix=CHECK-ASM %s
+# RUN: llvm-mc --triple=loongarch64 --mattr=-relax --filetype=obj %s \
+# RUN:     | llvm-readobj -r - | FileCheck --check-prefix=CHECK-RELOC %s
+
+## Test the operation of the push and pop assembler directives when
+## using .option relax. Checks that using .option pop correctly restores 
+## all target features to their state at the point where .option pop was
+## last used.
+
+# CHECK-ASM: .option push
+.option push # relax = false
+
+# CHECK-ASM: .option relax
+.option relax # relax = true
+
+# CHECK-ASM: pcalau12i $a0, %pc_hi20(sym1)
+# CHECK-ASM-NEXT: addi.d $a0, $a0, %pc_lo12(sym1)
+# CHECK-RELOC:      R_LARCH_PCALA_HI20 sym1 0x0
+# CHECK-RELOC-NEXT: R_LARCH_RELAX - 0x0
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 sym1 0x0
+# CHECK-RELOC-NEXT: R_LARCH_RELAX - 0x0
+la.pcrel $a0, sym1
+
+# CHECK-ASM: .option push
+.option push # relax = true
+
+# CHECK-ASM: .option norelax
+.option norelax # relax = false
+
+# CHECK-ASM: pcalau12i $a0, %pc_hi20(sym2)
+# CHECK-ASM-NEXT: addi.d $a0, $a0, %pc_lo12(sym2)
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_HI20 sym2 0x0
+# CHECK-RELOC-NOT:  R_LARCH_RELAX - 0x0
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 sym2 0x0
+# CHECK-RELOC-NOT:  R_LARCH_RELAX - 0x0
+la.pcrel $a0, sym2
+
+# CHECK-ASM: .option pop
+.option pop # relax = true
+
+# CHECK-ASM: pcalau12i $a0, %pc_hi20(sym3)
+# CHECK-ASM-NEXT: addi.d $a0, $a0, %pc_lo12(sym3)
+# CHECK-RELOC:      R_LARCH_PCALA_HI20 sym3 0x0
+# CHECK-RELOC-NEXT: R_LARCH_RELAX - 0x0
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 sym3 0x0
+# CHECK-RELOC-NEXT: R_LARCH_RELAX - 0x0
+la.pcrel $a0, sym3
+
+# CHECK-ASM: .option pop
+.option pop # relax = false
+
+la.pcrel $a0, sym4
+# CHECK-ASM: pcalau12i $a0, %pc_hi20(sym4)
+# CHECK-ASM-NEXT: addi.d $a0, $a0, %pc_lo12(sym4)
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_HI20 sym4 0x0
+# CHECK-RELOC-NOT:  R_LARCH_RELAX - 0x0
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 sym4 0x0
+# CHECK-RELOC-NOT:  R_LARCH_RELAX - 0x0
diff --git a/llvm/test/MC/LoongArch/Directives/option-relax.s b/llvm/test/MC/LoongArch/Directives/option-relax.s
new file mode 100644
index 0000000000000000000000000000000000000000..587e1833085622a620f9b5f4fb401d0b2b199de7
--- /dev/null
+++ b/llvm/test/MC/LoongArch/Directives/option-relax.s
@@ -0,0 +1,30 @@
+# RUN: llvm-mc --triple=loongarch64 %s | FileCheck --check-prefix=CHECK-ASM %s
+# RUN: llvm-mc -filetype=obj --triple=loongarch64 %s \
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
+
+## Check .option relax causes R_LARCH_RELAX to be emitted, and .option
+## norelax suppresses it.
+
+# CHECK-ASM: .option relax
+.option relax
+
+# CHECK-ASM: pcalau12i $a0, %pc_hi20(sym1)
+# CHECK-ASM-NEXT: addi.d $a0, $a0, %pc_lo12(sym1)
+
+# CHECK-RELOC:      R_LARCH_PCALA_HI20 sym1 0x0
+# CHECK-RELOC-NEXT: R_LARCH_RELAX - 0x0
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 sym1 0x0
+# CHECK-RELOC-NEXT: R_LARCH_RELAX - 0x0
+la.pcrel $a0, sym1
+
+# CHECK-ASM: .option norelax
+.option norelax
+
+# CHECK-ASM: pcalau12i $a0, %pc_hi20(sym2)
+# CHECK-ASM-NEXT: addi.d $a0, $a0, %pc_lo12(sym2)
+
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_HI20 sym2 0x0
+# CHECK-RELOC-NOT:  R_LARCH_RELAX - 0x0
+# CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 sym2 0x0
+# CHECK-RELOC-NOT:  R_LARCH_RELAX - 0x0
+la.pcrel $a0, sym2
diff --git a/llvm/test/MC/RISCV/insn-invalid.s b/llvm/test/MC/RISCV/insn-invalid.s
index ef2f3c382972abc91ff3b94298f7f6b654768c91..40d57a2b6fddf18044b2670b9b02cb48d895d829 100644
--- a/llvm/test/MC/RISCV/insn-invalid.s
+++ b/llvm/test/MC/RISCV/insn-invalid.s
@@ -24,8 +24,8 @@
 # Make fake mnemonics we use to match these in the tablegened asm match table isn't exposed.
 .insn_i  0x13,  0,  a0, a1, 13, 14 # CHECK: :[[@LINE]]:1: error: unknown directive
 
-.insn . # CHECK: :[[@LINE]]:7: error: expected instruction format or an integer constant
-.insn 0x2, # CHECK: :[[@LINE]]:12: error: expected an integer constant
+.insn . # CHECK: :[[@LINE]]:7: error: expected absolute expression
+.insn 0x2, # CHECK: :[[@LINE]]:12: error: unknown token in expression
 
 .insn 0x4, 0x13, 0 # CHECK: :[[@LINE]]:16: error: invalid operand for instruction
 
@@ -49,7 +49,10 @@
 .insn 0x2, 0x10001 # CHECK: :[[@LINE]]:7: error: encoding value does not fit into instruction
 .insn 0x4, 0x100000003 # CHECK: :[[@LINE]]:7: error: encoding value does not fit into instruction
 .insn 0x6, 0x100000000001f # CHECK: :[[@LINE]]:7: error: encoding value does not fit into instruction
-.insn 0x8, 0x1000000000000003f # CHECK: :[[@LINE]]:12: error: expected an integer constant
+.insn 0x8, 0x1000000000000003f # CHECK: :[[@LINE]]:12: error: literal value out of range for directive
 
 .insn 0x0010 # CHECK: :[[@LINE]]:7: error: compressed instructions are not allowed
 .insn 0x2, 0x0001 # CHECK: :[[@LINE]]:7: error: compressed instructions are not allowed
+
+.insn 0x2 + 0x2, 0x3 | (31 # CHECK: :[[@LINE]]:28: error: expected ')'
+.insn 0x4 * , 0xbf | (31 << 59) # CHECK: :[[@LINE]]:13: error: unknown token in expression
diff --git a/llvm/test/MC/RISCV/insn.s b/llvm/test/MC/RISCV/insn.s
index e32fec25bb16b436f40e1d742904bf5fda1093b4..829364c6328843e5761f98e0d4929ca6fb24be3c 100644
--- a/llvm/test/MC/RISCV/insn.s
+++ b/llvm/test/MC/RISCV/insn.s
@@ -170,17 +170,50 @@ target:
 # CHECK-OBJ: <unknown>
 .insn 6, 0x1f
 
-# CHECK-ASM: .insn 0x4, 65503
-# CHECK-ASM: encoding: [0xdf,0xff,0x00,0x00]
-# CHECK-OBJ: <unknown>
-.insn 0xffdf
-
 # CHECK-ASM: .insn 0x8, 63
 # CHECK-ASM: encoding: [0x3f,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 # CHECK-OBJ: <unknown>
 .insn 8, 0x3f
 
+# CHECK-ASM: .insn 0x6, 281474976710623
+# CHECK-ASM: encoding: [0xdf,0xff,0xff,0xff,0xff,0xff]
+# CHECK-OBJ: <unknown>
+.insn 0x6, 0xffffffffffdf
+
+# CHECK-ASM: .insn 0x8, -65
+# CHECK-ASM: encoding: [0xbf,0xff,0xff,0xff,0xff,0xff,0xff,0xff]
+# CHECK-OBJ: <unknown>
+.insn 0x8, 0xffffffffffffffbf
+
+# CHECK-ASM: .insn 0x4, 3971
+# CHECK-ASM: encoding: [0x83,0x0f,0x00,0x00]
+# CHECK-OBJ: lb t6, 0x0(zero)
+.insn 0x2 + 0x2, 0x3 | (31 << 7)
+
+# CHECK-ASM: .insn 0x8, -576460752303423297
+# CHECK-ASM: encoding: [0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0xf8]
+# CHECK-OBJ: <unknown>
+.insn 0x4 * 0x2, 0xbf | (31 << 59)
+
+odd_lengths:
+# CHECK-ASM-LABEL: odd_lengths:
+# CHECK-OBJ-LABEL: <odd_lengths>:
+
+## These deliberately disagree with the lengths objdump expects them to have, so
+## keep them at the end so that the disassembled instruction stream is not out
+## of sync with the encoded instruction stream. We don't check for `<unknown>`
+## as we could get any number of those, so instead check for the encoding
+## halfwords. These might be split into odd 16-bit chunks, so each chunk is on
+## one line.
+
+# CHECK-ASM: .insn 0x4, 65503
+# CHECK-ASM: encoding: [0xdf,0xff,0x00,0x00]
+# CHECK-OBJ: ffdf
+# CHECK-OBJ: 0000
+.insn 0xffdf
+
 # CHECK-ASM: .insn 0x4, 65471
 # CHECK-ASM: encoding: [0xbf,0xff,0x00,0x00]
-# CHECK-OBJ: <unknown>
+# CHECK-OBJ: ffbf
+# CHECK-OBJ: 0000
 .insn 0xffbf
diff --git a/llvm/test/MC/RISCV/rv32c-valid.s b/llvm/test/MC/RISCV/rv32c-valid.s
index c9e9b0053173be067b1728c77de602f9c2ca92cc..bcdf27a2ba783be65adfd41774d8b88f3ee76e7f 100644
--- a/llvm/test/MC/RISCV/rv32c-valid.s
+++ b/llvm/test/MC/RISCV/rv32c-valid.s
@@ -147,6 +147,11 @@ c.sub a4, a5
 # CHECK-ASM: encoding: [0x01,0x00]
 # CHECK-NO-EXT:  error: instruction requires the following: 'C' (Compressed Instructions) or 'Zca' (part of the C extension, excluding compressed floating point loads/stores){{$}}
 c.nop
+# CHECK-ASM: c.addi zero, 0
+# CHECK-OBJ: c.nop
+# CHECK-ASM: encoding: [0x01,0x00]
+# CHECK-NO-EXT:  error: instruction requires the following: 'C' (Compressed Instructions) or 'Zca' (part of the C extension, excluding compressed floating point loads/stores){{$}}
+c.addi x0, 0
 # CHECK-ASM-AND-OBJ: c.ebreak
 # CHECK-ASM: encoding: [0x02,0x90]
 # CHECK-NO-EXT:  error: instruction requires the following: 'C' (Compressed Instructions) or 'Zca' (part of the C extension, excluding compressed floating point loads/stores){{$}}
diff --git a/llvm/test/MachineVerifier/test_multiple_errors.mir b/llvm/test/MachineVerifier/test_multiple_errors.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e1ce348565c1aa68dd0c6877faf9903495bc2350
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_multiple_errors.mir
@@ -0,0 +1,22 @@
+# RUN: not --crash llc -mtriple=aarch64 -o /dev/null -run-pass=none %s 2>&1 | FileCheck %s --implicit-check-not="Bad machine code"
+# REQUIRES: aarch64-registered-target
+
+# Since we abort after reporting the first error, we should only expect one error to be reported.
+# CHECK: *** Bad machine code: Generic virtual register use cannot be undef ***
+# CHECK: Found 1 machine code errors.
+
+---
+name:            foo
+liveins:
+body:             |
+  bb.0:
+    $x0 = COPY undef %0:_(s64)
+...
+
+---
+name:            bar
+liveins:
+body:             |
+  bb.0:
+    $x0 = COPY undef %0:_(s64)
+...
diff --git a/llvm/test/ThinLTO/X86/memprof-icp.ll b/llvm/test/ThinLTO/X86/memprof-icp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5c6d4e383d3217ce04897dfd1f7285b93094c221
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/memprof-icp.ll
@@ -0,0 +1,322 @@
+;; Test that cloning of an indirect call works. We should perform ICP and update
+;; promoted call to the correct clone.
+
+;; This was created from the following source code, for which both memprof and
+;; instrumentation PGO was collected and then applied, then the IR was reduced
+;; using llvm-reduce with the expected FileCheck input.
+;; TODO: Consider adding a sample PGO based test, however, that uses the same VP
+;; metadata and should behave the same.
+
+;; -- virtfunc.h: --
+;; void external(int *x);
+;;
+;; class B0 {
+;;  public:
+;;   virtual int bar(unsigned s);
+;; };
+;;
+;; class B : public B0 {
+;;  public:
+;;   int bar(unsigned s) override;
+;; };
+;;
+;; int foo(B0 &b, unsigned s);
+
+;; -- virtfunc.cc: --
+;; #include "virtfunc.h"
+;;
+;; int foo(B0 &b, unsigned s) {
+;;   return b.bar(s);
+;; }
+
+;; -- virtfunc_main.cc: --
+;; #include "virtfunc.h"
+;; #include <stdio.h>
+;; #include <unistd.h>
+;;
+;; int main() {
+;;   B b;
+;;   int x = foo(b, 1);
+;;   printf("%d\n", x);
+;;   int y = foo(b, 10);
+;;   printf("%d\n", y);
+;;   B0 b0;
+;;   x = foo(b0, 1);
+;;   printf("%d\n", x);
+;;   y = foo(b0, 10);
+;;   printf("%d\n", y);
+;;   return 0;
+;; }
+;;
+;; int B0::bar(unsigned s) {
+;;   int *x = new int;
+;;   sleep(s);
+;;   external(x);
+;;   delete x;
+;;   return 1;
+;; }
+;;
+;; int B::bar(unsigned s) {
+;;   int *x = new int;
+;;   sleep(s);
+;;   external(x);
+;;   delete x;
+;;   return 2;
+;; }
+
+;; -stats requires asserts
+; REQUIRES: asserts
+
+; RUN: split-file %s %t
+
+; RUN: opt -thinlto-bc %t/main.ll >%t/main.o
+; RUN: opt -thinlto-bc %t/foo.ll >%t/foo.o
+
+;; Check that we get the synthesized callsite records. There should be 2, one
+;; for each profiled target in the VP metadata. They will have the same stackIds
+;; since the debug information for the callsite is the same.
+; RUN: llvm-dis %t/foo.o -o - | FileCheck %s --check-prefix=CALLSITES
+; CALLSITES: gv: (name: "_Z3fooR2B0j", {{.*}} callsites: ((callee: ^{{[0-9]+}}, clones: (0), stackIds: (16345663650247127235)), (callee: ^{{[0-9]+}}, clones: (0), stackIds: (16345663650247127235)))
+
+;; Make sure that we don't get the synthesized callsite records if the
+;; -enable-memprof-indirect-call-support flag is false.
+; RUN: opt -thinlto-bc %t/foo.ll -enable-memprof-indirect-call-support=false -o - \
+; RUN: 	| llvm-dis -o - | FileCheck %s --implicit-check-not callsites
+
+;; First perform in-process ThinLTO
+; RUN: llvm-lto2 run %t/main.o %t/foo.o -enable-memprof-context-disambiguation \
+; RUN:  -supports-hot-cold-new \
+; RUN:  -r=%t/foo.o,_Z3fooR2B0j,plx \
+; RUN:  -r=%t/main.o,_Z3fooR2B0j, \
+; RUN:  -r=%t/main.o,_Znwm, \
+; RUN:  -r=%t/main.o,_ZdlPvm, \
+; RUN:  -r=%t/main.o,_Z8externalPi, \
+; RUN:  -r=%t/main.o,main,plx \
+; RUN:  -r=%t/main.o,_ZN2B03barEj,plx \
+; RUN:  -r=%t/main.o,_ZN1B3barEj,plx \
+; RUN:  -r=%t/main.o,_ZTV1B,plx \
+; RUN:  -r=%t/main.o,_ZTVN10__cxxabiv120__si_class_type_infoE,plx \
+; RUN:  -r=%t/main.o,_ZTS1B,plx \
+; RUN:  -r=%t/main.o,_ZTVN10__cxxabiv117__class_type_infoE,plx \
+; RUN:  -r=%t/main.o,_ZTS2B0,plx \
+; RUN:  -r=%t/main.o,_ZTI2B0,plx \
+; RUN:  -r=%t/main.o,_ZTI1B,plx \
+; RUN:  -r=%t/main.o,_ZTV2B0,plx \
+; RUN:	-thinlto-threads=1 \
+; RUN:  -memprof-verify-ccg -memprof-verify-nodes -stats \
+; RUN:  -pass-remarks=. -save-temps \
+; RUN:  -o %t.out 2>&1 | FileCheck %s --check-prefix=STATS \
+; RUN:  --check-prefix=STATS-BE --check-prefix=REMARKS-MAIN \
+; RUN:  --check-prefix=REMARKS-FOO
+
+; RUN: llvm-dis %t.out.2.4.opt.bc -o - | FileCheck %s --check-prefix=IR
+
+;; Try again but with distributed ThinLTO
+; RUN: llvm-lto2 run %t/main.o %t/foo.o -enable-memprof-context-disambiguation \
+; RUN:  -supports-hot-cold-new \
+; RUN:  -thinlto-distributed-indexes \
+; RUN:  -r=%t/foo.o,_Z3fooR2B0j,plx \
+; RUN:  -r=%t/main.o,_Z3fooR2B0j, \
+; RUN:  -r=%t/main.o,_Znwm, \
+; RUN:  -r=%t/main.o,_ZdlPvm, \
+; RUN:  -r=%t/main.o,_Z8externalPi, \
+; RUN:  -r=%t/main.o,main,plx \
+; RUN:  -r=%t/main.o,_ZN2B03barEj,plx \
+; RUN:  -r=%t/main.o,_ZN1B3barEj,plx \
+; RUN:  -r=%t/main.o,_ZTV1B,plx \
+; RUN:  -r=%t/main.o,_ZTVN10__cxxabiv120__si_class_type_infoE,plx \
+; RUN:  -r=%t/main.o,_ZTS1B,plx \
+; RUN:  -r=%t/main.o,_ZTVN10__cxxabiv117__class_type_infoE,plx \
+; RUN:  -r=%t/main.o,_ZTS2B0,plx \
+; RUN:  -r=%t/main.o,_ZTI2B0,plx \
+; RUN:  -r=%t/main.o,_ZTI1B,plx \
+; RUN:  -r=%t/main.o,_ZTV2B0,plx \
+; RUN:  -memprof-verify-ccg -memprof-verify-nodes -stats \
+; RUN:  -o %t.out 2>&1 | FileCheck %s --check-prefix=STATS
+
+;; Run ThinLTO backend
+; RUN: opt -import-all-index -passes=function-import,memprof-context-disambiguation,inline \
+; RUN:  -summary-file=%t/foo.o.thinlto.bc -memprof-import-summary=%t/foo.o.thinlto.bc \
+; RUN:  -enable-import-metadata -stats -pass-remarks=. \
+; RUN:  %t/foo.o -S 2>&1 | FileCheck %s --check-prefix=IR \
+; RUN:  --check-prefix=STATS-BE-DISTRIB --check-prefix=REMARKS-FOO
+
+; REMARKS-MAIN: call in clone main assigned to call function clone _Z3fooR2B0j.memprof.1
+; REMARKS-MAIN: call in clone main assigned to call function clone _Z3fooR2B0j.memprof.1
+; REMARKS-MAIN: created clone _ZN2B03barEj.memprof.1
+; REMARKS-MAIN: call in clone _ZN2B03barEj marked with memprof allocation attribute notcold
+; REMARKS-MAIN: call in clone _ZN2B03barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS-MAIN: created clone _ZN1B3barEj.memprof.1
+; REMARKS-MAIN: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
+; REMARKS-MAIN: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS-FOO: created clone _Z3fooR2B0j.memprof.1
+;; In each version of foo we should have promoted the indirect call to two conditional
+;; direct calls, one to B::bar and one to B0::bar. The cloned version of foo should call
+;; the cloned versions of bar for both promotions.
+; REMARKS-FOO: Promote indirect call to _ZN1B3barEj with count 2 out of 4
+; REMARKS-FOO: call in clone _Z3fooR2B0j promoted and assigned to call function clone _ZN1B3barEj
+; REMARKS-FOO: Promote indirect call to _ZN1B3barEj with count 2 out of 4
+; REMARKS-FOO: call in clone _Z3fooR2B0j.memprof.1 promoted and assigned to call function clone _ZN1B3barEj.memprof.1
+; REMARKS-FOO: Promote indirect call to _ZN2B03barEj with count 2 out of 2
+; REMARKS-FOO: call in clone _Z3fooR2B0j promoted and assigned to call function clone _ZN2B03barEj
+; REMARKS-FOO: Promote indirect call to _ZN2B03barEj with count 2 out of 2
+; REMARKS-FOO: call in clone _Z3fooR2B0j.memprof.1 promoted and assigned to call function clone _ZN2B03barEj.memprof.1
+; REMARKS-FOO: created clone _ZN2B03barEj.memprof.1
+; REMARKS-FOO: call in clone _ZN2B03barEj marked with memprof allocation attribute notcold
+; REMARKS-FOO: call in clone _ZN2B03barEj.memprof.1 marked with memprof allocation attribute cold
+; REMARKS-FOO: created clone _ZN1B3barEj.memprof.1
+; REMARKS-FOO: call in clone _ZN1B3barEj marked with memprof allocation attribute notcold
+; REMARKS-FOO: call in clone _ZN1B3barEj.memprof.1 marked with memprof allocation attribute cold
+
+; STATS: 2 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during whole program analysis
+; STATS-BE: 4 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during ThinLTO backend
+; STATS: 2 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during whole program analysis
+; STATS-BE: 4 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during ThinLTO backend
+; STATS: 3 memprof-context-disambiguation - Number of function clones created during whole program analysis
+; STATS-BE: 5 memprof-context-disambiguation - Number of function clones created during ThinLTO backend
+
+; IR: define {{.*}} @_Z3fooR2B0j(
+; IR:   %1 = icmp eq ptr %0, @_ZN1B3barEj
+; IR:   br i1 %1, label %if.true.direct_targ, label %if.false.orig_indirect
+; IR: if.true.direct_targ:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[NOTCOLD:[0-9]+]]
+; IR: if.false.orig_indirect:
+; IR:   %2 = icmp eq ptr %0, @_ZN2B03barEj
+; IR:   br i1 %2, label %if.true.direct_targ1, label %if.false.orig_indirect2
+; IR: if.true.direct_targ1:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[NOTCOLD]]
+; IR: if.false.orig_indirect2:
+; IR:   call {{.*}} %0
+
+; IR: define {{.*}} @_Z3fooR2B0j.memprof.1(
+;; We should still compare against the original versions of bar since that is
+;; what is in the vtable. However, we should have called the cloned versions
+;; that perform cold allocations, which were subsequently inlined.
+; IR:   %1 = icmp eq ptr %0, @_ZN1B3barEj
+; IR:   br i1 %1, label %if.true.direct_targ, label %if.false.orig_indirect
+; IR: if.true.direct_targ:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[COLD:[0-9]+]]
+; IR: if.false.orig_indirect:
+; IR:   %2 = icmp eq ptr %0, @_ZN2B03barEj
+; IR:   br i1 %2, label %if.true.direct_targ1, label %if.false.orig_indirect2
+; IR: if.true.direct_targ1:
+; IR:   call {{.*}} @_Znwm(i64 noundef 4) #[[COLD]]
+; IR: if.false.orig_indirect2:
+; IR:   call {{.*}} %0
+
+; IR: attributes #[[NOTCOLD]] = {{.*}} "memprof"="notcold"
+; IR: attributes #[[COLD]] = {{.*}} "memprof"="cold"
+
+; STATS-BE-DISTRIB: 2 memprof-context-disambiguation - Number of cold static allocations (possibly cloned) during ThinLTO backend
+; STATS-BE-DISTRIB: 2 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned) during ThinLTO backend
+; STATS-BE-DISTRIB: 3 memprof-context-disambiguation - Number of function clones created during ThinLTO backend
+
+;--- foo.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @_Z3fooR2B0j(ptr %b) {
+entry:
+  %0 = load ptr, ptr %b, align 8
+  %call = tail call i32 %0(ptr null, i32 0), !prof !0, !callsite !1
+  ret i32 0
+}
+
+!0 = !{!"VP", i32 0, i64 4, i64 4445083295448962937, i64 2, i64 -2718743882639408571, i64 2}
+!1 = !{i64 -2101080423462424381}
+
+;--- main.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@_ZTV1B = external constant { [3 x ptr] }
+@_ZTVN10__cxxabiv120__si_class_type_infoE = external global [0 x ptr]
+@_ZTS1B = external constant [3 x i8]
+@_ZTVN10__cxxabiv117__class_type_infoE = external global [0 x ptr]
+@_ZTS2B0 = external constant [4 x i8]
+@_ZTI2B0 = external constant { ptr, ptr }
+@_ZTI1B = external constant { ptr, ptr, ptr }
+@_ZTV2B0 = external constant { [3 x ptr] }
+
+define i32 @main() !prof !29 {
+entry:
+  %call2 = call i32 @_Z3fooR2B0j(ptr null, i32 0), !callsite !30
+  %call4 = call i32 @_Z3fooR2B0j(ptr null, i32 0), !callsite !31
+  %call6 = call i32 @_Z3fooR2B0j(ptr null, i32 0), !callsite !32
+  ret i32 0
+}
+
+declare i32 @_Z3fooR2B0j(ptr, i32)
+
+define i32 @_ZN2B03barEj(ptr %this, i32 %s) {
+entry:
+  %call = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !33, !callsite !38
+  store volatile i32 0, ptr %call, align 4
+  ret i32 0
+}
+
+declare ptr @_Znwm(i64)
+
+declare void @_Z8externalPi()
+
+declare void @_ZdlPvm()
+
+define i32 @_ZN1B3barEj(ptr %this, i32 %s) {
+entry:
+  %call = tail call ptr @_Znwm(i64 noundef 4) #0, !memprof !39, !callsite !44
+  store volatile i32 0, ptr %call, align 4
+  ret i32 0
+}
+
+; uselistorder directives
+uselistorder ptr @_Z3fooR2B0j, { 2, 1, 0 }
+
+attributes #0 = { builtin allocsize(0) }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9, !10, !11}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 13}
+!4 = !{!"MaxCount", i64 4}
+!5 = !{!"MaxInternalCount", i64 0}
+!6 = !{!"MaxFunctionCount", i64 4}
+!7 = !{!"NumCounts", i64 5}
+!8 = !{!"NumFunctions", i64 5}
+!9 = !{!"IsPartialProfile", i64 0}
+!10 = !{!"PartialProfileRatio", double 0.000000e+00}
+!11 = !{!"DetailedSummary", !12}
+!12 = !{!13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28}
+!13 = !{i32 10000, i64 0, i32 0}
+!14 = !{i32 100000, i64 4, i32 2}
+!15 = !{i32 200000, i64 4, i32 2}
+!16 = !{i32 300000, i64 4, i32 2}
+!17 = !{i32 400000, i64 4, i32 2}
+!18 = !{i32 500000, i64 4, i32 2}
+!19 = !{i32 600000, i64 4, i32 2}
+!20 = !{i32 700000, i64 2, i32 4}
+!21 = !{i32 800000, i64 2, i32 4}
+!22 = !{i32 900000, i64 2, i32 4}
+!23 = !{i32 950000, i64 2, i32 4}
+!24 = !{i32 990000, i64 2, i32 4}
+!25 = !{i32 999000, i64 2, i32 4}
+!26 = !{i32 999900, i64 2, i32 4}
+!27 = !{i32 999990, i64 2, i32 4}
+!28 = !{i32 999999, i64 2, i32 4}
+!29 = !{!"function_entry_count", i64 1}
+!30 = !{i64 -6490791336773930154}
+!31 = !{i64 5188446645037944434}
+!32 = !{i64 5583420417449503557}
+!33 = !{!34, !36}
+!34 = !{!35, !"notcold"}
+!35 = !{i64 -852997907418798798, i64 -2101080423462424381, i64 5188446645037944434}
+!36 = !{!37, !"cold"}
+!37 = !{i64 -852997907418798798, i64 -2101080423462424381, i64 5583420417449503557}
+!38 = !{i64 -852997907418798798}
+!39 = !{!40, !42}
+!40 = !{!41, !"notcold"}
+!41 = !{i64 4457553070050523782, i64 -2101080423462424381, i64 132626519179914298}
+!42 = !{!43, !"cold"}
+!43 = !{i64 4457553070050523782, i64 -2101080423462424381, i64 -6490791336773930154}
+!44 = !{i64 4457553070050523782}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-minimum-maximum.ll b/llvm/test/Transforms/Attributor/nofpclass-minimum-maximum.ll
index f60581c0089afea7d582a15dec83bc688cbcd041..d9b1375d3c6337578a2d3c58b2108fe7894e7456 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-minimum-maximum.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-minimum-maximum.ll
@@ -511,6 +511,6 @@ attributes #3 = { "denormal-fp-math"="dynamic,dynamic" }
 attributes #4 = { "denormal-fp-math"="ieee,preserve-sign" }
 attributes #5 = { "denormal-fp-math"="preserve-sign,ieee" }
 attributes #6 = { "denormal-fp-math"="ieee,positive-zero" }
-attributes #7 = { "denormal-fp-math"="positive-zero-sign,ieee" }
+attributes #7 = { "denormal-fp-math"="positive-zero,ieee" }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll b/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll
index cea11a3c58e3c4ca065e354b59586fef2a87e244..ddfddbc22a51795fdfb289ef76c2eec6dae7ac7b 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll
@@ -511,6 +511,6 @@ attributes #3 = { "denormal-fp-math"="dynamic,dynamic" }
 attributes #4 = { "denormal-fp-math"="ieee,preserve-sign" }
 attributes #5 = { "denormal-fp-math"="preserve-sign,ieee" }
 attributes #6 = { "denormal-fp-math"="ieee,positive-zero" }
-attributes #7 = { "denormal-fp-math"="positive-zero-sign,ieee" }
+attributes #7 = { "denormal-fp-math"="positive-zero,ieee" }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-only-one-live-use.ll b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-only-one-live-use.ll
new file mode 100644
index 0000000000000000000000000000000000000000..043b73e28b89377005375fe809daeb149e644988
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-only-one-live-use.ll
@@ -0,0 +1,22 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=inline -inline-threshold=0 -debug-only=inline-cost %s -o - 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: Analyzing call of callee_not_only_one_live_use... (caller:caller)
+; CHECK: Cost: -30
+; CHECK: Analyzing call of callee_only_one_live_use... (caller:caller)
+; CHECK: Cost: -165030
+
+define internal void @callee_not_only_one_live_use() {
+  ret void
+}
+
+define internal void @callee_only_one_live_use() {
+  ret void
+}
+
+define void @caller() {
+  call void @callee_not_only_one_live_use()
+  call void @callee_not_only_one_live_use()
+  call void @callee_only_one_live_use()
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/and-or-icmps.ll b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
index ad28ad980de5b47fbbbec1548c8143ee5c0665f5..eb4723c86542de5f38ccc68928c0752f2d3aded8 100644
--- a/llvm/test/Transforms/InstCombine/and-or-icmps.ll
+++ b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
@@ -983,7 +983,7 @@ define <2 x i1> @substitute_constant_or_ne_slt_swap_vec_poison(<2 x i8> %x, <2 x
 define <2 x i1> @substitute_constant_or_ne_slt_swap_vec_logical(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @substitute_constant_or_ne_slt_swap_vec_logical(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ne <2 x i8> [[X:%.*]], <i8 42, i8 poison>
-; CHECK-NEXT:    [[C2:%.*]] = icmp slt <2 x i8> [[Y:%.*]], [[X]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt <2 x i8> [[Y:%.*]], <i8 42, i8 poison>
 ; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C1]], <2 x i1> <i1 true, i1 true>, <2 x i1> [[C2]]
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/fcmp-denormals-are-zero.ll b/llvm/test/Transforms/InstCombine/fcmp-denormals-are-zero.ll
index 22c422c816039558a185890ff9a53a9481d7192b..eea1dda6230a9d316b34be6336d931cd41bb525b 100644
--- a/llvm/test/Transforms/InstCombine/fcmp-denormals-are-zero.ll
+++ b/llvm/test/Transforms/InstCombine/fcmp-denormals-are-zero.ll
@@ -350,5 +350,5 @@ declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
 
 attributes #0 = { "denormal-fp-math"="ieee,preserve-sign" }
 attributes #1 = { "denormal-fp-math"="ieee,positive-zero" }
-attributes #2 = { "denormal-fp-math"="ieee,iee" }
+attributes #2 = { "denormal-fp-math"="ieee,ieee" }
 attributes #3 = { "denormal-fp-math-f32"="ieee,preserve-sign" }
diff --git a/llvm/test/Transforms/InstCombine/freeze.ll b/llvm/test/Transforms/InstCombine/freeze.ll
index 5fedb1f857503584afd5abd2f43858966a63bdc0..23785eba055544d073340781cc5f6dc75e822352 100644
--- a/llvm/test/Transforms/InstCombine/freeze.ll
+++ b/llvm/test/Transforms/InstCombine/freeze.ll
@@ -1182,6 +1182,17 @@ define ptr @propagate_drop_flags_gep_nuw(ptr %p) {
   ret ptr %gep.fr
 }
 
+define i1 @propagate_drop_flags_icmp(i32 %a, i32 %b) {
+; CHECK-LABEL: @propagate_drop_flags_icmp(
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i32 [[A:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i32 [[A_FR]], 3
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %ret = icmp samesign ult i32 %a, 3
+  %ret.fr = freeze i1 %ret
+  ret i1 %ret.fr
+}
+
 declare i32 @llvm.umax.i32(i32 %a, i32 %b)
 
 define i32 @freeze_call_with_range_attr(i32 %a) {
diff --git a/llvm/test/Transforms/InstCombine/is_fpclass.ll b/llvm/test/Transforms/InstCombine/is_fpclass.ll
index ad0bcd1b0c437cbd1d85cdc161f74d925b99716d..24a331dfb48bb014f2d9bf94883118344631eb2f 100644
--- a/llvm/test/Transforms/InstCombine/is_fpclass.ll
+++ b/llvm/test/Transforms/InstCombine/is_fpclass.ll
@@ -376,7 +376,7 @@ define i1 @test_class_is_not_p0_n0_psub_nsub_f32_dapz(float %x) "denormal-fp-mat
   ret i1 %val
 }
 
-define i1 @test_class_is_not_p0_n0_psub_nsub_f32_dynamic(float %x) "denormal-fp-math"="ieee,dynamiz" {
+define i1 @test_class_is_not_p0_n0_psub_nsub_f32_dynamic(float %x) "denormal-fp-math"="ieee,dynamic" {
 ; CHECK-LABEL: @test_class_is_not_p0_n0_psub_nsub_f32_dynamic(
 ; CHECK-NEXT:    [[VAL:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X:%.*]], i32 783)
 ; CHECK-NEXT:    ret i1 [[VAL]]
diff --git a/llvm/test/Transforms/InstCombine/ispow2.ll b/llvm/test/Transforms/InstCombine/ispow2.ll
index c21ad95f83a1c4d06d3db2ab98319bbd8d655e6c..348508769c581fe6ba1355ceb9f7e8d4abe6b6df 100644
--- a/llvm/test/Transforms/InstCombine/ispow2.ll
+++ b/llvm/test/Transforms/InstCombine/ispow2.ll
@@ -977,11 +977,7 @@ define i1 @is_pow2or0_ctpop_wrong_pred2_logical(i32 %x) {
 
 define <2 x i1> @is_pow2or0_ctpop_commute_vec_wrong_pred3(<2 x i8> %x) {
 ; CHECK-LABEL: @is_pow2or0_ctpop_commute_vec_wrong_pred3(
-; CHECK-NEXT:    [[T0:%.*]] = tail call range(i8 0, 9) <2 x i8> @llvm.ctpop.v2i8(<2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[T0]], <i8 1, i8 1>
-; CHECK-NEXT:    [[ISZERO:%.*]] = icmp eq <2 x i8> [[X]], zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = and <2 x i1> [[CMP]], [[ISZERO]]
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %t0 = tail call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> %x)
   %cmp = icmp eq <2 x i8> %t0, <i8 1, i8 1>
@@ -1174,11 +1170,7 @@ define i1 @isnot_pow2nor0_ctpop_wrong_pred2_logical(i32 %x) {
 
 define <2 x i1> @isnot_pow2nor0_wrong_pred3_ctpop_commute_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @isnot_pow2nor0_wrong_pred3_ctpop_commute_vec(
-; CHECK-NEXT:    [[T0:%.*]] = tail call range(i8 0, 9) <2 x i8> @llvm.ctpop.v2i8(<2 x i8> [[X:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i8> [[T0]], <i8 1, i8 1>
-; CHECK-NEXT:    [[NOTZERO:%.*]] = icmp ne <2 x i8> [[X]], zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = or <2 x i1> [[CMP]], [[NOTZERO]]
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
 ;
   %t0 = tail call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> %x)
   %cmp = icmp ne <2 x i8> %t0, <i8 1, i8 1>
@@ -1522,3 +1514,61 @@ define <2 x i1> @not_pow2_or_z_known_bits_fail_wrong_cmp(<2 x i32> %xin) {
   %r = icmp ugt <2 x i32> %cnt, <i32 2, i32 2>
   ret <2 x i1> %r
 }
+
+; Make sure that range attributes on return values are dropped after merging these two icmps
+
+define i1 @has_single_bit(i32 %x) {
+; CHECK-LABEL: @has_single_bit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[POPCNT:%.*]] = call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[X:%.*]])
+; CHECK-NEXT:    [[SEL:%.*]] = icmp eq i32 [[POPCNT]], 1
+; CHECK-NEXT:    ret i1 [[SEL]]
+;
+entry:
+  %cmp1 = icmp ne i32 %x, 0
+  %popcnt = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp2 = icmp ult i32 %popcnt, 2
+  %sel = select i1 %cmp1, i1 %cmp2, i1 false
+  ret i1 %sel
+}
+
+define i1 @has_single_bit_inv(i32 %x) {
+; CHECK-LABEL: @has_single_bit_inv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[POPCNT:%.*]] = call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[X:%.*]])
+; CHECK-NEXT:    [[SEL:%.*]] = icmp ne i32 [[POPCNT]], 1
+; CHECK-NEXT:    ret i1 [[SEL]]
+;
+entry:
+  %cmp1 = icmp eq i32 %x, 0
+  %popcnt = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp2 = icmp ugt i32 %popcnt, 1
+  %sel = select i1 %cmp1, i1 true, i1 %cmp2
+  ret i1 %sel
+}
+
+define i1 @is_power2_or_zero_with_range(i32 %x) {
+; CHECK-LABEL: @is_power2_or_zero_with_range(
+; CHECK-NEXT:    [[CTPOP:%.*]] = call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[X:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i32 [[CTPOP]], 2
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %ctpop = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp eq i32 %ctpop, 1
+  %notzero = icmp eq i32 %x, 0
+  %res = select i1 %notzero, i1 true, i1 %cmp
+  ret i1 %res
+}
+
+define i1 @is_power2_or_zero_inv_with_range(i32 %x) {
+; CHECK-LABEL: @is_power2_or_zero_inv_with_range(
+; CHECK-NEXT:    [[CTPOP:%.*]] = call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[X:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ugt i32 [[CTPOP]], 1
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %ctpop = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 %x)
+  %cmp = icmp ne i32 %ctpop, 1
+  %notzero = icmp ne i32 %x, 0
+  %res = select i1 %notzero, i1 %cmp, i1 false
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/select-binop-cmp.ll b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
index 647287ef5ebad1749d31afb8f3d70980c27a6679..cd8c29ba4cd8196fb8bb7e92cd45bf48ef448e13 100644
--- a/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
@@ -552,12 +552,12 @@ define i32 @select_xor_icmp_bad_6(i32 %x, i32 %y, i32 %z) {
   ret i32 %C
 }
 
-; Value equivalence substitution is all-or-nothing, so needs a scalar compare.
+; Value equivalence substitution is valid.
 
-define <2 x i8> @select_xor_icmp_vec_bad(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
-; CHECK-LABEL: @select_xor_icmp_vec_bad(
+define <2 x i8> @select_xor_icmp_vec_equivalence(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_xor_icmp_vec_equivalence(
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 5, i8 3>
-; CHECK-NEXT:    [[B:%.*]] = xor <2 x i8> [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = xor <2 x i8> [[Z:%.*]], <i8 5, i8 3>
 ; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i8> [[B]], <2 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x i8> [[C]]
 ;
@@ -567,7 +567,7 @@ define <2 x i8> @select_xor_icmp_vec_bad(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z)
   ret <2 x i8>  %C
 }
 
-; Value equivalence substitution is all-or-nothing, so needs a scalar compare.
+; Value equivalence substitution is invalid due to lane-crossing shufflevector.
 
 define <2 x i32> @vec_select_no_equivalence(<2 x i32> %x) {
 ; CHECK-LABEL: @vec_select_no_equivalence(
diff --git a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
index cc8f5d53fddddd953b609f2dfea2532bc993abc3..2e6fe26c03051bb3c2ec0c9e61252520368e8848 100644
--- a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
+++ b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
@@ -208,10 +208,8 @@ define <2 x i32> @select_clz_to_ctz_vec_with_undef(<2 x i32> %a) {
 ; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i32> zeroinitializer, [[A:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[A]], [[SUB]]
 ; CHECK-NEXT:    [[LZ:%.*]] = tail call range(i32 0, 33) <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[AND]], i1 true)
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq <2 x i32> [[A]], zeroinitializer
 ; CHECK-NEXT:    [[SUB1:%.*]] = xor <2 x i32> [[LZ]], <i32 31, i32 undef>
-; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[TOBOOL]], <2 x i32> [[LZ]], <2 x i32> [[SUB1]]
-; CHECK-NEXT:    ret <2 x i32> [[COND]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB1]]
 ;
   %sub = sub <2 x i32> zeroinitializer, %a
   %and = and <2 x i32> %sub, %a
diff --git a/llvm/test/Transforms/InstCombine/select-value-equivalence.ll b/llvm/test/Transforms/InstCombine/select-value-equivalence.ll
new file mode 100644
index 0000000000000000000000000000000000000000..28984282511924b5e311423e59a3cc1a4d0067da
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-value-equivalence.ll
@@ -0,0 +1,311 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=instcombine -S %s | FileCheck %s
+
+define <2 x i8> @select_icmp_insertelement_eq(<2 x i8> %x, <2 x i8> %y, i8 %i) {
+; CHECK-LABEL: define <2 x i8> @select_icmp_insertelement_eq(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], i8 [[I:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[Y]], <i8 2, i8 2>
+; CHECK-NEXT:    [[INSERT:%.*]] = insertelement <2 x i8> <i8 2, i8 2>, i8 0, i8 [[I]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[INSERT]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[RETVAL]]
+;
+  %cmp = icmp eq <2 x i8> %y, <i8 2, i8 2>
+  %insert = insertelement <2 x i8> %y, i8 0, i8 %i
+  %retval = select <2 x i1> %cmp, <2 x i8> %insert, <2 x i8> %x
+  ret <2 x i8> %retval
+}
+
+define <2 x i8> @select_icmp_insertelement_ne(<2 x i8> %x, <2 x i8> %y, i8 %i) {
+; CHECK-LABEL: define <2 x i8> @select_icmp_insertelement_ne(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], i8 [[I:%.*]]) {
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq <2 x i8> [[Y]], <i8 2, i8 2>
+; CHECK-NEXT:    [[INSERT:%.*]] = insertelement <2 x i8> <i8 2, i8 2>, i8 0, i8 [[I]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[CMP_NOT]], <2 x i8> [[INSERT]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[RETVAL]]
+;
+  %cmp = icmp ne <2 x i8> %y, <i8 2, i8 2>
+  %insert = insertelement <2 x i8> %y, i8 0, i8 %i
+  %retval = select <2 x i1> %cmp, <2 x i8> %x, <2 x i8> %insert
+  ret <2 x i8> %retval
+}
+
+define <2 x i8> @select_icmp_shufflevector_identity(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: define <2 x i8> @select_icmp_shufflevector_identity(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[Y]], <i8 2, i8 2>
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 2, i8 2>, <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[RETVAL]]
+;
+  %cmp = icmp eq <2 x i8> %y, <i8 2, i8 2>
+  %shuffle = shufflevector <2 x i8> %y, <2 x i8> poison, <2 x i32> <i32 0, i32 1>
+  %retval = select <2 x i1> %cmp, <2 x i8> %shuffle, <2 x i8> %x
+  ret <2 x i8> %retval
+}
+
+define <4 x i8> @select_icmp_shufflevector_select(<4 x i8> %x, <4 x i8> %y, <4 x i8> %z) {
+; CHECK-LABEL: define <4 x i8> @select_icmp_shufflevector_select(
+; CHECK-SAME: <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i8> [[Z:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i8> [[Y]], <i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i8> [[Z]], <4 x i8> <i8 poison, i8 2, i8 poison, i8 2>, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <4 x i1> [[CMP]], <4 x i8> [[SHUFFLE]], <4 x i8> [[X]]
+; CHECK-NEXT:    ret <4 x i8> [[RETVAL]]
+;
+  %cmp = icmp eq <4 x i8> %y, <i8 2, i8 2, i8 2, i8 2>
+  %shuffle = shufflevector <4 x i8> %y, <4 x i8> %z, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %retval = select <4 x i1> %cmp, <4 x i8> %shuffle, <4 x i8> %x
+  ret <4 x i8> %retval
+}
+
+define <2 x i8> @select_icmp_shufflevector_lanecrossing(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: define <2 x i8> @select_icmp_shufflevector_lanecrossing(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[Y]], <i8 2, i8 2>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i8> [[Y]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[SHUFFLE]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[RETVAL]]
+;
+  %cmp = icmp eq <2 x i8> %y, <i8 2, i8 2>
+  %shuffle = shufflevector <2 x i8> %y, <2 x i8> poison, <2 x i32> <i32 1, i32 0>
+  %retval = select <2 x i1> %cmp, <2 x i8> %shuffle, <2 x i8> %x
+  ret <2 x i8> %retval
+}
+
+declare <2 x i8> @fn(<2 x i8>) speculatable
+
+define <2 x i8> @select_icmp_call_possibly_lanecrossing(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: define <2 x i8> @select_icmp_call_possibly_lanecrossing(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[Y]], <i8 2, i8 2>
+; CHECK-NEXT:    [[CALL:%.*]] = call <2 x i8> @fn(<2 x i8> [[Y]])
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[CALL]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[RETVAL]]
+;
+  %cmp = icmp eq <2 x i8> %y, <i8 2, i8 2>
+  %call = call <2 x i8> @fn(<2 x i8> %y)
+  %retval = select <2 x i1> %cmp, <2 x i8> %call, <2 x i8> %x
+  ret <2 x i8> %retval
+}
+
+define float @select_fcmp_fadd_oeq_not_zero(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_oeq_not_zero(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp oeq float [[Y]], 2.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[FADD]], float [[X]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp oeq float %y, 2.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %fadd, float %x
+  ret float %retval
+}
+
+define float @select_fcmp_fadd_une_not_zero(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_une_not_zero(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp une float [[Y]], 2.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[X]], float [[FADD]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp une float %y, 2.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %x, float %fadd
+  ret float %retval
+}
+
+define float @select_fcmp_fadd_ueq_nnan_not_zero(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_ueq_nnan_not_zero(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp nnan ueq float [[Y]], 2.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[FADD]], float [[X]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp nnan ueq float %y, 2.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %fadd, float %x
+  ret float %retval
+}
+
+define float @select_fcmp_fadd_one_nnan_not_zero(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_one_nnan_not_zero(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp nnan one float [[Y]], 2.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[X]], float [[FADD]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp nnan one float %y, 2.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %x, float %fadd
+  ret float %retval
+}
+
+define float @select_fcmp_fadd_ueq(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_ueq(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp ueq float [[Y]], 2.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[FADD]], float [[X]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp ueq float %y, 2.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %fadd, float %x
+  ret float %retval
+}
+
+define float @select_fcmp_fadd_one(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_one(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp one float [[Y]], 2.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[X]], float [[FADD]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp one float %y, 2.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %x, float %fadd
+  ret float %retval
+}
+
+define float @select_fcmp_fadd_oeq_zero(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_oeq_zero(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[FADD]], float [[X]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp oeq float %y, 0.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %fadd, float %x
+  ret float %retval
+}
+
+define float @select_fcmp_fadd_une_zero(float %x, float %y) {
+; CHECK-LABEL: define float @select_fcmp_fadd_une_zero(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp une float [[Y]], 0.000000e+00
+; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[FCMP]], float [[X]], float [[FADD]]
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %fcmp = fcmp une float %y, 0.0
+  %fadd = fadd float %x, %y
+  %retval = select i1 %fcmp, float %x, float %fadd
+  ret float %retval
+}
+
+define <2 x float> @select_fcmp_fadd_oeq_not_zero_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_oeq_not_zero_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp oeq <2 x float> [[Y]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[FADD]], <2 x float> [[X]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp oeq <2 x float> %y, <float 2.0, float 2.0>
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %fadd, <2 x float> %x
+  ret <2 x float> %retval
+}
+
+define <2 x float> @select_fcmp_fadd_une_not_zero_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_une_not_zero_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp une <2 x float> [[Y]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[X]], <2 x float> [[FADD]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp une <2 x float> %y, <float 2.0, float 2.0>
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %x, <2 x float> %fadd
+  ret <2 x float> %retval
+}
+
+define <2 x float> @select_fcmp_fadd_ueq_nnan_not_zero_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_ueq_nnan_not_zero_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp nnan ueq <2 x float> [[Y]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[FADD]], <2 x float> [[X]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp nnan ueq <2 x float> %y, <float 2.0, float 2.0>
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %fadd, <2 x float> %x
+  ret <2 x float> %retval
+}
+
+define <2 x float> @select_fcmp_fadd_one_nnan_not_zero_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_one_nnan_not_zero_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp nnan one <2 x float> [[Y]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[X]], <2 x float> [[FADD]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp nnan one <2 x float> %y, <float 2.0, float 2.0>
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %x, <2 x float> %fadd
+  ret <2 x float> %retval
+}
+
+define <2 x float> @select_fcmp_fadd_ueq_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_ueq_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp ueq <2 x float> [[Y]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[FADD]], <2 x float> [[X]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp ueq <2 x float> %y, <float 2.0, float 2.0>
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %fadd, <2 x float> %x
+  ret <2 x float> %retval
+}
+
+define <2 x float> @select_fcmp_fadd_one_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_one_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp one <2 x float> [[Y]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[X]], <2 x float> [[FADD]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp one <2 x float> %y, <float 2.0, float 2.0>
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %x, <2 x float> %fadd
+  ret <2 x float> %retval
+}
+
+define <2 x float> @select_fcmp_fadd_oeq_zero_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_oeq_zero_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp oeq <2 x float> [[Y]], zeroinitializer
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[FADD]], <2 x float> [[X]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp oeq <2 x float> %y, zeroinitializer
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %fadd, <2 x float> %x
+  ret <2 x float> %retval
+}
+
+define <2 x float> @select_fcmp_fadd_une_zero_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: define <2 x float> @select_fcmp_fadd_une_zero_vec(
+; CHECK-SAME: <2 x float> [[X:%.*]], <2 x float> [[Y:%.*]]) {
+; CHECK-NEXT:    [[FCMP:%.*]] = fcmp une <2 x float> [[Y]], zeroinitializer
+; CHECK-NEXT:    [[FADD:%.*]] = fadd <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[FCMP]], <2 x float> [[X]], <2 x float> [[FADD]]
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %fcmp = fcmp une <2 x float> %y, zeroinitializer
+  %fadd = fadd <2 x float> %x, %y
+  %retval = select <2 x i1> %fcmp, <2 x float> %x, <2 x float> %fadd
+  ret <2 x float> %retval
+}
diff --git a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
index f1f1708333901af50888ac5f8ffdc8f31c09c530..65faf0974b503f195f4a8538422b0ed7e5c6d40d 100644
--- a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
@@ -998,8 +998,8 @@ define nofpclass(nan inf) float @pow_f32(float nofpclass(nan inf) %arg, float no
 ; CHECK-NEXT:    [[I12:%.*]] = select i1 [[I11]], float [[ARG]], float 1.000000e+00
 ; CHECK-NEXT:    [[I13:%.*]] = tail call noundef float @llvm.copysign.f32(float noundef [[I4]], float noundef [[I12]])
 ; CHECK-NEXT:    [[I17:%.*]] = fcmp oeq float [[ARG]], 0.000000e+00
-; CHECK-NEXT:    [[I21:%.*]] = select i1 [[I11]], float [[ARG]], float 0.000000e+00
-; CHECK-NEXT:    [[I22:%.*]] = tail call noundef nofpclass(nan sub norm) float @llvm.copysign.f32(float noundef 0.000000e+00, float noundef [[I21]])
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call nofpclass(nan sub norm) float @llvm.copysign.f32(float 0.000000e+00, float [[ARG]])
+; CHECK-NEXT:    [[I22:%.*]] = select i1 [[I11]], float [[TMP0]], float 0.000000e+00
 ; CHECK-NEXT:    [[I23:%.*]] = select i1 [[I17]], float [[I22]], float [[I13]]
 ; CHECK-NEXT:    [[I24:%.*]] = fcmp oeq float [[ARG]], 1.000000e+00
 ; CHECK-NEXT:    [[I25:%.*]] = fcmp oeq float [[ARG1]], 0.000000e+00
diff --git a/llvm/test/Transforms/InstSimplify/select-abs.ll b/llvm/test/Transforms/InstSimplify/select-abs.ll
index 6b4708443fb26182228114f107ea35e16dd37717..dad2dc620c7c7cf08deb1d5ac3195c3bc391c1a3 100644
--- a/llvm/test/Transforms/InstSimplify/select-abs.ll
+++ b/llvm/test/Transforms/InstSimplify/select-abs.ll
@@ -232,14 +232,11 @@ entry:
   ret i64 %res
 }
 
-; TODO: Handle vector cases?
 define <4 x i16> @select_v4i16_eq0_abs_t(<4 x i16> %a) {
 ; CHECK-LABEL: @select_v4i16_eq0_abs_t(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq <4 x i16> [[A:%.*]], zeroinitializer
-; CHECK-NEXT:    [[ABS:%.*]] = tail call <4 x i16> @llvm.abs.v4i16(<4 x i16> [[A]], i1 true)
-; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[COND]], <4 x i16> zeroinitializer, <4 x i16> [[ABS]]
-; CHECK-NEXT:    ret <4 x i16> [[RES]]
+; CHECK-NEXT:    [[ABS:%.*]] = tail call <4 x i16> @llvm.abs.v4i16(<4 x i16> [[A:%.*]], i1 true)
+; CHECK-NEXT:    ret <4 x i16> [[ABS]]
 ;
 entry:
   %cond = icmp eq <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
@@ -251,10 +248,8 @@ entry:
 define <4 x i16> @select_v4i16_ne0_abs_t(<4 x i16> %a) {
 ; CHECK-LABEL: @select_v4i16_ne0_abs_t(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[COND:%.*]] = icmp ne <4 x i16> [[A:%.*]], zeroinitializer
-; CHECK-NEXT:    [[ABS:%.*]] = tail call <4 x i16> @llvm.abs.v4i16(<4 x i16> [[A]], i1 true)
-; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[COND]], <4 x i16> [[ABS]], <4 x i16> zeroinitializer
-; CHECK-NEXT:    ret <4 x i16> [[RES]]
+; CHECK-NEXT:    [[ABS:%.*]] = tail call <4 x i16> @llvm.abs.v4i16(<4 x i16> [[A:%.*]], i1 true)
+; CHECK-NEXT:    ret <4 x i16> [[ABS]]
 ;
 entry:
   %cond = icmp ne <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
@@ -305,10 +300,8 @@ entry:
 define <4 x i16> @badsplat1_select_v4i16_ne0_abs(<4 x i16> %a) {
 ; CHECK-LABEL: @badsplat1_select_v4i16_ne0_abs(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[COND:%.*]] = icmp ne <4 x i16> [[A:%.*]], <i16 0, i16 1, i16 0, i16 0>
-; CHECK-NEXT:    [[ABS:%.*]] = tail call <4 x i16> @llvm.abs.v4i16(<4 x i16> [[A]], i1 true)
-; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[COND]], <4 x i16> [[ABS]], <4 x i16> <i16 0, i16 1, i16 0, i16 0>
-; CHECK-NEXT:    ret <4 x i16> [[RES]]
+; CHECK-NEXT:    [[ABS:%.*]] = tail call <4 x i16> @llvm.abs.v4i16(<4 x i16> [[A:%.*]], i1 true)
+; CHECK-NEXT:    ret <4 x i16> [[ABS]]
 ;
 entry:
   %cond = icmp ne <4 x i16> %a, <i16 0, i16 1, i16 0, i16 0>
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
index 1e503afae1a69d9f1079c59b1cf00d02656cf74d..4618e1143dc8f977016f533a3425811ca434519c 100644
--- a/llvm/test/Transforms/InstSimplify/select.ll
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -1087,13 +1087,10 @@ define i32 @select_ctpop_zero(i32 %x) {
   ret i32 %sel
 }
 
-; FIXME: This is safe to fold.
 define <2 x i32> @select_ctpop_zero_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @select_ctpop_zero_vec(
-; CHECK-NEXT:    [[T0:%.*]] = icmp eq <2 x i32> [[X:%.*]], zeroinitializer
-; CHECK-NEXT:    [[T1:%.*]] = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[X]])
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[T0]], <2 x i32> zeroinitializer, <2 x i32> [[T1]]
-; CHECK-NEXT:    ret <2 x i32> [[SEL]]
+; CHECK-NEXT:    [[T1:%.*]] = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i32> [[T1]]
 ;
   %t0 = icmp eq <2 x i32> %x, zeroinitializer
   %t1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %x)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1ec384b05779a86b3df287c4c1450e2941fe1d26
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -0,0 +1,397 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; REQUIRES: asserts
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -mattr=+sve 2>%t | FileCheck %s
+; RUN: cat %t | FileCheck %s --check-prefix=DEBUG
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; DEBUG-LABEL: LV: Checking a loop in 'low_vf_ic_is_better'
+; DEBUG: LV: Found trip count: 0
+; DEBUG: LV: Found maximum trip count: 19
+; DEBUG: LV: IC is 1
+; DEBUG: LV: VF is vscale x 8
+; DEBUG: Main Loop VF:vscale x 8, Main Loop UF:1, Epilogue Loop VF:vscale x 4, Epilogue Loop UF:1
+
+; DEBUG-LABEL: LV: Checking a loop in 'trip_count_too_small'
+; DEBUG: LV: Found a loop with a very small trip count. This loop is worth vectorizing only if no scalar iteration overheads are incurred.
+; DEBUG: LV: Not vectorizing: The trip count is below the minial threshold value..
+
+; DEBUG-LABEL: LV: Checking a loop in 'too_many_runtime_checks'
+; DEBUG: LV: Found trip count: 0
+; DEBUG: LV: Found maximum trip count: 16
+; DEBUG: LV: Clamping the MaxVF to maximum power of two not exceeding the constant trip count: 16
+; DEBUG: LV: IC is 1
+; DEBUG: LV: VF is 16
+; DEBUG: LV: Vectorization is not beneficial: expected trip count < minimum profitable VF (16 < 32)
+; DEBUG: LV: Too many memory checks needed.
+
+; DEBUG-LABEL: LV: Checking a loop in 'overflow_indvar_known_false'
+; DEBUG: LV: Found trip count: 0
+; DEBUG: LV: Found maximum trip count: 1027
+; DEBUG: LV: can fold tail by masking.
+; DEBUG: Executing best plan with VF=vscale x 16, UF=1
+
+define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef %val) {
+; CHECK-LABEL: define void @low_vf_ic_is_better(
+; CHECK-SAME: ptr nocapture noundef [[P:%.*]], i32 [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 19
+; CHECK-NEXT:    br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[WHILE_END:.*]]
+; CHECK:       [[ITER_CHECK]]:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i16 [[VAL]] to i8
+; CHECK-NEXT:    [[V:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[TC]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[TC]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TC]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 19, [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt i64 [[TMP8]], 4294967295
+; CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP21]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP22]], align 1
+; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 8 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    store <vscale x 8 x i8> [[TMP23]], ptr [[TMP22]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT:    [[IND_END5:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 4
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP33]]
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK:       [[VEC_EPILOG_PH]]:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[TMP34:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP35:%.*]] = mul i64 [[TMP34]], 4
+; CHECK-NEXT:    [[N_MOD_VF3:%.*]] = urem i64 [[TMP3]], [[TMP35]]
+; CHECK-NEXT:    [[N_VEC4:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF3]]
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[TMP0]], [[N_VEC4]]
+; CHECK-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[CONV]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX7:%.*]] = add i64 [[TMP0]], [[INDEX6]]
+; CHECK-NEXT:    [[TMP38:%.*]] = add i64 [[OFFSET_IDX7]], 0
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[TMP39]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 4 x i8>, ptr [[TMP40]], align 1
+; CHECK-NEXT:    [[TMP41:%.*]] = add <vscale x 4 x i8> [[WIDE_LOAD7]], [[BROADCAST_SPLAT9]]
+; CHECK-NEXT:    store <vscale x 4 x i8> [[TMP41]], ptr [[TMP40]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT11]] = add nuw i64 [[INDEX6]], [[TMP37]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC4]]
+; CHECK-NEXT:    br i1 [[TMP42]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC4]]
+; CHECK-NEXT:    br i1 [[CMP_N12]], label %[[WHILE_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[ITER_CHECK]] ]
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[TMP43]], [[CONV]]
+; CHECK-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[TMP44:%.*]] = and i64 [[INDVARS_IV_NEXT]], 4294967295
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP44]], 19
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[WHILE_END_LOOPEXIT]], label %[[WHILE_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
+; CHECK:       [[WHILE_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp7 = icmp ult i32 %tc, 19
+  br i1 %cmp7, label %while.preheader, label %while.end
+
+while.preheader:
+  %conv = trunc i16 %val to i8
+  %v = getelementptr inbounds nuw i8, ptr %p, i64 4
+  %0 = zext nneg i32 %tc to i64
+  br label %while.body
+
+while.body:
+  %iv = phi i64 [ %0, %while.preheader ], [ %iv.next, %while.body ]
+  %iv.next = add nuw nsw i64 %iv, 1
+  %arrayidx = getelementptr inbounds nuw i8, ptr %v, i64 %iv
+  %1 = load i8, ptr %arrayidx, align 1
+  %add = add i8 %1, %conv
+  store i8 %add, ptr %arrayidx, align 1
+  %2 = and i64 %iv.next, 4294967295
+  %exitcond.not = icmp eq i64 %2, 19
+  br i1 %exitcond.not, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
+
+define void @trip_count_too_small(ptr nocapture noundef %p, i32 noundef %tc, i16 noundef %val) {
+; CHECK-LABEL: define void @trip_count_too_small(
+; CHECK-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 3
+; CHECK-NEXT:    br i1 [[CMP7]], label %[[WHILE_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_PREHEADER]]:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i16 [[VAL]] to i8
+; CHECK-NEXT:    [[V:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[TC]] to i64
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[WHILE_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[TMP43]], [[CONV]]
+; CHECK-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[TMP44:%.*]] = and i64 [[INDVARS_IV_NEXT]], 4294967295
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP44]], 3
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[WHILE_BODY]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
+; CHECK:       [[WHILE_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp7 = icmp ult i32 %tc, 3
+  br i1 %cmp7, label %while.preheader, label %while.end
+
+while.preheader:
+  %conv = trunc i16 %val to i8
+  %v = getelementptr inbounds nuw i8, ptr %p, i64 4
+  %0 = zext nneg i32 %tc to i64
+  br label %while.body
+
+while.body:
+  %iv = phi i64 [ %0, %while.preheader ], [ %iv.next, %while.body ]
+  %iv.next = add nuw nsw i64 %iv, 1
+  %arrayidx = getelementptr inbounds nuw i8, ptr %v, i64 %iv
+  %1 = load i8, ptr %arrayidx, align 1
+  %add = add i8 %1, %conv
+  store i8 %add, ptr %arrayidx, align 1
+  %2 = and i64 %iv.next, 4294967295
+  %exitcond.not = icmp eq i64 %2, 3
+  br i1 %exitcond.not, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
+
+define void @too_many_runtime_checks(ptr nocapture noundef %p, ptr nocapture noundef %p1, ptr nocapture noundef readonly %p2, ptr nocapture noundef readonly %p3, i32 noundef %tc, i16 noundef %val) {
+; CHECK-LABEL: define void @too_many_runtime_checks(
+; CHECK-SAME: ptr nocapture noundef [[P:%.*]], ptr nocapture noundef [[P1:%.*]], ptr nocapture noundef readonly [[P2:%.*]], ptr nocapture noundef readonly [[P3:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TC]], 16
+; CHECK-NEXT:    br i1 [[CMP20]], label %[[WHILE_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_PREHEADER]]:
+; CHECK-NEXT:    [[CONV8:%.*]] = trunc i16 [[VAL]] to i8
+; CHECK-NEXT:    [[V:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[TC]] to i64
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP1]], %[[WHILE_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P2]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP60:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[P3]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP61:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[TMP61]], [[TMP60]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P1]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP62:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[MUL]], [[TMP62]]
+; CHECK-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX5]], align 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP63:%.*]] = load i8, ptr [[ARRAYIDX10]], align 1
+; CHECK-NEXT:    [[ADD12:%.*]] = add i8 [[TMP63]], [[CONV8]]
+; CHECK-NEXT:    store i8 [[ADD12]], ptr [[ARRAYIDX10]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[TMP64:%.*]] = and i64 [[INDVARS_IV_NEXT]], 4294967295
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP64]], 16
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[WHILE_BODY]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
+; CHECK:       [[WHILE_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp20 = icmp ult i32 %tc, 16
+  br i1 %cmp20, label %while.preheader, label %while.end
+
+while.preheader:
+  %0 = trunc i16 %val to i8
+  %v = getelementptr inbounds nuw i8, ptr %p, i64 4
+  %1 = zext nneg i32 %tc to i64
+  br label %while.body
+
+while.body:
+  %iv = phi i64 [ %1, %while.preheader ], [ %iv.next, %while.body ]
+  %arrayidx = getelementptr inbounds nuw i8, ptr %p2, i64 %iv
+  %2 = load i8, ptr %arrayidx, align 1
+  %arrayidx2 = getelementptr inbounds nuw i8, ptr %p3, i64 %iv
+  %3 = load i8, ptr %arrayidx2, align 1
+  %mul = mul i8 %3, %2
+  %arrayidx5 = getelementptr inbounds nuw i8, ptr %p1, i64 %iv
+  %4 = load i8, ptr %arrayidx5, align 1
+  %add = add i8 %mul, %4
+  store i8 %add, ptr %arrayidx5, align 1
+  %arrayidx10 = getelementptr inbounds nuw i8, ptr %v, i64 %iv
+  %5 = load i8, ptr %arrayidx10, align 1
+  %add12 = add i8 %5, %0
+  store i8 %add12, ptr %arrayidx10, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %6 = and i64 %iv.next, 4294967295
+  %exitcond.not = icmp eq i64 %6, 16
+  br i1 %exitcond.not, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
+
+define void @overflow_indvar_known_false(ptr nocapture noundef %p, i32 noundef %tc, i16 noundef %val) vscale_range(1,16) {
+; CHECK-LABEL: define void @overflow_indvar_known_false(
+; CHECK-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[TC:%.*]], i16 noundef [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TC]], 1027
+; CHECK-NEXT:    br i1 [[CMP7]], label %[[WHILE_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; CHECK:       [[WHILE_PREHEADER]]:
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i16 [[VAL]] to i8
+; CHECK-NEXT:    [[V:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
+; CHECK-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[TC]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TC]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP19]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 1028, [[TMP20]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TC]], 1
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i64 1027, [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = add i32 [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp ult i32 [[TMP25]], [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ugt i64 [[TMP23]], 4294967295
+; CHECK-NEXT:    [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    br i1 [[TMP28]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 16
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP1]])
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[CONV]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[V]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP14]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP15]], ptr [[TMP14]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP1]])
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 16 x i1> insertelement (<vscale x 16 x i1> poison, i1 true, i64 0), <vscale x 16 x i1> poison, <vscale x 16 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[TMP16]], i32 0
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[WHILE_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[TMP18]], [[CONV]]
+; CHECK-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[INDVARS_IV_NEXT]], 4294967295
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP29]], 1027
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[WHILE_END_LOOPEXIT]], label %[[WHILE_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[WHILE_END]]
+; CHECK:       [[WHILE_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp7 = icmp ult i32 %tc, 1027
+  br i1 %cmp7, label %while.preheader, label %while.end
+
+while.preheader:
+  %conv = trunc i16 %val to i8
+  %v = getelementptr inbounds nuw i8, ptr %p, i64 4
+  %0 = zext nneg i32 %tc to i64
+  br label %while.body
+
+while.body:
+  %iv = phi i64 [ %0, %while.preheader ], [ %iv.next, %while.body ]
+  %iv.next = add nuw nsw i64 %iv, 1
+  %arrayidx = getelementptr inbounds nuw i8, ptr %v, i64 %iv
+  %1 = load i8, ptr %arrayidx, align 1
+  %add = add i8 %1, %conv
+  store i8 %add, ptr %arrayidx, align 1
+  %2 = and i64 %iv.next, 4294967295
+  %exitcond.not = icmp eq i64 %2, 1027
+  br i1 %exitcond.not, label %while.end, label %while.body, !llvm.loop !0
+
+while.end:
+  ret void
+}
+
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7db47cb9171d245c49a1d49d18b0a2671e9ae4bd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
@@ -0,0 +1,167 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+target triple = "riscv64-unknown-linux-gnu"
+
+; Test case for https://github.com/llvm/llvm-project/issues/111874.
+define i32 @any_of_reduction_used_in_blend(ptr %src, i64 %N, i1 %c.0, i1 %c.1) #0 {
+; CHECK-LABEL: define i32 @any_of_reduction_used_in_blend(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[ANY_OF_RED:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    br i1 [[C_0]], label %[[LOOP_LATCH]], label %[[ELSE_1:.*]]
+; CHECK:       [[ELSE_1]]:
+; CHECK-NEXT:    br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[ELSE_2:.*]]
+; CHECK:       [[ELSE_2]]:
+; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[SRC]], align 8
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq ptr [[L]], null
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C_2]], i32 0, i32 [[ANY_OF_RED]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ANY_OF_RED_NEXT]] = phi i32 [ [[ANY_OF_RED]], %[[LOOP_HEADER]] ], [ [[ANY_OF_RED]], %[[ELSE_1]] ], [ [[SEL]], %[[ELSE_2]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %any.of.red = phi i32 [ 0, %entry ], [ %any.of.red.next, %loop.latch ]
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  br i1 %c.0, label %loop.latch, label %else.1
+
+else.1:
+  br i1 %c.1, label %loop.latch, label %else.2
+
+else.2:
+  %l = load ptr, ptr %src, align 8
+  %c.2 = icmp eq ptr %l, null
+  %sel = select i1 %c.2, i32 0, i32 %any.of.red
+  br label %loop.latch
+
+loop.latch:
+  %any.of.red.next = phi i32 [ %any.of.red, %loop.header ], [ %any.of.red, %else.1 ], [ %sel, %else.2 ]
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  %res = phi i32 [ %any.of.red.next, %loop.latch ]
+  ret i32 %res
+}
+
+define i32 @any_of_reduction_used_in_blend_with_mutliple_phis(ptr %src, i64 %N, i1 %c.0, i1 %c.1) #0 {
+; CHECK-LABEL: define i32 @any_of_reduction_used_in_blend_with_mutliple_phis(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[SRC]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <vscale x 2 x i1> [[BROADCAST_SPLAT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <vscale x 2 x i1> [[BROADCAST_SPLAT2]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x ptr> @llvm.masked.gather.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT4]], i32 8, <vscale x 2 x i1> [[TMP8]], <vscale x 2 x ptr> poison)
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <vscale x 2 x ptr> [[WIDE_MASKED_GATHER]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <vscale x 2 x i1> [[VEC_PHI]], [[TMP9]]
+; CHECK-NEXT:    [[PREDPHI]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[PREDPHI]])
+; CHECK-NEXT:    [[TMP13:%.*]] = freeze i1 [[TMP12]]
+; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP13]], i32 0, i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[ANY_OF_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    br i1 [[C_0]], label %[[X_1:.*]], label %[[ELSE_1:.*]]
+; CHECK:       [[ELSE_1]]:
+; CHECK-NEXT:    br i1 [[C_1]], label %[[X_1]], label %[[ELSE_2:.*]]
+; CHECK:       [[ELSE_2]]:
+; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[SRC]], align 8
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq ptr [[L]], null
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C_2]], i32 0, i32 [[ANY_OF_RED]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[X_1]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[ANY_OF_RED]], %[[LOOP_HEADER]] ], [ [[ANY_OF_RED]], %[[ELSE_1]] ]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ANY_OF_RED_NEXT]] = phi i32 [ [[P]], %[[X_1]] ], [ [[SEL]], %[[ELSE_2]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[LOOP_LATCH]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %any.of.red = phi i32 [ 0, %entry ], [ %any.of.red.next, %loop.latch ]
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  br i1 %c.0, label %x.1, label %else.1
+
+else.1:
+  br i1 %c.1, label %x.1, label %else.2
+
+else.2:
+  %l = load ptr, ptr %src, align 8
+  %c.2 = icmp eq ptr %l, null
+  %sel = select i1 %c.2, i32 0, i32 %any.of.red
+  br label %loop.latch
+
+x.1:
+  %p = phi i32 [ %any.of.red, %loop.header ], [ %any.of.red, %else.1 ]
+  br label %loop.latch
+
+loop.latch:
+  %any.of.red.next = phi i32 [ %p, %x.1 ], [ %sel, %else.2 ]
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  %res = phi i32 [ %any.of.red.next, %loop.latch ]
+  ret i32 %res
+}
+
+attributes #0 = { "target-cpu"="sifive-p670" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 1d5e6c117a2eac639b935ec4ef9be718ef264fc0..9a716f7756072e131f79a58937bf67544e68d2c5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -20,6 +20,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: We can vectorize this loop (with a runtime bound check)!
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  LV: Found trip count: 0
+; CHECK-NEXT:  LV: Found maximum trip count: 4294967295
 ; CHECK-NEXT:  LV: Scalable vectorization is available
 ; CHECK-NEXT:  LV: The max safe fixed VF is: 67108864.
 ; CHECK-NEXT:  LV: The max safe scalable VF is: vscale x 4294967295.
@@ -224,6 +225,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: We can vectorize this loop (with a runtime bound check)!
 ; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  LV: Found trip count: 0
+; CHECK-NEXT:  LV: Found maximum trip count: 4294967295
 ; CHECK-NEXT:  LV: Scalable vectorization is available
 ; CHECK-NEXT:  LV: The max safe fixed VF is: 67108864.
 ; CHECK-NEXT:  LV: The max safe scalable VF is: vscale x 4294967295.
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index 6fba8ccd590c6201ee68b1ce44897d098efe4e35..c63dc9979bce3c10eda87d93c09182422072093b 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -463,9 +463,10 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(ptr %dst, i64 %N
 ; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
 ; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
-; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP8:%.*]] = freeze i64 [[TMP0]]
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP8]], i64 [[SMAX]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -598,9 +599,10 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_frozen_value_in_latch(ptr %dst,
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[FR_N:%.*]] = freeze i64 [[N]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
 ; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[FR_N]]
-; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[TMP0]]
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[SMAX]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[UMIN]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -786,12 +788,13 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
 ; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
 ; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 42, [[TMP1]]
 ; CHECK-NEXT:    [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
-; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[SMAX1]])
+; CHECK-NEXT:    [[TMP10:%.*]] = freeze i64 [[SMAX1]]
+; CHECK-NEXT:    [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP10]], i64 [[SMAX2]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nuw i64 [[UMIN]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -1004,9 +1007,10 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(ptr %dst, i64 %
 ; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
 ; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
-; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP8:%.*]] = freeze i64 [[TMP0]]
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP8]], i64 [[SMAX]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index 697eb1e3405f339b0e18d05c15ffe5450d8b6bb7..1ac4dc098887850f09424ec26125d47495161ca9 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -11,9 +11,6 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
 ; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX27:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
 ; CHECK-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
 ; CHECK-NEXT:    [[CONV33:%.*]] = zext i8 [[TMP10]] to i32
@@ -24,9 +21,6 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
 ; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
 ; CHECK-NEXT:    [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX10_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX13_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX15_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 5
 ; CHECK-NEXT:    [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
 ; CHECK-NEXT:    [[CONV33_1:%.*]] = zext i8 [[TMP5]] to i32
@@ -34,309 +28,313 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
 ; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
 ; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
-; CHECK-NEXT:    [[TMP19:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ADD_PTR_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i8>, ptr [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP21:%.*]] = zext <2 x i8> [[TMP19]] to <2 x i32>
-; CHECK-NEXT:    [[TMP22:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ADD_PTR64_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_1]], align 1
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP31:%.*]] = zext <2 x i8> [[TMP22]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP23:%.*]] = sub <2 x i32> [[TMP21]], [[TMP31]]
-; CHECK-NEXT:    [[TMP49:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
+; CHECK-NEXT:    [[TMP49:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP26:%.*]] = zext <2 x i8> [[TMP49]] to <2 x i32>
-; CHECK-NEXT:    [[TMP27:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP16:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP16]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP50:%.*]] = zext <2 x i8> [[TMP27]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP24:%.*]] = sub <2 x i32> [[TMP26]], [[TMP50]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = shl <2 x i32> [[TMP24]], <i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP30:%.*]] = add <2 x i32> [[TMP25]], [[TMP23]]
-; CHECK-NEXT:    [[TMP32:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX8_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP51:%.*]] = zext <2 x i8> [[TMP32]] to <2 x i32>
-; CHECK-NEXT:    [[TMP56:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP57:%.*]] = zext <2 x i8> [[TMP56]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP35:%.*]] = sub <2 x i32> [[TMP51]], [[TMP57]]
-; CHECK-NEXT:    [[TMP38:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX13_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP39:%.*]] = zext <2 x i8> [[TMP38]] to <2 x i32>
-; CHECK-NEXT:    [[TMP40:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP28:%.*]] = zext <2 x i8> [[TMP40]] to <2 x i32>
-; CHECK-NEXT:    [[TMP36:%.*]] = sub <2 x i32> [[TMP39]], [[TMP28]]
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <4 x i8> [[TMP16]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP61:%.*]] = zext <2 x i8> [[TMP40]] to <2 x i32>
+; CHECK-NEXT:    [[TMP36:%.*]] = sub <2 x i32> [[TMP39]], [[TMP61]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = shl <2 x i32> [[TMP36]], <i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP42:%.*]] = add <2 x i32> [[TMP37]], [[TMP35]]
-; CHECK-NEXT:    [[TMP72:%.*]] = add <2 x i32> [[TMP42]], [[TMP30]]
-; CHECK-NEXT:    [[TMP29:%.*]] = sub <2 x i32> [[TMP30]], [[TMP42]]
-; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <2 x i32> [[TMP72]], i32 0
-; CHECK-NEXT:    [[TMP146:%.*]] = extractelement <2 x i32> [[TMP72]], i32 1
-; CHECK-NEXT:    [[ADD55_2:%.*]] = add i32 [[TMP146]], [[TMP33]]
-; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr i8, ptr null, i64 4
+; CHECK-NEXT:    [[TMP43:%.*]] = add <2 x i32> [[TMP42]], [[TMP30]]
+; CHECK-NEXT:    [[TMP44:%.*]] = sub <2 x i32> [[TMP30]], [[TMP42]]
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <2 x i32> [[TMP43]], i32 0
+; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <2 x i32> [[TMP43]], i32 1
+; CHECK-NEXT:    [[ADD48_2:%.*]] = add i32 [[TMP84]], [[TMP73]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <2 x i32> [[TMP44]], i32 0
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <2 x i32> [[TMP44]], i32 1
+; CHECK-NEXT:    [[ADD55_2:%.*]] = add i32 [[TMP48]], [[TMP47]]
 ; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
-; CHECK-NEXT:    [[ARRAYIDX8_3:%.*]] = getelementptr i8, ptr null, i64 1
-; CHECK-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr i8, ptr null, i64 1
-; CHECK-NEXT:    [[ARRAYIDX15_3:%.*]] = getelementptr i8, ptr null, i64 5
-; CHECK-NEXT:    [[TMP53:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP53:%.*]] = load <2 x i8>, ptr null, align 1
 ; CHECK-NEXT:    [[TMP52:%.*]] = load i8, ptr null, align 1
 ; CHECK-NEXT:    [[TMP62:%.*]] = zext <2 x i8> [[TMP53]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP77:%.*]] = zext i8 [[TMP52]] to i32
-; CHECK-NEXT:    [[TMP54:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP54:%.*]] = load <2 x i8>, ptr null, align 1
 ; CHECK-NEXT:    [[TMP55:%.*]] = zext <2 x i8> [[TMP54]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP59:%.*]] = sub <2 x i32> [[TMP62]], [[TMP55]]
-; CHECK-NEXT:    [[TMP41:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_3]], i64 -4, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP41:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 4, <2 x i1> <i1 true, i1 true>, i32 2)
 ; CHECK-NEXT:    [[TMP58:%.*]] = zext <2 x i8> [[TMP41]] to <2 x i32>
-; CHECK-NEXT:    [[TMP63:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <2 x i32> [[TMP58]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP63:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_3]], align 1
 ; CHECK-NEXT:    [[TMP76:%.*]] = zext <2 x i8> [[TMP63]] to <2 x i32>
-; CHECK-NEXT:    [[TMP45:%.*]] = sub <2 x i32> [[TMP58]], [[TMP76]]
+; CHECK-NEXT:    [[TMP45:%.*]] = sub <2 x i32> [[TMP60]], [[TMP76]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = shl <2 x i32> [[TMP45]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP60:%.*]] = add <2 x i32> [[TMP46]], [[TMP59]]
-; CHECK-NEXT:    [[TMP64:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX8_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP90:%.*]] = add <2 x i32> [[TMP46]], [[TMP59]]
+; CHECK-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr i8, ptr null, i64 2
+; CHECK-NEXT:    [[ARRAYIDX22_3:%.*]] = getelementptr i8, ptr null, i64 2
+; CHECK-NEXT:    [[ARRAYIDX27_3:%.*]] = getelementptr i8, ptr null, i64 6
+; CHECK-NEXT:    [[TMP64:%.*]] = load <2 x i8>, ptr [[ARRAYIDX20_3]], align 1
 ; CHECK-NEXT:    [[TMP79:%.*]] = zext <2 x i8> [[TMP64]] to <2 x i32>
-; CHECK-NEXT:    [[TMP82:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP82:%.*]] = load <2 x i8>, ptr [[ARRAYIDX22_3]], align 1
 ; CHECK-NEXT:    [[TMP91:%.*]] = zext <2 x i8> [[TMP82]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP65:%.*]] = sub <2 x i32> [[TMP79]], [[TMP91]]
 ; CHECK-NEXT:    [[TMP75:%.*]] = call <2 x i8> @llvm.masked.gather.v2i8.v2p0(<2 x ptr> zeroinitializer, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i8> poison)
 ; CHECK-NEXT:    [[TMP98:%.*]] = zext <2 x i8> [[TMP75]] to <2 x i32>
-; CHECK-NEXT:    [[TMP83:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP84:%.*]] = zext <2 x i8> [[TMP83]] to <2 x i32>
-; CHECK-NEXT:    [[TMP69:%.*]] = sub <2 x i32> [[TMP98]], [[TMP84]]
+; CHECK-NEXT:    [[TMP100:%.*]] = load <2 x i8>, ptr [[ARRAYIDX27_3]], align 1
+; CHECK-NEXT:    [[TMP103:%.*]] = zext <2 x i8> [[TMP100]] to <2 x i32>
+; CHECK-NEXT:    [[TMP69:%.*]] = sub <2 x i32> [[TMP98]], [[TMP103]]
 ; CHECK-NEXT:    [[TMP70:%.*]] = shl <2 x i32> [[TMP69]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP142:%.*]] = add <2 x i32> [[TMP70]], [[TMP65]]
-; CHECK-NEXT:    [[TMP115:%.*]] = add <2 x i32> [[TMP142]], [[TMP60]]
-; CHECK-NEXT:    [[TMP61:%.*]] = sub <2 x i32> [[TMP60]], [[TMP142]]
-; CHECK-NEXT:    [[TMP147:%.*]] = extractelement <2 x i32> [[TMP115]], i32 0
-; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <2 x i32> [[TMP115]], i32 1
-; CHECK-NEXT:    [[ADD55_3:%.*]] = add i32 [[TMP81]], [[TMP147]]
-; CHECK-NEXT:    [[TMP68:%.*]] = shufflevector <2 x i32> [[TMP61]], <2 x i32> [[TMP29]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[TMP71:%.*]] = shufflevector <2 x i32> [[TMP61]], <2 x i32> [[TMP29]], <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:    [[TMP78:%.*]] = sub <2 x i32> [[TMP68]], [[TMP71]]
-; CHECK-NEXT:    [[ADD94:%.*]] = add i32 [[ADD55_3]], [[ADD55_2]]
-; CHECK-NEXT:    [[SUB102_1:%.*]] = sub i32 [[ADD55_2]], [[ADD55_3]]
-; CHECK-NEXT:    [[SHR_I49_1:%.*]] = lshr i32 [[TMP77]], 15
+; CHECK-NEXT:    [[TMP74:%.*]] = add <2 x i32> [[TMP70]], [[TMP65]]
+; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <2 x i32> [[TMP90]], i32 0
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <2 x i32> [[TMP90]], i32 1
+; CHECK-NEXT:    [[ADD48_3:%.*]] = add i32 [[TMP71]], [[TMP78]]
+; CHECK-NEXT:    [[SUB51_3:%.*]] = sub i32 [[TMP78]], [[TMP71]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <2 x i32> [[TMP74]], i32 0
+; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <2 x i32> [[TMP74]], i32 1
+; CHECK-NEXT:    [[ADD55_3:%.*]] = add i32 [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[TMP107:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[ADD48_4:%.*]] = add i32 [[ADD55_3]], [[ADD48_3]]
+; CHECK-NEXT:    [[TMP113:%.*]] = shufflevector <2 x i32> [[TMP43]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT:    [[TMP122:%.*]] = insertelement <2 x i32> [[TMP113]], i32 [[ADD48_3]], i32 0
+; CHECK-NEXT:    [[TMP72:%.*]] = insertelement <2 x i32> [[TMP43]], i32 [[ADD55_3]], i32 0
+; CHECK-NEXT:    [[TMP123:%.*]] = sub <2 x i32> [[TMP122]], [[TMP72]]
+; CHECK-NEXT:    [[ADD55_4:%.*]] = add i32 [[TMP107]], [[SUB51_3]]
+; CHECK-NEXT:    [[TMP126:%.*]] = shufflevector <2 x i32> [[TMP44]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT:    [[TMP129:%.*]] = insertelement <2 x i32> [[TMP126]], i32 [[SUB51_3]], i32 0
+; CHECK-NEXT:    [[TMP130:%.*]] = insertelement <2 x i32> [[TMP44]], i32 [[TMP107]], i32 0
+; CHECK-NEXT:    [[TMP143:%.*]] = sub <2 x i32> [[TMP129]], [[TMP130]]
+; CHECK-NEXT:    [[ADD94:%.*]] = add i32 [[ADD48_4]], [[ADD48_2]]
+; CHECK-NEXT:    [[SUB102:%.*]] = sub i32 [[ADD48_2]], [[ADD48_4]]
+; CHECK-NEXT:    [[SHR_I_2:%.*]] = lshr i32 [[TMP77]], 15
+; CHECK-NEXT:    [[AND_I_2:%.*]] = and i32 [[SHR_I_2]], 65537
+; CHECK-NEXT:    [[MUL_I_2:%.*]] = mul i32 [[AND_I_2]], 65535
+; CHECK-NEXT:    [[SHR_I49_1:%.*]] = lshr i32 [[TMP84]], 15
 ; CHECK-NEXT:    [[AND_I50_1:%.*]] = and i32 [[SHR_I49_1]], 65537
 ; CHECK-NEXT:    [[ADD94_2:%.*]] = mul i32 [[AND_I50_1]], 65535
-; CHECK-NEXT:    [[SHR_I49_2:%.*]] = lshr i32 [[TMP146]], 15
+; CHECK-NEXT:    [[TMP144:%.*]] = extractelement <2 x i32> [[TMP123]], i32 0
+; CHECK-NEXT:    [[TMP145:%.*]] = extractelement <2 x i32> [[TMP123]], i32 1
+; CHECK-NEXT:    [[ADD94_4:%.*]] = add i32 [[TMP144]], [[TMP145]]
+; CHECK-NEXT:    [[TMP169:%.*]] = sub i32 [[TMP145]], [[TMP144]]
+; CHECK-NEXT:    [[SHR_I49_2:%.*]] = lshr i32 [[CONV_1]], 15
 ; CHECK-NEXT:    [[AND_I50_2:%.*]] = and i32 [[SHR_I49_2]], 65537
 ; CHECK-NEXT:    [[MUL_I51_2:%.*]] = mul i32 [[AND_I50_2]], 65535
-; CHECK-NEXT:    [[SUB59_3:%.*]] = extractelement <2 x i32> [[TMP78]], i32 0
-; CHECK-NEXT:    [[SUB59_2:%.*]] = extractelement <2 x i32> [[TMP78]], i32 1
-; CHECK-NEXT:    [[ADD94_5:%.*]] = add i32 [[SUB59_3]], [[SUB59_2]]
-; CHECK-NEXT:    [[SUB102_3:%.*]] = sub i32 [[SUB59_2]], [[SUB59_3]]
+; CHECK-NEXT:    [[TMP146:%.*]] = extractelement <2 x i32> [[TMP143]], i32 0
+; CHECK-NEXT:    [[TMP147:%.*]] = extractelement <2 x i32> [[TMP143]], i32 1
+; CHECK-NEXT:    [[ADD94_3:%.*]] = add i32 [[TMP146]], [[TMP147]]
+; CHECK-NEXT:    [[SUB102_3:%.*]] = sub i32 [[TMP147]], [[TMP146]]
 ; CHECK-NEXT:    [[SHR_I49_4:%.*]] = lshr i32 [[CONV1]], 15
 ; CHECK-NEXT:    [[AND_I50_4:%.*]] = and i32 [[SHR_I49_4]], 65537
 ; CHECK-NEXT:    [[MUL_I51_4:%.*]] = mul i32 [[AND_I50_4]], 65535
 ; CHECK-NEXT:    [[TMP66:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8]], align 1
 ; CHECK-NEXT:    [[TMP102:%.*]] = zext <2 x i8> [[TMP66]] to <2 x i32>
-; CHECK-NEXT:    [[TMP67:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[PIX2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP148:%.*]] = load <4 x i8>, ptr [[PIX2]], align 1
+; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <4 x i8> [[TMP148]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP85:%.*]] = zext <2 x i8> [[TMP67]] to <2 x i32>
-; CHECK-NEXT:    [[TMP73:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[TMP1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP74:%.*]] = zext <2 x i8> [[TMP73]] to <2 x i32>
-; CHECK-NEXT:    [[TMP89:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP90:%.*]] = zext <2 x i8> [[TMP89]] to <2 x i32>
-; CHECK-NEXT:    [[TMP87:%.*]] = sub <2 x i32> [[TMP74]], [[TMP90]]
+; CHECK-NEXT:    [[TMP149:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <4 x i8> [[TMP149]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP108:%.*]] = zext <2 x i8> [[TMP106]] to <2 x i32>
+; CHECK-NEXT:    [[TMP150:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
+; CHECK-NEXT:    [[TMP109:%.*]] = shufflevector <4 x i8> [[TMP150]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP89:%.*]] = zext <2 x i8> [[TMP109]] to <2 x i32>
+; CHECK-NEXT:    [[TMP87:%.*]] = sub <2 x i32> [[TMP108]], [[TMP89]]
 ; CHECK-NEXT:    [[TMP88:%.*]] = shl <2 x i32> [[TMP87]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP112:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX22]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP112:%.*]] = shufflevector <4 x i8> [[TMP148]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP120:%.*]] = zext <2 x i8> [[TMP112]] to <2 x i32>
-; CHECK-NEXT:    [[TMP94:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX25]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP148:%.*]] = zext <2 x i8> [[TMP94]] to <2 x i32>
-; CHECK-NEXT:    [[TMP80:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX27]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP149:%.*]] = zext <2 x i8> [[TMP80]] to <2 x i32>
-; CHECK-NEXT:    [[TMP95:%.*]] = sub <2 x i32> [[TMP148]], [[TMP149]]
+; CHECK-NEXT:    [[TMP94:%.*]] = shufflevector <4 x i8> [[TMP149]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP128:%.*]] = zext <2 x i8> [[TMP94]] to <2 x i32>
+; CHECK-NEXT:    [[TMP131:%.*]] = shufflevector <4 x i8> [[TMP150]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP132:%.*]] = zext <2 x i8> [[TMP131]] to <2 x i32>
+; CHECK-NEXT:    [[TMP95:%.*]] = sub <2 x i32> [[TMP128]], [[TMP132]]
 ; CHECK-NEXT:    [[TMP96:%.*]] = shl <2 x i32> [[TMP95]], <i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <2 x i32> [[TMP102]], i32 [[CONV33]], i32 1
 ; CHECK-NEXT:    [[TMP117:%.*]] = sub <2 x i32> [[TMP97]], [[TMP120]]
 ; CHECK-NEXT:    [[TMP105:%.*]] = add <2 x i32> [[TMP96]], [[TMP117]]
 ; CHECK-NEXT:    [[TMP86:%.*]] = insertelement <2 x i32> [[TMP102]], i32 [[CONV1]], i32 0
-; CHECK-NEXT:    [[TMP150:%.*]] = sub <2 x i32> [[TMP86]], [[TMP85]]
-; CHECK-NEXT:    [[TMP92:%.*]] = add <2 x i32> [[TMP88]], [[TMP150]]
+; CHECK-NEXT:    [[TMP119:%.*]] = sub <2 x i32> [[TMP86]], [[TMP85]]
+; CHECK-NEXT:    [[TMP92:%.*]] = add <2 x i32> [[TMP88]], [[TMP119]]
 ; CHECK-NEXT:    [[TMP93:%.*]] = shufflevector <2 x i32> [[TMP105]], <2 x i32> [[TMP92]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP101:%.*]] = add <2 x i32> [[TMP105]], [[TMP92]]
-; CHECK-NEXT:    [[TMP111:%.*]] = sub <2 x i32> [[TMP92]], [[TMP105]]
-; CHECK-NEXT:    [[TMP144:%.*]] = extractelement <2 x i32> [[TMP101]], i32 0
+; CHECK-NEXT:    [[TMP151:%.*]] = sub <2 x i32> [[TMP92]], [[TMP105]]
+; CHECK-NEXT:    [[TMP111:%.*]] = extractelement <2 x i32> [[TMP101]], i32 0
 ; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <2 x i32> [[TMP101]], i32 1
-; CHECK-NEXT:    [[ADD48:%.*]] = add i32 [[TMP99]], [[TMP144]]
-; CHECK-NEXT:    [[TMP118:%.*]] = shufflevector <2 x i32> [[TMP101]], <2 x i32> [[TMP29]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP119:%.*]] = shufflevector <2 x i32> [[TMP101]], <2 x i32> [[TMP29]], <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    [[TMP100:%.*]] = sub <2 x i32> [[TMP118]], [[TMP119]]
-; CHECK-NEXT:    [[TMP126:%.*]] = add <2 x i32> [[TMP118]], [[TMP119]]
-; CHECK-NEXT:    [[TMP127:%.*]] = shufflevector <2 x i32> [[TMP100]], <2 x i32> [[TMP126]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP103:%.*]] = shufflevector <2 x i32> [[TMP72]], <2 x i32> [[TMP111]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP104:%.*]] = shufflevector <2 x i32> [[TMP72]], <2 x i32> [[TMP111]], <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    [[TMP128:%.*]] = sub <2 x i32> [[TMP103]], [[TMP104]]
-; CHECK-NEXT:    [[TMP106:%.*]] = add <2 x i32> [[TMP103]], [[TMP104]]
-; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <2 x i32> [[TMP128]], <2 x i32> [[TMP106]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP108:%.*]] = extractelement <2 x i32> [[TMP111]], i32 0
-; CHECK-NEXT:    [[TMP109:%.*]] = extractelement <2 x i32> [[TMP111]], i32 1
-; CHECK-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+; CHECK-NEXT:    [[ADD55:%.*]] = add i32 [[TMP99]], [[TMP111]]
+; CHECK-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP111]], [[TMP99]]
+; CHECK-NEXT:    [[TMP153:%.*]] = extractelement <2 x i32> [[TMP151]], i32 0
+; CHECK-NEXT:    [[TMP157:%.*]] = extractelement <2 x i32> [[TMP151]], i32 1
+; CHECK-NEXT:    [[ADD78_1:%.*]] = add i32 [[TMP157]], [[TMP153]]
+; CHECK-NEXT:    [[SUB59:%.*]] = sub i32 [[TMP153]], [[TMP157]]
 ; CHECK-NEXT:    [[SHR_I59_1:%.*]] = lshr i32 [[TMP99]], 15
 ; CHECK-NEXT:    [[AND_I60_1:%.*]] = and i32 [[SHR_I59_1]], 65537
 ; CHECK-NEXT:    [[MUL_I61_1:%.*]] = mul i32 [[AND_I60_1]], 65535
-; CHECK-NEXT:    [[TMP134:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8_1]], align 1
-; CHECK-NEXT:    [[TMP110:%.*]] = zext <2 x i8> [[TMP134]] to <2 x i32>
-; CHECK-NEXT:    [[TMP214:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ADD_PTR644]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP113:%.*]] = zext <2 x i8> [[TMP214]] to <2 x i32>
-; CHECK-NEXT:    [[TMP114:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP143:%.*]] = zext <2 x i8> [[TMP114]] to <2 x i32>
-; CHECK-NEXT:    [[TMP116:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP215:%.*]] = zext <2 x i8> [[TMP116]] to <2 x i32>
-; CHECK-NEXT:    [[TMP124:%.*]] = sub <2 x i32> [[TMP143]], [[TMP215]]
+; CHECK-NEXT:    [[SHR_I59_4:%.*]] = lshr i32 [[TMP157]], 15
+; CHECK-NEXT:    [[AND_I60_4:%.*]] = and i32 [[SHR_I59_4]], 65537
+; CHECK-NEXT:    [[MUL_I61_4:%.*]] = mul i32 [[AND_I60_4]], 65535
+; CHECK-NEXT:    [[TMP104:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8_1]], align 1
+; CHECK-NEXT:    [[TMP110:%.*]] = zext <2 x i8> [[TMP104]] to <2 x i32>
+; CHECK-NEXT:    [[TMP158:%.*]] = load <4 x i8>, ptr [[ADD_PTR644]], align 1
+; CHECK-NEXT:    [[TMP114:%.*]] = shufflevector <4 x i8> [[TMP158]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP133:%.*]] = zext <2 x i8> [[TMP114]] to <2 x i32>
+; CHECK-NEXT:    [[TMP121:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
+; CHECK-NEXT:    [[TMP116:%.*]] = shufflevector <4 x i8> [[TMP121]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP115:%.*]] = zext <2 x i8> [[TMP116]] to <2 x i32>
+; CHECK-NEXT:    [[TMP159:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
+; CHECK-NEXT:    [[TMP118:%.*]] = shufflevector <4 x i8> [[TMP159]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP134:%.*]] = zext <2 x i8> [[TMP118]] to <2 x i32>
+; CHECK-NEXT:    [[TMP124:%.*]] = sub <2 x i32> [[TMP115]], [[TMP134]]
 ; CHECK-NEXT:    [[TMP125:%.*]] = shl <2 x i32> [[TMP124]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP216:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP121:%.*]] = zext <2 x i8> [[TMP216]] to <2 x i32>
-; CHECK-NEXT:    [[TMP122:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX13_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP123:%.*]] = zext <2 x i8> [[TMP122]] to <2 x i32>
-; CHECK-NEXT:    [[TMP157:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP158:%.*]] = zext <2 x i8> [[TMP157]] to <2 x i32>
-; CHECK-NEXT:    [[TMP135:%.*]] = sub <2 x i32> [[TMP123]], [[TMP158]]
+; CHECK-NEXT:    [[TMP127:%.*]] = shufflevector <4 x i8> [[TMP158]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP191:%.*]] = zext <2 x i8> [[TMP127]] to <2 x i32>
+; CHECK-NEXT:    [[TMP160:%.*]] = shufflevector <4 x i8> [[TMP121]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP161:%.*]] = zext <2 x i8> [[TMP160]] to <2 x i32>
+; CHECK-NEXT:    [[TMP171:%.*]] = shufflevector <4 x i8> [[TMP159]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP172:%.*]] = zext <2 x i8> [[TMP171]] to <2 x i32>
+; CHECK-NEXT:    [[TMP135:%.*]] = sub <2 x i32> [[TMP161]], [[TMP172]]
 ; CHECK-NEXT:    [[TMP136:%.*]] = shl <2 x i32> [[TMP135]], <i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP137:%.*]] = insertelement <2 x i32> [[TMP110]], i32 [[CONV33_1]], i32 1
-; CHECK-NEXT:    [[TMP129:%.*]] = sub <2 x i32> [[TMP137]], [[TMP121]]
-; CHECK-NEXT:    [[TMP130:%.*]] = add <2 x i32> [[TMP136]], [[TMP129]]
-; CHECK-NEXT:    [[TMP131:%.*]] = insertelement <2 x i32> [[TMP110]], i32 [[CONV_1]], i32 0
-; CHECK-NEXT:    [[TMP132:%.*]] = sub <2 x i32> [[TMP131]], [[TMP113]]
-; CHECK-NEXT:    [[TMP133:%.*]] = add <2 x i32> [[TMP125]], [[TMP132]]
-; CHECK-NEXT:    [[TMP156:%.*]] = add <2 x i32> [[TMP130]], [[TMP133]]
-; CHECK-NEXT:    [[TMP165:%.*]] = sub <2 x i32> [[TMP133]], [[TMP130]]
-; CHECK-NEXT:    [[TMP166:%.*]] = extractelement <2 x i32> [[TMP156]], i32 0
-; CHECK-NEXT:    [[SUB47_1:%.*]] = extractelement <2 x i32> [[TMP156]], i32 1
-; CHECK-NEXT:    [[ADD48_1:%.*]] = add i32 [[SUB47_1]], [[TMP166]]
-; CHECK-NEXT:    [[TMP138:%.*]] = shufflevector <2 x i32> [[TMP156]], <2 x i32> [[TMP61]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP139:%.*]] = shufflevector <2 x i32> [[TMP156]], <2 x i32> [[TMP61]], <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    [[TMP140:%.*]] = sub <2 x i32> [[TMP138]], [[TMP139]]
-; CHECK-NEXT:    [[TMP141:%.*]] = add <2 x i32> [[TMP138]], [[TMP139]]
-; CHECK-NEXT:    [[TMP155:%.*]] = shufflevector <2 x i32> [[TMP140]], <2 x i32> [[TMP141]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP171:%.*]] = extractelement <2 x i32> [[TMP155]], i32 1
-; CHECK-NEXT:    [[TMP217:%.*]] = extractelement <2 x i32> [[TMP127]], i32 1
-; CHECK-NEXT:    [[TMP145:%.*]] = shufflevector <2 x i32> [[TMP155]], <2 x i32> [[TMP127]], <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:    [[ADD94_1:%.*]] = add i32 [[TMP171]], [[TMP217]]
-; CHECK-NEXT:    [[TMP167:%.*]] = shufflevector <2 x i32> [[TMP115]], <2 x i32> [[TMP165]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP169:%.*]] = shufflevector <2 x i32> [[TMP115]], <2 x i32> [[TMP165]], <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    [[TMP170:%.*]] = sub <2 x i32> [[TMP167]], [[TMP169]]
-; CHECK-NEXT:    [[TMP189:%.*]] = add <2 x i32> [[TMP167]], [[TMP169]]
-; CHECK-NEXT:    [[TMP191:%.*]] = shufflevector <2 x i32> [[TMP170]], <2 x i32> [[TMP189]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP151:%.*]] = extractelement <2 x i32> [[TMP191]], i32 0
-; CHECK-NEXT:    [[TMP152:%.*]] = extractelement <2 x i32> [[TMP107]], i32 0
-; CHECK-NEXT:    [[TMP153:%.*]] = shufflevector <2 x i32> [[TMP191]], <2 x i32> [[TMP107]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[ADD94_6:%.*]] = add i32 [[TMP151]], [[TMP152]]
-; CHECK-NEXT:    [[TMP219:%.*]] = extractelement <2 x i32> [[TMP165]], i32 0
-; CHECK-NEXT:    [[TMP220:%.*]] = extractelement <2 x i32> [[TMP165]], i32 1
-; CHECK-NEXT:    [[SUB59_1:%.*]] = sub i32 [[TMP219]], [[TMP220]]
-; CHECK-NEXT:    [[SHR_I54_2:%.*]] = lshr i32 [[SUB47_1]], 15
-; CHECK-NEXT:    [[AND_I55_2:%.*]] = and i32 [[SHR_I54_2]], 65537
-; CHECK-NEXT:    [[MUL_I56_2:%.*]] = mul i32 [[AND_I55_2]], 65535
-; CHECK-NEXT:    [[SHR_I54_1:%.*]] = lshr i32 [[TMP220]], 15
+; CHECK-NEXT:    [[TMP173:%.*]] = sub <2 x i32> [[TMP137]], [[TMP191]]
+; CHECK-NEXT:    [[TMP174:%.*]] = add <2 x i32> [[TMP136]], [[TMP173]]
+; CHECK-NEXT:    [[TMP140:%.*]] = insertelement <2 x i32> [[TMP110]], i32 [[CONV_1]], i32 0
+; CHECK-NEXT:    [[TMP141:%.*]] = sub <2 x i32> [[TMP140]], [[TMP133]]
+; CHECK-NEXT:    [[TMP192:%.*]] = add <2 x i32> [[TMP125]], [[TMP141]]
+; CHECK-NEXT:    [[TMP156:%.*]] = add <2 x i32> [[TMP174]], [[TMP192]]
+; CHECK-NEXT:    [[TMP155:%.*]] = sub <2 x i32> [[TMP192]], [[TMP174]]
+; CHECK-NEXT:    [[TMP139:%.*]] = extractelement <2 x i32> [[TMP156]], i32 0
+; CHECK-NEXT:    [[TMP142:%.*]] = extractelement <2 x i32> [[TMP156]], i32 1
+; CHECK-NEXT:    [[ADD48_1:%.*]] = add i32 [[TMP142]], [[TMP139]]
+; CHECK-NEXT:    [[SUB45_1:%.*]] = sub i32 [[TMP139]], [[TMP142]]
+; CHECK-NEXT:    [[TMP138:%.*]] = extractelement <2 x i32> [[TMP155]], i32 0
+; CHECK-NEXT:    [[SUB47_1:%.*]] = extractelement <2 x i32> [[TMP155]], i32 1
+; CHECK-NEXT:    [[ADD94_1:%.*]] = add i32 [[SUB47_1]], [[TMP138]]
+; CHECK-NEXT:    [[SUB59_1:%.*]] = sub i32 [[TMP138]], [[SUB47_1]]
+; CHECK-NEXT:    [[SHR_I54:%.*]] = lshr i32 [[TMP142]], 15
+; CHECK-NEXT:    [[AND_I55:%.*]] = and i32 [[SHR_I54]], 65537
+; CHECK-NEXT:    [[MUL_I56:%.*]] = mul i32 [[AND_I55]], 65535
+; CHECK-NEXT:    [[SHR_I54_1:%.*]] = lshr i32 [[SUB47_1]], 15
 ; CHECK-NEXT:    [[AND_I55_1:%.*]] = and i32 [[SHR_I54_1]], 65537
 ; CHECK-NEXT:    [[MUL_I56_1:%.*]] = mul i32 [[AND_I55_1]], 65535
 ; CHECK-NEXT:    [[TMP194:%.*]] = lshr <2 x i32> [[TMP110]], <i32 15, i32 15>
 ; CHECK-NEXT:    [[TMP154:%.*]] = and <2 x i32> [[TMP194]], <i32 65537, i32 65537>
 ; CHECK-NEXT:    [[TMP195:%.*]] = mul <2 x i32> [[TMP154]], <i32 65535, i32 65535>
-; CHECK-NEXT:    [[ADD78:%.*]] = add i32 [[ADD48_1]], [[ADD48]]
-; CHECK-NEXT:    [[SUB86:%.*]] = sub i32 [[ADD48]], [[ADD48_1]]
+; CHECK-NEXT:    [[ADD78:%.*]] = add i32 [[ADD48_1]], [[ADD55]]
+; CHECK-NEXT:    [[SUB86:%.*]] = sub i32 [[ADD55]], [[ADD48_1]]
 ; CHECK-NEXT:    [[ADD103:%.*]] = add i32 [[ADD94]], [[ADD78]]
 ; CHECK-NEXT:    [[SUB104:%.*]] = sub i32 [[ADD78]], [[ADD94]]
-; CHECK-NEXT:    [[ADD105:%.*]] = add i32 [[SUB102_1]], [[SUB86]]
-; CHECK-NEXT:    [[SUB106:%.*]] = sub i32 [[SUB86]], [[SUB102_1]]
-; CHECK-NEXT:    [[ADD_I:%.*]] = add i32 [[ADD94_2]], [[ADD103]]
+; CHECK-NEXT:    [[ADD105:%.*]] = add i32 [[SUB102]], [[SUB86]]
+; CHECK-NEXT:    [[SUB106:%.*]] = sub i32 [[SUB86]], [[SUB102]]
+; CHECK-NEXT:    [[ADD_I:%.*]] = add i32 [[MUL_I_2]], [[ADD103]]
 ; CHECK-NEXT:    [[XOR_I:%.*]] = xor i32 [[ADD_I]], [[TMP77]]
-; CHECK-NEXT:    [[ADD_I62:%.*]] = add i32 [[MUL_I51_2]], [[ADD105]]
-; CHECK-NEXT:    [[XOR_I63:%.*]] = xor i32 [[ADD_I62]], [[TMP146]]
-; CHECK-NEXT:    [[ADD_I57:%.*]] = add i32 [[MUL_I56_2]], [[SUB104]]
-; CHECK-NEXT:    [[XOR_I58:%.*]] = xor i32 [[ADD_I57]], [[SUB47_1]]
-; CHECK-NEXT:    [[ADD_I63:%.*]] = add i32 [[MUL_I61_1]], [[SUB106]]
-; CHECK-NEXT:    [[XOR_I64:%.*]] = xor i32 [[ADD_I63]], [[TMP99]]
-; CHECK-NEXT:    [[ADD110:%.*]] = add i32 [[XOR_I63]], [[XOR_I]]
+; CHECK-NEXT:    [[ADD_I52:%.*]] = add i32 [[ADD94_2]], [[ADD105]]
+; CHECK-NEXT:    [[XOR_I53:%.*]] = xor i32 [[ADD_I52]], [[TMP84]]
+; CHECK-NEXT:    [[ADD_I57:%.*]] = add i32 [[MUL_I56]], [[SUB104]]
+; CHECK-NEXT:    [[XOR_I58:%.*]] = xor i32 [[ADD_I57]], [[TMP142]]
+; CHECK-NEXT:    [[ADD_I62:%.*]] = add i32 [[MUL_I61_1]], [[SUB106]]
+; CHECK-NEXT:    [[XOR_I63:%.*]] = xor i32 [[ADD_I62]], [[TMP99]]
+; CHECK-NEXT:    [[ADD110:%.*]] = add i32 [[XOR_I53]], [[XOR_I]]
 ; CHECK-NEXT:    [[ADD112:%.*]] = add i32 [[ADD110]], [[XOR_I58]]
-; CHECK-NEXT:    [[ADD113:%.*]] = add i32 [[ADD112]], [[XOR_I64]]
-; CHECK-NEXT:    [[TMP159:%.*]] = extractelement <2 x i32> [[TMP191]], i32 1
-; CHECK-NEXT:    [[TMP160:%.*]] = extractelement <2 x i32> [[TMP107]], i32 1
-; CHECK-NEXT:    [[TMP161:%.*]] = shufflevector <2 x i32> [[TMP191]], <2 x i32> [[TMP107]], <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:    [[ADD78_1:%.*]] = add i32 [[TMP159]], [[TMP160]]
-; CHECK-NEXT:    [[SUB104_1:%.*]] = sub i32 [[ADD78_1]], [[ADD94_1]]
-; CHECK-NEXT:    [[ADD_I57_1:%.*]] = add i32 [[MUL_I56_1]], [[SUB104_1]]
-; CHECK-NEXT:    [[XOR_I58_1:%.*]] = xor i32 [[ADD_I57_1]], [[TMP220]]
-; CHECK-NEXT:    [[TMP162:%.*]] = extractelement <2 x i32> [[TMP155]], i32 0
-; CHECK-NEXT:    [[TMP163:%.*]] = extractelement <2 x i32> [[TMP127]], i32 0
-; CHECK-NEXT:    [[TMP202:%.*]] = shufflevector <2 x i32> [[TMP155]], <2 x i32> [[TMP127]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[ADD78_2:%.*]] = add i32 [[TMP162]], [[TMP163]]
-; CHECK-NEXT:    [[TMP201:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_2]], i32 0
-; CHECK-NEXT:    [[TMP198:%.*]] = shufflevector <2 x i32> [[TMP201]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP199:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_6]], i32 0
-; CHECK-NEXT:    [[TMP200:%.*]] = shufflevector <2 x i32> [[TMP199]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP225:%.*]] = add <2 x i32> [[TMP198]], [[TMP200]]
-; CHECK-NEXT:    [[TMP226:%.*]] = sub <2 x i32> [[TMP198]], [[TMP200]]
-; CHECK-NEXT:    [[TMP227:%.*]] = shufflevector <2 x i32> [[TMP225]], <2 x i32> [[TMP226]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP172:%.*]] = insertelement <2 x i32> [[TMP111]], i32 [[CONV_1]], i32 0
-; CHECK-NEXT:    [[TMP173:%.*]] = lshr <2 x i32> [[TMP172]], <i32 15, i32 15>
-; CHECK-NEXT:    [[TMP174:%.*]] = and <2 x i32> [[TMP173]], <i32 65537, i32 65537>
-; CHECK-NEXT:    [[TMP175:%.*]] = mul <2 x i32> [[TMP174]], <i32 65535, i32 65535>
-; CHECK-NEXT:    [[TMP176:%.*]] = sub <2 x i32> [[TMP107]], [[TMP191]]
-; CHECK-NEXT:    [[TMP177:%.*]] = sub <2 x i32> [[TMP127]], [[TMP155]]
-; CHECK-NEXT:    [[TMP178:%.*]] = load <2 x i8>, ptr [[ADD_PTR_1]], align 1
-; CHECK-NEXT:    [[TMP179:%.*]] = zext <2 x i8> [[TMP178]] to <2 x i32>
-; CHECK-NEXT:    [[TMP180:%.*]] = lshr <2 x i32> [[TMP179]], <i32 15, i32 15>
-; CHECK-NEXT:    [[TMP181:%.*]] = and <2 x i32> [[TMP180]], <i32 65537, i32 65537>
-; CHECK-NEXT:    [[TMP182:%.*]] = mul <2 x i32> [[TMP181]], <i32 65535, i32 65535>
-; CHECK-NEXT:    [[TMP183:%.*]] = shufflevector <2 x i32> [[TMP177]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
-; CHECK-NEXT:    [[TMP184:%.*]] = insertelement <2 x i32> [[TMP183]], i32 [[ADD94_1]], i32 1
-; CHECK-NEXT:    [[TMP185:%.*]] = shufflevector <2 x i32> [[TMP176]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
-; CHECK-NEXT:    [[TMP186:%.*]] = insertelement <2 x i32> [[TMP185]], i32 [[ADD78_1]], i32 1
-; CHECK-NEXT:    [[TMP187:%.*]] = add <2 x i32> [[TMP184]], [[TMP186]]
-; CHECK-NEXT:    [[TMP188:%.*]] = add <2 x i32> [[TMP182]], [[TMP187]]
-; CHECK-NEXT:    [[TMP164:%.*]] = xor <2 x i32> [[TMP188]], [[TMP179]]
-; CHECK-NEXT:    [[TMP190:%.*]] = extractelement <2 x i32> [[TMP164]], i32 0
-; CHECK-NEXT:    [[ADD108_1:%.*]] = add i32 [[TMP190]], [[ADD113]]
-; CHECK-NEXT:    [[TMP212:%.*]] = extractelement <2 x i32> [[TMP164]], i32 1
-; CHECK-NEXT:    [[ADD110_1:%.*]] = add i32 [[ADD108_1]], [[TMP212]]
+; CHECK-NEXT:    [[ADD113:%.*]] = add i32 [[ADD112]], [[XOR_I63]]
+; CHECK-NEXT:    [[TMP193:%.*]] = load <2 x i8>, ptr [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[TMP196:%.*]] = zext <2 x i8> [[TMP193]] to <2 x i32>
+; CHECK-NEXT:    [[TMP198:%.*]] = insertelement <2 x i32> poison, i32 [[ADD55_2]], i32 0
+; CHECK-NEXT:    [[TMP199:%.*]] = shufflevector <2 x i32> [[TMP198]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP200:%.*]] = insertelement <2 x i32> poison, i32 [[ADD55_4]], i32 0
+; CHECK-NEXT:    [[TMP201:%.*]] = shufflevector <2 x i32> [[TMP200]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP202:%.*]] = sub <2 x i32> [[TMP199]], [[TMP201]]
+; CHECK-NEXT:    [[TMP203:%.*]] = add <2 x i32> [[TMP199]], [[TMP201]]
+; CHECK-NEXT:    [[TMP204:%.*]] = shufflevector <2 x i32> [[TMP202]], <2 x i32> [[TMP203]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP205:%.*]] = lshr <2 x i32> [[TMP196]], <i32 15, i32 15>
+; CHECK-NEXT:    [[TMP206:%.*]] = and <2 x i32> [[TMP205]], <i32 65537, i32 65537>
+; CHECK-NEXT:    [[TMP207:%.*]] = mul <2 x i32> [[TMP206]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP208:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_1]], i32 0
+; CHECK-NEXT:    [[TMP209:%.*]] = shufflevector <2 x i32> [[TMP208]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP210:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_1]], i32 0
+; CHECK-NEXT:    [[TMP211:%.*]] = shufflevector <2 x i32> [[TMP210]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP212:%.*]] = sub <2 x i32> [[TMP209]], [[TMP211]]
+; CHECK-NEXT:    [[TMP213:%.*]] = add <2 x i32> [[TMP209]], [[TMP211]]
+; CHECK-NEXT:    [[TMP214:%.*]] = shufflevector <2 x i32> [[TMP212]], <2 x i32> [[TMP213]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[ADD94_5:%.*]] = extractelement <2 x i32> [[TMP204]], i32 1
+; CHECK-NEXT:    [[ADD103_1:%.*]] = extractelement <2 x i32> [[TMP214]], i32 1
+; CHECK-NEXT:    [[SUB104_2:%.*]] = sub i32 [[ADD103_1]], [[ADD94_5]]
+; CHECK-NEXT:    [[TMP215:%.*]] = add <2 x i32> [[TMP204]], [[TMP214]]
+; CHECK-NEXT:    [[SUB102_1:%.*]] = extractelement <2 x i32> [[TMP204]], i32 0
+; CHECK-NEXT:    [[SUB104_1:%.*]] = extractelement <2 x i32> [[TMP214]], i32 0
+; CHECK-NEXT:    [[TMP216:%.*]] = shufflevector <2 x i32> [[TMP214]], <2 x i32> [[TMP204]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[SUB106_1:%.*]] = sub i32 [[SUB104_1]], [[SUB102_1]]
+; CHECK-NEXT:    [[TMP217:%.*]] = add <2 x i32> [[TMP207]], [[TMP215]]
+; CHECK-NEXT:    [[TMP218:%.*]] = xor <2 x i32> [[TMP217]], [[TMP196]]
+; CHECK-NEXT:    [[ADD_I57_1:%.*]] = add i32 [[MUL_I56_1]], [[SUB104_2]]
+; CHECK-NEXT:    [[XOR_I58_1:%.*]] = xor i32 [[ADD_I57_1]], [[SUB47_1]]
+; CHECK-NEXT:    [[ADD_I62_1:%.*]] = add i32 [[MUL_I61_4]], [[SUB106_1]]
+; CHECK-NEXT:    [[XOR_I63_1:%.*]] = xor i32 [[ADD_I62_1]], [[TMP157]]
+; CHECK-NEXT:    [[XOR_I53_1:%.*]] = extractelement <2 x i32> [[TMP218]], i32 0
+; CHECK-NEXT:    [[ADD108_1:%.*]] = add i32 [[XOR_I53_1]], [[ADD113]]
+; CHECK-NEXT:    [[XOR_I_1:%.*]] = extractelement <2 x i32> [[TMP218]], i32 1
+; CHECK-NEXT:    [[ADD110_1:%.*]] = add i32 [[ADD108_1]], [[XOR_I_1]]
 ; CHECK-NEXT:    [[ADD112_1:%.*]] = add i32 [[ADD110_1]], [[XOR_I58_1]]
-; CHECK-NEXT:    [[TMP192:%.*]] = add <2 x i32> [[TMP176]], [[TMP177]]
-; CHECK-NEXT:    [[TMP205:%.*]] = sub <2 x i32> [[TMP176]], [[TMP177]]
-; CHECK-NEXT:    [[TMP221:%.*]] = shufflevector <2 x i32> [[TMP192]], <2 x i32> [[TMP205]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP222:%.*]] = add <2 x i32> [[TMP175]], [[TMP221]]
-; CHECK-NEXT:    [[TMP196:%.*]] = xor <2 x i32> [[TMP222]], [[TMP172]]
-; CHECK-NEXT:    [[TMP197:%.*]] = extractelement <2 x i32> [[TMP196]], i32 1
-; CHECK-NEXT:    [[ADD113_1:%.*]] = add i32 [[ADD112_1]], [[TMP197]]
-; CHECK-NEXT:    [[TMP206:%.*]] = extractelement <2 x i32> [[TMP176]], i32 0
-; CHECK-NEXT:    [[TMP207:%.*]] = extractelement <2 x i32> [[TMP177]], i32 0
-; CHECK-NEXT:    [[TMP208:%.*]] = shufflevector <2 x i32> [[TMP177]], <2 x i32> [[TMP176]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SUB106_2:%.*]] = sub i32 [[TMP207]], [[TMP206]]
-; CHECK-NEXT:    [[TMP209:%.*]] = add <2 x i32> [[TMP195]], [[TMP227]]
-; CHECK-NEXT:    [[TMP168:%.*]] = xor <2 x i32> [[TMP209]], [[TMP110]]
-; CHECK-NEXT:    [[SHR_I59_2:%.*]] = lshr i32 [[TMP144]], 15
+; CHECK-NEXT:    [[ADD113_1:%.*]] = add i32 [[ADD112_1]], [[XOR_I63_1]]
+; CHECK-NEXT:    [[ADD78_2:%.*]] = add i32 [[SUB45_1]], [[SUB51]]
+; CHECK-NEXT:    [[TMP170:%.*]] = sub i32 [[SUB51]], [[SUB45_1]]
+; CHECK-NEXT:    [[TMP162:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_2]], i32 0
+; CHECK-NEXT:    [[TMP163:%.*]] = shufflevector <2 x i32> [[TMP162]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP164:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_4]], i32 0
+; CHECK-NEXT:    [[TMP165:%.*]] = shufflevector <2 x i32> [[TMP164]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP166:%.*]] = add <2 x i32> [[TMP163]], [[TMP165]]
+; CHECK-NEXT:    [[TMP167:%.*]] = sub <2 x i32> [[TMP163]], [[TMP165]]
+; CHECK-NEXT:    [[TMP168:%.*]] = shufflevector <2 x i32> [[TMP166]], <2 x i32> [[TMP167]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[ADD105_2:%.*]] = add i32 [[TMP169]], [[TMP170]]
+; CHECK-NEXT:    [[SUB106_2:%.*]] = sub i32 [[TMP170]], [[TMP169]]
+; CHECK-NEXT:    [[ADD_I52_2:%.*]] = add i32 [[MUL_I51_2]], [[ADD105_2]]
+; CHECK-NEXT:    [[XOR_I53_2:%.*]] = xor i32 [[ADD_I52_2]], [[CONV_1]]
+; CHECK-NEXT:    [[TMP197:%.*]] = add <2 x i32> [[TMP195]], [[TMP168]]
+; CHECK-NEXT:    [[TMP152:%.*]] = xor <2 x i32> [[TMP197]], [[TMP110]]
+; CHECK-NEXT:    [[SHR_I59_2:%.*]] = lshr i32 [[TMP111]], 15
 ; CHECK-NEXT:    [[AND_I60_2:%.*]] = and i32 [[SHR_I59_2]], 65537
 ; CHECK-NEXT:    [[MUL_I61_2:%.*]] = mul i32 [[AND_I60_2]], 65535
 ; CHECK-NEXT:    [[ADD_I62_2:%.*]] = add i32 [[MUL_I61_2]], [[SUB106_2]]
-; CHECK-NEXT:    [[XOR_I63_2:%.*]] = xor i32 [[ADD_I62_2]], [[TMP144]]
-; CHECK-NEXT:    [[XOR_I53_2:%.*]] = extractelement <2 x i32> [[TMP196]], i32 0
+; CHECK-NEXT:    [[XOR_I63_2:%.*]] = xor i32 [[ADD_I62_2]], [[TMP111]]
 ; CHECK-NEXT:    [[ADD108_2:%.*]] = add i32 [[XOR_I53_2]], [[ADD113_1]]
-; CHECK-NEXT:    [[TMP223:%.*]] = extractelement <2 x i32> [[TMP168]], i32 0
-; CHECK-NEXT:    [[ADD110_2:%.*]] = add i32 [[ADD108_2]], [[TMP223]]
-; CHECK-NEXT:    [[TMP224:%.*]] = extractelement <2 x i32> [[TMP168]], i32 1
-; CHECK-NEXT:    [[ADD112_2:%.*]] = add i32 [[ADD110_2]], [[TMP224]]
+; CHECK-NEXT:    [[TMP175:%.*]] = extractelement <2 x i32> [[TMP152]], i32 0
+; CHECK-NEXT:    [[ADD110_2:%.*]] = add i32 [[ADD108_2]], [[TMP175]]
+; CHECK-NEXT:    [[TMP176:%.*]] = extractelement <2 x i32> [[TMP152]], i32 1
+; CHECK-NEXT:    [[ADD112_2:%.*]] = add i32 [[ADD110_2]], [[TMP176]]
 ; CHECK-NEXT:    [[ADD113_2:%.*]] = add i32 [[ADD112_2]], [[XOR_I63_2]]
-; CHECK-NEXT:    [[ADD78_3:%.*]] = add i32 [[SUB59_1]], [[SUB60]]
-; CHECK-NEXT:    [[SUB86_3:%.*]] = sub i32 [[SUB60]], [[SUB59_1]]
-; CHECK-NEXT:    [[TMP203:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_3]], i32 0
-; CHECK-NEXT:    [[TMP204:%.*]] = shufflevector <2 x i32> [[TMP203]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP228:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_5]], i32 0
-; CHECK-NEXT:    [[TMP229:%.*]] = shufflevector <2 x i32> [[TMP228]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP230:%.*]] = add <2 x i32> [[TMP204]], [[TMP229]]
-; CHECK-NEXT:    [[TMP231:%.*]] = sub <2 x i32> [[TMP204]], [[TMP229]]
-; CHECK-NEXT:    [[TMP232:%.*]] = shufflevector <2 x i32> [[TMP230]], <2 x i32> [[TMP231]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[ADD78_3:%.*]] = add i32 [[SUB59_1]], [[SUB59]]
+; CHECK-NEXT:    [[SUB86_3:%.*]] = sub i32 [[SUB59]], [[SUB59_1]]
+; CHECK-NEXT:    [[TMP177:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_3]], i32 0
+; CHECK-NEXT:    [[TMP178:%.*]] = shufflevector <2 x i32> [[TMP177]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP179:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_3]], i32 0
+; CHECK-NEXT:    [[TMP180:%.*]] = shufflevector <2 x i32> [[TMP179]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP181:%.*]] = add <2 x i32> [[TMP178]], [[TMP180]]
+; CHECK-NEXT:    [[TMP182:%.*]] = sub <2 x i32> [[TMP178]], [[TMP180]]
+; CHECK-NEXT:    [[TMP183:%.*]] = shufflevector <2 x i32> [[TMP181]], <2 x i32> [[TMP182]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    [[ADD105_3:%.*]] = add i32 [[SUB102_3]], [[SUB86_3]]
 ; CHECK-NEXT:    [[SUB106_3:%.*]] = sub i32 [[SUB86_3]], [[SUB102_3]]
 ; CHECK-NEXT:    [[ADD_I52_3:%.*]] = add i32 [[MUL_I51_4]], [[ADD105_3]]
 ; CHECK-NEXT:    [[XOR_I53_3:%.*]] = xor i32 [[ADD_I52_3]], [[CONV1]]
-; CHECK-NEXT:    [[TMP210:%.*]] = lshr <2 x i32> [[TMP102]], <i32 15, i32 15>
-; CHECK-NEXT:    [[TMP211:%.*]] = and <2 x i32> [[TMP210]], <i32 65537, i32 65537>
-; CHECK-NEXT:    [[TMP233:%.*]] = mul <2 x i32> [[TMP211]], <i32 65535, i32 65535>
-; CHECK-NEXT:    [[TMP213:%.*]] = add <2 x i32> [[TMP233]], [[TMP232]]
-; CHECK-NEXT:    [[TMP234:%.*]] = xor <2 x i32> [[TMP213]], [[TMP102]]
+; CHECK-NEXT:    [[TMP184:%.*]] = lshr <2 x i32> [[TMP102]], <i32 15, i32 15>
+; CHECK-NEXT:    [[TMP185:%.*]] = and <2 x i32> [[TMP184]], <i32 65537, i32 65537>
+; CHECK-NEXT:    [[TMP186:%.*]] = mul <2 x i32> [[TMP185]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP187:%.*]] = add <2 x i32> [[TMP186]], [[TMP183]]
+; CHECK-NEXT:    [[TMP188:%.*]] = xor <2 x i32> [[TMP187]], [[TMP102]]
 ; CHECK-NEXT:    [[SHR_I59_3:%.*]] = lshr i32 [[CONV33]], 15
 ; CHECK-NEXT:    [[AND_I60_3:%.*]] = and i32 [[SHR_I59_3]], 65537
 ; CHECK-NEXT:    [[MUL_I61_3:%.*]] = mul i32 [[AND_I60_3]], 65535
 ; CHECK-NEXT:    [[ADD_I62_3:%.*]] = add i32 [[MUL_I61_3]], [[SUB106_3]]
 ; CHECK-NEXT:    [[XOR_I63_3:%.*]] = xor i32 [[ADD_I62_3]], [[CONV33]]
 ; CHECK-NEXT:    [[ADD108_3:%.*]] = add i32 [[XOR_I53_3]], [[ADD113_2]]
-; CHECK-NEXT:    [[TMP218:%.*]] = extractelement <2 x i32> [[TMP234]], i32 0
-; CHECK-NEXT:    [[ADD110_3:%.*]] = add i32 [[ADD108_3]], [[TMP218]]
-; CHECK-NEXT:    [[TMP193:%.*]] = extractelement <2 x i32> [[TMP234]], i32 1
-; CHECK-NEXT:    [[ADD112_3:%.*]] = add i32 [[ADD110_3]], [[TMP193]]
+; CHECK-NEXT:    [[TMP189:%.*]] = extractelement <2 x i32> [[TMP188]], i32 0
+; CHECK-NEXT:    [[ADD110_3:%.*]] = add i32 [[ADD108_3]], [[TMP189]]
+; CHECK-NEXT:    [[TMP190:%.*]] = extractelement <2 x i32> [[TMP188]], i32 1
+; CHECK-NEXT:    [[ADD112_3:%.*]] = add i32 [[ADD110_3]], [[TMP190]]
 ; CHECK-NEXT:    [[ADD113_3:%.*]] = add i32 [[ADD112_3]], [[XOR_I63_3]]
 ; CHECK-NEXT:    ret i32 [[ADD113_3]]
 ;
@@ -348,9 +346,6 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
 ; THR15-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
 ; THR15-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
-; THR15-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
-; THR15-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
-; THR15-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
 ; THR15-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
 ; THR15-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
 ; THR15-NEXT:    [[CONV33:%.*]] = zext i8 [[TMP1]] to i32
@@ -361,143 +356,151 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
 ; THR15-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
 ; THR15-NEXT:    [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
-; THR15-NEXT:    [[ARRAYIDX10_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 1
-; THR15-NEXT:    [[ARRAYIDX13_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 5
-; THR15-NEXT:    [[ARRAYIDX15_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 5
 ; THR15-NEXT:    [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
 ; THR15-NEXT:    [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
-; THR15-NEXT:    [[ADD31_1:%.*]] = zext i8 [[TMP5]] to i32
+; THR15-NEXT:    [[CONV33_1:%.*]] = zext i8 [[TMP5]] to i32
 ; THR15-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
 ; THR15-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
 ; THR15-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
 ; THR15-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
-; THR15-NEXT:    [[TMP19:%.*]] = load <2 x i8>, ptr [[ADD_PTR_1]], align 1
-; THR15-NEXT:    [[TMP66:%.*]] = zext <2 x i8> [[TMP19]] to <2 x i32>
-; THR15-NEXT:    [[TMP21:%.*]] = load <2 x i8>, ptr [[ADD_PTR64_1]], align 1
+; THR15-NEXT:    [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
+; THR15-NEXT:    [[TMP4:%.*]] = load <4 x i8>, ptr [[ADD_PTR_1]], align 1
+; THR15-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; THR15-NEXT:    [[TMP6:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; THR15-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; THR15-NEXT:    [[TMP20:%.*]] = zext <2 x i8> [[TMP19]] to <2 x i32>
+; THR15-NEXT:    [[TMP87:%.*]] = zext i8 [[TMP6]] to i32
+; THR15-NEXT:    [[TMP9:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_1]], align 1
+; THR15-NEXT:    [[TMP21:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; THR15-NEXT:    [[TMP22:%.*]] = zext <2 x i8> [[TMP21]] to <2 x i32>
-; THR15-NEXT:    [[TMP23:%.*]] = sub <2 x i32> [[TMP66]], [[TMP22]]
-; THR15-NEXT:    [[TMP24:%.*]] = load <2 x i8>, ptr [[ARRAYIDX3_2]], align 1
+; THR15-NEXT:    [[TMP23:%.*]] = sub <2 x i32> [[TMP20]], [[TMP22]]
+; THR15-NEXT:    [[TMP13:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
+; THR15-NEXT:    [[TMP24:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; THR15-NEXT:    [[TMP25:%.*]] = zext <2 x i8> [[TMP24]] to <2 x i32>
-; THR15-NEXT:    [[TMP26:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_2]], align 1
+; THR15-NEXT:    [[TMP16:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
+; THR15-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i8> [[TMP16]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; THR15-NEXT:    [[TMP27:%.*]] = zext <2 x i8> [[TMP26]] to <2 x i32>
 ; THR15-NEXT:    [[TMP28:%.*]] = sub <2 x i32> [[TMP25]], [[TMP27]]
 ; THR15-NEXT:    [[TMP29:%.*]] = shl <2 x i32> [[TMP28]], <i32 16, i32 16>
-; THR15-NEXT:    [[TMP30:%.*]] = add <2 x i32> [[TMP29]], [[TMP23]]
-; THR15-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 2
-; THR15-NEXT:    [[ARRAYIDX22_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 2
-; THR15-NEXT:    [[ARRAYIDX25_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 6
-; THR15-NEXT:    [[ARRAYIDX27_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 6
-; THR15-NEXT:    [[TMP31:%.*]] = load <2 x i8>, ptr [[ARRAYIDX20_2]], align 1
+; THR15-NEXT:    [[TMP59:%.*]] = add <2 x i32> [[TMP29]], [[TMP23]]
+; THR15-NEXT:    [[TMP31:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; THR15-NEXT:    [[TMP32:%.*]] = zext <2 x i8> [[TMP31]] to <2 x i32>
-; THR15-NEXT:    [[TMP33:%.*]] = load <2 x i8>, ptr [[ARRAYIDX22_2]], align 1
+; THR15-NEXT:    [[TMP86:%.*]] = zext i8 [[TMP7]] to i32
+; THR15-NEXT:    [[TMP33:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; THR15-NEXT:    [[TMP34:%.*]] = zext <2 x i8> [[TMP33]] to <2 x i32>
 ; THR15-NEXT:    [[TMP35:%.*]] = sub <2 x i32> [[TMP32]], [[TMP34]]
-; THR15-NEXT:    [[TMP36:%.*]] = load <2 x i8>, ptr [[ARRAYIDX25_2]], align 1
+; THR15-NEXT:    [[TMP36:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; THR15-NEXT:    [[TMP37:%.*]] = zext <2 x i8> [[TMP36]] to <2 x i32>
-; THR15-NEXT:    [[TMP38:%.*]] = load <2 x i8>, ptr [[ARRAYIDX27_2]], align 1
+; THR15-NEXT:    [[TMP38:%.*]] = shufflevector <4 x i8> [[TMP16]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; THR15-NEXT:    [[TMP39:%.*]] = zext <2 x i8> [[TMP38]] to <2 x i32>
 ; THR15-NEXT:    [[TMP40:%.*]] = sub <2 x i32> [[TMP37]], [[TMP39]]
 ; THR15-NEXT:    [[TMP41:%.*]] = shl <2 x i32> [[TMP40]], <i32 16, i32 16>
-; THR15-NEXT:    [[TMP42:%.*]] = add <2 x i32> [[TMP41]], [[TMP35]]
+; THR15-NEXT:    [[TMP76:%.*]] = add <2 x i32> [[TMP41]], [[TMP35]]
+; THR15-NEXT:    [[TMP30:%.*]] = add <2 x i32> [[TMP76]], [[TMP59]]
+; THR15-NEXT:    [[TMP42:%.*]] = sub <2 x i32> [[TMP59]], [[TMP76]]
 ; THR15-NEXT:    [[TMP43:%.*]] = extractelement <2 x i32> [[TMP30]], i32 0
 ; THR15-NEXT:    [[TMP44:%.*]] = extractelement <2 x i32> [[TMP30]], i32 1
 ; THR15-NEXT:    [[ADD44_2:%.*]] = add i32 [[TMP44]], [[TMP43]]
-; THR15-NEXT:    [[SUB45_2:%.*]] = sub i32 [[TMP43]], [[TMP44]]
-; THR15-NEXT:    [[TMP48:%.*]] = extractelement <2 x i32> [[TMP42]], i32 0
+; THR15-NEXT:    [[TMP45:%.*]] = extractelement <2 x i32> [[TMP42]], i32 0
 ; THR15-NEXT:    [[TMP46:%.*]] = extractelement <2 x i32> [[TMP42]], i32 1
-; THR15-NEXT:    [[ADD46_2:%.*]] = add i32 [[TMP46]], [[TMP48]]
-; THR15-NEXT:    [[SUB47_2:%.*]] = sub i32 [[TMP48]], [[TMP46]]
-; THR15-NEXT:    [[ADD48_2:%.*]] = add i32 [[ADD46_2]], [[ADD44_2]]
-; THR15-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr i8, ptr null, i64 4
+; THR15-NEXT:    [[ADD46_2:%.*]] = add i32 [[TMP46]], [[TMP45]]
 ; THR15-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
-; THR15-NEXT:    [[ARRAYIDX8_3:%.*]] = getelementptr i8, ptr null, i64 1
-; THR15-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr i8, ptr null, i64 1
-; THR15-NEXT:    [[ARRAYIDX15_3:%.*]] = getelementptr i8, ptr null, i64 5
-; THR15-NEXT:    [[TMP47:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP45:%.*]] = load i8, ptr null, align 1
+; THR15-NEXT:    [[TMP47:%.*]] = load <2 x i8>, ptr null, align 1
+; THR15-NEXT:    [[TMP48:%.*]] = load i8, ptr null, align 1
 ; THR15-NEXT:    [[TMP49:%.*]] = zext <2 x i8> [[TMP47]] to <2 x i32>
-; THR15-NEXT:    [[TMP87:%.*]] = zext i8 [[TMP45]] to i32
-; THR15-NEXT:    [[TMP50:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[CONV_3:%.*]] = zext i8 [[TMP48]] to i32
+; THR15-NEXT:    [[TMP50:%.*]] = load <2 x i8>, ptr null, align 1
 ; THR15-NEXT:    [[TMP51:%.*]] = zext <2 x i8> [[TMP50]] to <2 x i32>
 ; THR15-NEXT:    [[TMP52:%.*]] = sub <2 x i32> [[TMP49]], [[TMP51]]
-; THR15-NEXT:    [[TMP53:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_3]], i64 -4, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP53:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 4, <2 x i1> <i1 true, i1 true>, i32 2)
 ; THR15-NEXT:    [[TMP54:%.*]] = zext <2 x i8> [[TMP53]] to <2 x i32>
-; THR15-NEXT:    [[TMP55:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP77:%.*]] = shufflevector <2 x i32> [[TMP54]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; THR15-NEXT:    [[TMP55:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_3]], align 1
 ; THR15-NEXT:    [[TMP56:%.*]] = zext <2 x i8> [[TMP55]] to <2 x i32>
-; THR15-NEXT:    [[TMP57:%.*]] = sub <2 x i32> [[TMP54]], [[TMP56]]
+; THR15-NEXT:    [[TMP57:%.*]] = sub <2 x i32> [[TMP77]], [[TMP56]]
 ; THR15-NEXT:    [[TMP58:%.*]] = shl <2 x i32> [[TMP57]], <i32 16, i32 16>
-; THR15-NEXT:    [[TMP59:%.*]] = add <2 x i32> [[TMP58]], [[TMP52]]
-; THR15-NEXT:    [[TMP60:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX8_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP72:%.*]] = add <2 x i32> [[TMP58]], [[TMP52]]
+; THR15-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr i8, ptr null, i64 2
+; THR15-NEXT:    [[ARRAYIDX22_3:%.*]] = getelementptr i8, ptr null, i64 2
+; THR15-NEXT:    [[ARRAYIDX27_3:%.*]] = getelementptr i8, ptr null, i64 6
+; THR15-NEXT:    [[TMP60:%.*]] = load <2 x i8>, ptr [[ARRAYIDX20_3]], align 1
 ; THR15-NEXT:    [[TMP61:%.*]] = zext <2 x i8> [[TMP60]] to <2 x i32>
-; THR15-NEXT:    [[TMP62:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP62:%.*]] = load <2 x i8>, ptr [[ARRAYIDX22_3]], align 1
 ; THR15-NEXT:    [[TMP63:%.*]] = zext <2 x i8> [[TMP62]] to <2 x i32>
-; THR15-NEXT:    [[TMP80:%.*]] = sub <2 x i32> [[TMP61]], [[TMP63]]
+; THR15-NEXT:    [[TMP64:%.*]] = sub <2 x i32> [[TMP61]], [[TMP63]]
 ; THR15-NEXT:    [[TMP65:%.*]] = call <2 x i8> @llvm.masked.gather.v2i8.v2p0(<2 x ptr> zeroinitializer, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i8> poison)
-; THR15-NEXT:    [[TMP81:%.*]] = zext <2 x i8> [[TMP65]] to <2 x i32>
-; THR15-NEXT:    [[TMP67:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP66:%.*]] = zext <2 x i8> [[TMP65]] to <2 x i32>
+; THR15-NEXT:    [[TMP67:%.*]] = load <2 x i8>, ptr [[ARRAYIDX27_3]], align 1
 ; THR15-NEXT:    [[TMP68:%.*]] = zext <2 x i8> [[TMP67]] to <2 x i32>
-; THR15-NEXT:    [[TMP69:%.*]] = sub <2 x i32> [[TMP81]], [[TMP68]]
+; THR15-NEXT:    [[TMP69:%.*]] = sub <2 x i32> [[TMP66]], [[TMP68]]
 ; THR15-NEXT:    [[TMP70:%.*]] = shl <2 x i32> [[TMP69]], <i32 16, i32 16>
-; THR15-NEXT:    [[TMP71:%.*]] = add <2 x i32> [[TMP70]], [[TMP80]]
-; THR15-NEXT:    [[TMP72:%.*]] = add <2 x i32> [[TMP71]], [[TMP59]]
-; THR15-NEXT:    [[TMP73:%.*]] = sub <2 x i32> [[TMP59]], [[TMP71]]
+; THR15-NEXT:    [[TMP73:%.*]] = add <2 x i32> [[TMP70]], [[TMP64]]
 ; THR15-NEXT:    [[TMP74:%.*]] = extractelement <2 x i32> [[TMP72]], i32 0
 ; THR15-NEXT:    [[TMP75:%.*]] = extractelement <2 x i32> [[TMP72]], i32 1
 ; THR15-NEXT:    [[ADD48_3:%.*]] = add i32 [[TMP75]], [[TMP74]]
-; THR15-NEXT:    [[TMP76:%.*]] = insertelement <2 x i32> [[TMP72]], i32 [[ADD44_2]], i32 1
-; THR15-NEXT:    [[TMP77:%.*]] = shufflevector <2 x i32> [[TMP72]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
-; THR15-NEXT:    [[TMP78:%.*]] = insertelement <2 x i32> [[TMP77]], i32 [[ADD46_2]], i32 1
-; THR15-NEXT:    [[TMP79:%.*]] = sub <2 x i32> [[TMP76]], [[TMP78]]
-; THR15-NEXT:    [[TMP83:%.*]] = shufflevector <2 x i32> [[TMP73]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
-; THR15-NEXT:    [[TMP84:%.*]] = insertelement <2 x i32> [[TMP83]], i32 [[SUB47_2]], i32 1
-; THR15-NEXT:    [[TMP82:%.*]] = insertelement <2 x i32> [[TMP73]], i32 [[SUB45_2]], i32 1
-; THR15-NEXT:    [[TMP86:%.*]] = add <2 x i32> [[TMP84]], [[TMP82]]
+; THR15-NEXT:    [[SUB45_3:%.*]] = sub i32 [[TMP74]], [[TMP75]]
+; THR15-NEXT:    [[TMP80:%.*]] = extractelement <2 x i32> [[TMP73]], i32 0
+; THR15-NEXT:    [[TMP81:%.*]] = extractelement <2 x i32> [[TMP73]], i32 1
+; THR15-NEXT:    [[ADD55_3:%.*]] = add i32 [[TMP81]], [[TMP80]]
+; THR15-NEXT:    [[SUB47_3:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+; THR15-NEXT:    [[ADD48_4:%.*]] = add i32 [[ADD55_3]], [[ADD48_3]]
+; THR15-NEXT:    [[TMP78:%.*]] = shufflevector <2 x i32> [[TMP30]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
+; THR15-NEXT:    [[TMP71:%.*]] = insertelement <2 x i32> [[TMP78]], i32 [[ADD48_3]], i32 0
+; THR15-NEXT:    [[TMP83:%.*]] = insertelement <2 x i32> [[TMP30]], i32 [[ADD55_3]], i32 0
+; THR15-NEXT:    [[TMP79:%.*]] = sub <2 x i32> [[TMP71]], [[TMP83]]
+; THR15-NEXT:    [[ADD55_4:%.*]] = add i32 [[SUB47_3]], [[SUB45_3]]
+; THR15-NEXT:    [[TMP137:%.*]] = shufflevector <2 x i32> [[TMP42]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
+; THR15-NEXT:    [[TMP82:%.*]] = insertelement <2 x i32> [[TMP137]], i32 [[SUB45_3]], i32 0
+; THR15-NEXT:    [[TMP84:%.*]] = insertelement <2 x i32> [[TMP42]], i32 [[SUB47_3]], i32 0
 ; THR15-NEXT:    [[TMP85:%.*]] = sub <2 x i32> [[TMP82]], [[TMP84]]
-; THR15-NEXT:    [[ADD94:%.*]] = add i32 [[ADD48_3]], [[ADD48_2]]
-; THR15-NEXT:    [[SUB102:%.*]] = sub i32 [[ADD48_2]], [[ADD48_3]]
+; THR15-NEXT:    [[ADD94:%.*]] = add i32 [[ADD48_4]], [[ADD44_2]]
+; THR15-NEXT:    [[SUB102:%.*]] = sub i32 [[ADD44_2]], [[ADD48_4]]
+; THR15-NEXT:    [[SHR_I:%.*]] = lshr i32 [[CONV_3]], 15
+; THR15-NEXT:    [[AND_I:%.*]] = and i32 [[SHR_I]], 65537
+; THR15-NEXT:    [[MUL_I:%.*]] = mul i32 [[AND_I]], 65535
+; THR15-NEXT:    [[SHR_I49:%.*]] = lshr i32 [[TMP44]], 15
+; THR15-NEXT:    [[AND_I50:%.*]] = and i32 [[SHR_I49]], 65537
+; THR15-NEXT:    [[MUL_I51:%.*]] = mul i32 [[AND_I50]], 65535
+; THR15-NEXT:    [[ADD94_1:%.*]] = add i32 [[ADD55_4]], [[ADD46_2]]
+; THR15-NEXT:    [[SUB102_1:%.*]] = sub i32 [[ADD46_2]], [[ADD55_4]]
+; THR15-NEXT:    [[SHR_I_1:%.*]] = lshr i32 [[TMP86]], 15
+; THR15-NEXT:    [[AND_I_1:%.*]] = and i32 [[SHR_I_1]], 65537
+; THR15-NEXT:    [[MUL_I_1:%.*]] = mul i32 [[AND_I_1]], 65535
 ; THR15-NEXT:    [[SHR_I49_1:%.*]] = lshr i32 [[TMP87]], 15
 ; THR15-NEXT:    [[AND_I50_1:%.*]] = and i32 [[SHR_I49_1]], 65537
 ; THR15-NEXT:    [[MUL_I51_1:%.*]] = mul i32 [[AND_I50_1]], 65535
-; THR15-NEXT:    [[SHR_I49_2:%.*]] = lshr i32 [[ADD46_2]], 15
+; THR15-NEXT:    [[TMP88:%.*]] = extractelement <2 x i32> [[TMP79]], i32 0
+; THR15-NEXT:    [[TMP89:%.*]] = extractelement <2 x i32> [[TMP79]], i32 1
+; THR15-NEXT:    [[ADD94_2:%.*]] = add i32 [[TMP88]], [[TMP89]]
+; THR15-NEXT:    [[SUB102_2:%.*]] = sub i32 [[TMP89]], [[TMP88]]
+; THR15-NEXT:    [[SHR_I49_2:%.*]] = lshr i32 [[CONV_1]], 15
 ; THR15-NEXT:    [[AND_I50_2:%.*]] = and i32 [[SHR_I49_2]], 65537
 ; THR15-NEXT:    [[MUL_I51_2:%.*]] = mul i32 [[AND_I50_2]], 65535
-; THR15-NEXT:    [[TMP90:%.*]] = extractelement <2 x i32> [[TMP86]], i32 0
-; THR15-NEXT:    [[TMP91:%.*]] = extractelement <2 x i32> [[TMP86]], i32 1
-; THR15-NEXT:    [[ADD94_1:%.*]] = add i32 [[TMP90]], [[TMP91]]
+; THR15-NEXT:    [[TMP90:%.*]] = extractelement <2 x i32> [[TMP85]], i32 0
+; THR15-NEXT:    [[TMP91:%.*]] = extractelement <2 x i32> [[TMP85]], i32 1
+; THR15-NEXT:    [[ADD94_3:%.*]] = add i32 [[TMP90]], [[TMP91]]
 ; THR15-NEXT:    [[SUB102_3:%.*]] = sub i32 [[TMP91]], [[TMP90]]
-; THR15-NEXT:    [[TMP64:%.*]] = extractelement <2 x i32> [[TMP66]], i32 0
-; THR15-NEXT:    [[SHR_I49_3:%.*]] = lshr i32 [[TMP64]], 15
+; THR15-NEXT:    [[SHR_I49_3:%.*]] = lshr i32 [[CONV]], 15
 ; THR15-NEXT:    [[AND_I50_3:%.*]] = and i32 [[SHR_I49_3]], 65537
 ; THR15-NEXT:    [[MUL_I51_3:%.*]] = mul i32 [[AND_I50_3]], 65535
-; THR15-NEXT:    [[TMP88:%.*]] = extractelement <2 x i32> [[TMP79]], i32 0
-; THR15-NEXT:    [[TMP89:%.*]] = extractelement <2 x i32> [[TMP79]], i32 1
-; THR15-NEXT:    [[ADD94_2:%.*]] = add i32 [[TMP88]], [[TMP89]]
-; THR15-NEXT:    [[SUB102_2:%.*]] = sub i32 [[TMP89]], [[TMP88]]
-; THR15-NEXT:    [[SHR_I49_4:%.*]] = lshr i32 [[CONV_1]], 15
-; THR15-NEXT:    [[AND_I50_4:%.*]] = and i32 [[SHR_I49_4]], 65537
-; THR15-NEXT:    [[MUL_I51_4:%.*]] = mul i32 [[AND_I50_4]], 65535
-; THR15-NEXT:    [[SUB59_3:%.*]] = extractelement <2 x i32> [[TMP85]], i32 0
-; THR15-NEXT:    [[SUB59_2:%.*]] = extractelement <2 x i32> [[TMP85]], i32 1
-; THR15-NEXT:    [[ADD94_3:%.*]] = add i32 [[SUB59_3]], [[SUB59_2]]
-; THR15-NEXT:    [[SUB102_4:%.*]] = sub i32 [[SUB59_2]], [[SUB59_3]]
-; THR15-NEXT:    [[SHR_I49_5:%.*]] = lshr i32 [[CONV]], 15
-; THR15-NEXT:    [[AND_I50_5:%.*]] = and i32 [[SHR_I49_5]], 65537
-; THR15-NEXT:    [[MUL_I51_5:%.*]] = mul i32 [[AND_I50_5]], 65535
 ; THR15-NEXT:    [[TMP92:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8]], align 1
 ; THR15-NEXT:    [[TMP93:%.*]] = zext <2 x i8> [[TMP92]] to <2 x i32>
-; THR15-NEXT:    [[TMP94:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[PIX2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP143:%.*]] = load <4 x i8>, ptr [[PIX2]], align 1
+; THR15-NEXT:    [[TMP94:%.*]] = shufflevector <4 x i8> [[TMP143]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; THR15-NEXT:    [[TMP95:%.*]] = zext <2 x i8> [[TMP94]] to <2 x i32>
-; THR15-NEXT:    [[TMP96:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP146:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
+; THR15-NEXT:    [[TMP96:%.*]] = shufflevector <4 x i8> [[TMP146]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; THR15-NEXT:    [[TMP97:%.*]] = zext <2 x i8> [[TMP96]] to <2 x i32>
-; THR15-NEXT:    [[TMP98:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP147:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
+; THR15-NEXT:    [[TMP98:%.*]] = shufflevector <4 x i8> [[TMP147]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
 ; THR15-NEXT:    [[TMP99:%.*]] = zext <2 x i8> [[TMP98]] to <2 x i32>
 ; THR15-NEXT:    [[TMP100:%.*]] = sub <2 x i32> [[TMP97]], [[TMP99]]
 ; THR15-NEXT:    [[TMP101:%.*]] = shl <2 x i32> [[TMP100]], <i32 16, i32 16>
-; THR15-NEXT:    [[TMP102:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP102:%.*]] = shufflevector <4 x i8> [[TMP143]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; THR15-NEXT:    [[TMP103:%.*]] = zext <2 x i8> [[TMP102]] to <2 x i32>
-; THR15-NEXT:    [[TMP104:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX13]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP104:%.*]] = shufflevector <4 x i8> [[TMP146]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; THR15-NEXT:    [[TMP105:%.*]] = zext <2 x i8> [[TMP104]] to <2 x i32>
-; THR15-NEXT:    [[TMP106:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; THR15-NEXT:    [[TMP106:%.*]] = shufflevector <4 x i8> [[TMP147]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
 ; THR15-NEXT:    [[TMP107:%.*]] = zext <2 x i8> [[TMP106]] to <2 x i32>
 ; THR15-NEXT:    [[TMP108:%.*]] = sub <2 x i32> [[TMP105]], [[TMP107]]
 ; THR15-NEXT:    [[TMP109:%.*]] = shl <2 x i32> [[TMP108]], <i32 16, i32 16>
@@ -517,7 +520,7 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP121:%.*]] = extractelement <2 x i32> [[TMP118]], i32 0
 ; THR15-NEXT:    [[TMP122:%.*]] = extractelement <2 x i32> [[TMP118]], i32 1
 ; THR15-NEXT:    [[ADD55:%.*]] = add i32 [[TMP122]], [[TMP121]]
-; THR15-NEXT:    [[TMP171:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+; THR15-NEXT:    [[SUB59:%.*]] = sub i32 [[TMP121]], [[TMP122]]
 ; THR15-NEXT:    [[SHR_I59:%.*]] = lshr i32 [[TMP120]], 15
 ; THR15-NEXT:    [[AND_I60:%.*]] = and i32 [[SHR_I59]], 65537
 ; THR15-NEXT:    [[MUL_I61:%.*]] = mul i32 [[AND_I60]], 65535
@@ -525,93 +528,90 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[AND_I60_1:%.*]] = and i32 [[SHR_I59_1]], 65537
 ; THR15-NEXT:    [[MUL_I61_1:%.*]] = mul i32 [[AND_I60_1]], 65535
 ; THR15-NEXT:    [[TMP123:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8_1]], align 1
-; THR15-NEXT:    [[TMP136:%.*]] = zext <2 x i8> [[TMP123]] to <2 x i32>
-; THR15-NEXT:    [[TMP124:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ADD_PTR644]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP154:%.*]] = zext <2 x i8> [[TMP124]] to <2 x i32>
-; THR15-NEXT:    [[TMP126:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP125:%.*]] = zext <2 x i8> [[TMP126]] to <2 x i32>
-; THR15-NEXT:    [[TMP127:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP133:%.*]] = zext <2 x i8> [[TMP127]] to <2 x i32>
-; THR15-NEXT:    [[TMP139:%.*]] = sub <2 x i32> [[TMP125]], [[TMP133]]
-; THR15-NEXT:    [[TMP140:%.*]] = shl <2 x i32> [[TMP139]], <i32 16, i32 16>
-; THR15-NEXT:    [[TMP145:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP146:%.*]] = zext <2 x i8> [[TMP145]] to <2 x i32>
-; THR15-NEXT:    [[TMP147:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX13_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP128:%.*]] = zext <2 x i8> [[TMP147]] to <2 x i32>
-; THR15-NEXT:    [[TMP144:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP130:%.*]] = zext <2 x i8> [[TMP144]] to <2 x i32>
+; THR15-NEXT:    [[TMP124:%.*]] = zext <2 x i8> [[TMP123]] to <2 x i32>
+; THR15-NEXT:    [[TMP148:%.*]] = load <4 x i8>, ptr [[ADD_PTR644]], align 1
+; THR15-NEXT:    [[TMP125:%.*]] = shufflevector <4 x i8> [[TMP148]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; THR15-NEXT:    [[TMP126:%.*]] = zext <2 x i8> [[TMP125]] to <2 x i32>
+; THR15-NEXT:    [[TMP152:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
+; THR15-NEXT:    [[TMP127:%.*]] = shufflevector <4 x i8> [[TMP152]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; THR15-NEXT:    [[TMP128:%.*]] = zext <2 x i8> [[TMP127]] to <2 x i32>
+; THR15-NEXT:    [[TMP153:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
+; THR15-NEXT:    [[TMP129:%.*]] = shufflevector <4 x i8> [[TMP153]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; THR15-NEXT:    [[TMP130:%.*]] = zext <2 x i8> [[TMP129]] to <2 x i32>
 ; THR15-NEXT:    [[TMP131:%.*]] = sub <2 x i32> [[TMP128]], [[TMP130]]
 ; THR15-NEXT:    [[TMP132:%.*]] = shl <2 x i32> [[TMP131]], <i32 16, i32 16>
-; THR15-NEXT:    [[TMP148:%.*]] = insertelement <2 x i32> [[TMP136]], i32 [[ADD31_1]], i32 1
-; THR15-NEXT:    [[TMP152:%.*]] = sub <2 x i32> [[TMP148]], [[TMP146]]
-; THR15-NEXT:    [[TMP186:%.*]] = add <2 x i32> [[TMP132]], [[TMP152]]
-; THR15-NEXT:    [[TMP153:%.*]] = insertelement <2 x i32> [[TMP136]], i32 [[CONV_1]], i32 0
-; THR15-NEXT:    [[TMP155:%.*]] = sub <2 x i32> [[TMP153]], [[TMP154]]
-; THR15-NEXT:    [[TMP187:%.*]] = add <2 x i32> [[TMP140]], [[TMP155]]
-; THR15-NEXT:    [[TMP188:%.*]] = add <2 x i32> [[TMP186]], [[TMP187]]
-; THR15-NEXT:    [[TMP138:%.*]] = sub <2 x i32> [[TMP187]], [[TMP186]]
-; THR15-NEXT:    [[TMP189:%.*]] = extractelement <2 x i32> [[TMP188]], i32 0
-; THR15-NEXT:    [[SUB47_1:%.*]] = extractelement <2 x i32> [[TMP188]], i32 1
-; THR15-NEXT:    [[ADD48_1:%.*]] = add i32 [[SUB47_1]], [[TMP189]]
-; THR15-NEXT:    [[SUB51_1:%.*]] = sub i32 [[TMP189]], [[SUB47_1]]
-; THR15-NEXT:    [[TMP190:%.*]] = extractelement <2 x i32> [[TMP138]], i32 0
-; THR15-NEXT:    [[TMP191:%.*]] = extractelement <2 x i32> [[TMP138]], i32 1
-; THR15-NEXT:    [[ADD55_1:%.*]] = add i32 [[TMP191]], [[TMP190]]
-; THR15-NEXT:    [[TMP170:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-; THR15-NEXT:    [[SHR_I54_1:%.*]] = lshr i32 [[SUB47_1]], 15
+; THR15-NEXT:    [[TMP138:%.*]] = shufflevector <4 x i8> [[TMP148]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; THR15-NEXT:    [[TMP139:%.*]] = zext <2 x i8> [[TMP138]] to <2 x i32>
+; THR15-NEXT:    [[TMP154:%.*]] = shufflevector <4 x i8> [[TMP152]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; THR15-NEXT:    [[TMP155:%.*]] = zext <2 x i8> [[TMP154]] to <2 x i32>
+; THR15-NEXT:    [[TMP133:%.*]] = shufflevector <4 x i8> [[TMP153]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; THR15-NEXT:    [[TMP134:%.*]] = zext <2 x i8> [[TMP133]] to <2 x i32>
+; THR15-NEXT:    [[TMP135:%.*]] = sub <2 x i32> [[TMP155]], [[TMP134]]
+; THR15-NEXT:    [[TMP170:%.*]] = shl <2 x i32> [[TMP135]], <i32 16, i32 16>
+; THR15-NEXT:    [[TMP140:%.*]] = insertelement <2 x i32> [[TMP124]], i32 [[CONV33_1]], i32 1
+; THR15-NEXT:    [[TMP141:%.*]] = sub <2 x i32> [[TMP140]], [[TMP139]]
+; THR15-NEXT:    [[TMP171:%.*]] = add <2 x i32> [[TMP170]], [[TMP141]]
+; THR15-NEXT:    [[TMP186:%.*]] = insertelement <2 x i32> [[TMP124]], i32 [[CONV_1]], i32 0
+; THR15-NEXT:    [[TMP187:%.*]] = sub <2 x i32> [[TMP186]], [[TMP126]]
+; THR15-NEXT:    [[TMP142:%.*]] = add <2 x i32> [[TMP132]], [[TMP187]]
+; THR15-NEXT:    [[TMP136:%.*]] = add <2 x i32> [[TMP171]], [[TMP142]]
+; THR15-NEXT:    [[TMP149:%.*]] = sub <2 x i32> [[TMP142]], [[TMP171]]
+; THR15-NEXT:    [[TMP144:%.*]] = extractelement <2 x i32> [[TMP136]], i32 0
+; THR15-NEXT:    [[TMP145:%.*]] = extractelement <2 x i32> [[TMP136]], i32 1
+; THR15-NEXT:    [[ADD48_2:%.*]] = add i32 [[TMP145]], [[TMP144]]
+; THR15-NEXT:    [[SUB45_1:%.*]] = sub i32 [[TMP144]], [[TMP145]]
+; THR15-NEXT:    [[TMP150:%.*]] = extractelement <2 x i32> [[TMP149]], i32 0
+; THR15-NEXT:    [[TMP151:%.*]] = extractelement <2 x i32> [[TMP149]], i32 1
+; THR15-NEXT:    [[ADD48_1:%.*]] = add i32 [[TMP151]], [[TMP150]]
+; THR15-NEXT:    [[SUB51_1:%.*]] = sub i32 [[TMP150]], [[TMP151]]
+; THR15-NEXT:    [[SHR_I54:%.*]] = lshr i32 [[TMP145]], 15
+; THR15-NEXT:    [[AND_I55:%.*]] = and i32 [[SHR_I54]], 65537
+; THR15-NEXT:    [[MUL_I56:%.*]] = mul i32 [[AND_I55]], 65535
+; THR15-NEXT:    [[SHR_I54_1:%.*]] = lshr i32 [[TMP151]], 15
 ; THR15-NEXT:    [[AND_I55_1:%.*]] = and i32 [[SHR_I54_1]], 65537
 ; THR15-NEXT:    [[MUL_I56_1:%.*]] = mul i32 [[AND_I55_1]], 65535
-; THR15-NEXT:    [[TMP156:%.*]] = lshr <2 x i32> [[TMP136]], <i32 15, i32 15>
+; THR15-NEXT:    [[TMP156:%.*]] = lshr <2 x i32> [[TMP124]], <i32 15, i32 15>
 ; THR15-NEXT:    [[TMP157:%.*]] = and <2 x i32> [[TMP156]], <i32 65537, i32 65537>
 ; THR15-NEXT:    [[TMP158:%.*]] = mul <2 x i32> [[TMP157]], <i32 65535, i32 65535>
-; THR15-NEXT:    [[ADD78:%.*]] = add i32 [[ADD48_1]], [[ADD48]]
-; THR15-NEXT:    [[SUB86:%.*]] = sub i32 [[ADD48]], [[ADD48_1]]
+; THR15-NEXT:    [[ADD78:%.*]] = add i32 [[ADD48_2]], [[ADD48]]
+; THR15-NEXT:    [[SUB86:%.*]] = sub i32 [[ADD48]], [[ADD48_2]]
 ; THR15-NEXT:    [[ADD103:%.*]] = add i32 [[ADD94]], [[ADD78]]
 ; THR15-NEXT:    [[SUB104:%.*]] = sub i32 [[ADD78]], [[ADD94]]
 ; THR15-NEXT:    [[ADD105:%.*]] = add i32 [[SUB102]], [[SUB86]]
 ; THR15-NEXT:    [[SUB106:%.*]] = sub i32 [[SUB86]], [[SUB102]]
-; THR15-NEXT:    [[ADD_I:%.*]] = add i32 [[MUL_I51_1]], [[ADD103]]
-; THR15-NEXT:    [[XOR_I:%.*]] = xor i32 [[ADD_I]], [[TMP87]]
-; THR15-NEXT:    [[ADD_I52:%.*]] = add i32 [[MUL_I51_2]], [[ADD105]]
-; THR15-NEXT:    [[XOR_I53:%.*]] = xor i32 [[ADD_I52]], [[ADD46_2]]
-; THR15-NEXT:    [[ADD_I57:%.*]] = add i32 [[MUL_I56_1]], [[SUB104]]
-; THR15-NEXT:    [[XOR_I58:%.*]] = xor i32 [[ADD_I57]], [[SUB47_1]]
+; THR15-NEXT:    [[ADD_I:%.*]] = add i32 [[MUL_I]], [[ADD103]]
+; THR15-NEXT:    [[XOR_I:%.*]] = xor i32 [[ADD_I]], [[CONV_3]]
+; THR15-NEXT:    [[ADD_I52:%.*]] = add i32 [[MUL_I51]], [[ADD105]]
+; THR15-NEXT:    [[XOR_I53:%.*]] = xor i32 [[ADD_I52]], [[TMP44]]
+; THR15-NEXT:    [[ADD_I57:%.*]] = add i32 [[MUL_I56]], [[SUB104]]
+; THR15-NEXT:    [[XOR_I58:%.*]] = xor i32 [[ADD_I57]], [[TMP145]]
 ; THR15-NEXT:    [[ADD_I62:%.*]] = add i32 [[MUL_I61]], [[SUB106]]
 ; THR15-NEXT:    [[XOR_I63:%.*]] = xor i32 [[ADD_I62]], [[TMP120]]
 ; THR15-NEXT:    [[ADD110:%.*]] = add i32 [[XOR_I53]], [[XOR_I]]
 ; THR15-NEXT:    [[ADD112:%.*]] = add i32 [[ADD110]], [[XOR_I58]]
 ; THR15-NEXT:    [[ADD113:%.*]] = add i32 [[ADD112]], [[XOR_I63]]
-; THR15-NEXT:    [[ADD78_1:%.*]] = add i32 [[ADD55_1]], [[ADD55]]
-; THR15-NEXT:    [[SUB86_1:%.*]] = sub i32 [[ADD55]], [[ADD55_1]]
-; THR15-NEXT:    [[ADD105_1:%.*]] = add i32 [[SUB102_3]], [[SUB86_1]]
-; THR15-NEXT:    [[SUB106_1:%.*]] = sub i32 [[SUB86_1]], [[SUB102_3]]
-; THR15-NEXT:    [[ADD_I52_1:%.*]] = add i32 [[MUL_I51_3]], [[ADD105_1]]
-; THR15-NEXT:    [[XOR_I53_1:%.*]] = xor i32 [[ADD_I52_1]], [[TMP64]]
-; THR15-NEXT:    [[TMP129:%.*]] = shufflevector <2 x i32> [[TMP66]], <2 x i32> [[TMP138]], <2 x i32> <i32 1, i32 3>
-; THR15-NEXT:    [[TMP192:%.*]] = lshr <2 x i32> [[TMP129]], <i32 15, i32 15>
-; THR15-NEXT:    [[TMP193:%.*]] = and <2 x i32> [[TMP192]], <i32 65537, i32 65537>
-; THR15-NEXT:    [[TMP194:%.*]] = mul <2 x i32> [[TMP193]], <i32 65535, i32 65535>
-; THR15-NEXT:    [[TMP149:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_1]], i32 0
-; THR15-NEXT:    [[TMP134:%.*]] = shufflevector <2 x i32> [[TMP149]], <2 x i32> poison, <2 x i32> zeroinitializer
-; THR15-NEXT:    [[TMP135:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_1]], i32 0
-; THR15-NEXT:    [[TMP195:%.*]] = shufflevector <2 x i32> [[TMP135]], <2 x i32> poison, <2 x i32> zeroinitializer
-; THR15-NEXT:    [[TMP137:%.*]] = add <2 x i32> [[TMP134]], [[TMP195]]
-; THR15-NEXT:    [[TMP150:%.*]] = sub <2 x i32> [[TMP134]], [[TMP195]]
-; THR15-NEXT:    [[TMP151:%.*]] = shufflevector <2 x i32> [[TMP137]], <2 x i32> [[TMP150]], <2 x i32> <i32 0, i32 3>
-; THR15-NEXT:    [[TMP159:%.*]] = add <2 x i32> [[TMP194]], [[TMP151]]
-; THR15-NEXT:    [[TMP141:%.*]] = xor <2 x i32> [[TMP159]], [[TMP129]]
+; THR15-NEXT:    [[ADD78_1:%.*]] = add i32 [[ADD48_1]], [[ADD55]]
+; THR15-NEXT:    [[SUB86_1:%.*]] = sub i32 [[ADD55]], [[ADD48_1]]
+; THR15-NEXT:    [[ADD103_1:%.*]] = add i32 [[ADD94_1]], [[ADD78_1]]
+; THR15-NEXT:    [[SUB104_1:%.*]] = sub i32 [[ADD78_1]], [[ADD94_1]]
+; THR15-NEXT:    [[ADD105_1:%.*]] = add i32 [[SUB102_1]], [[SUB86_1]]
+; THR15-NEXT:    [[SUB106_1:%.*]] = sub i32 [[SUB86_1]], [[SUB102_1]]
+; THR15-NEXT:    [[ADD_I_1:%.*]] = add i32 [[MUL_I_1]], [[ADD103_1]]
+; THR15-NEXT:    [[XOR_I_1:%.*]] = xor i32 [[ADD_I_1]], [[TMP86]]
+; THR15-NEXT:    [[ADD_I52_1:%.*]] = add i32 [[MUL_I51_1]], [[ADD105_1]]
+; THR15-NEXT:    [[XOR_I53_1:%.*]] = xor i32 [[ADD_I52_1]], [[TMP87]]
+; THR15-NEXT:    [[ADD_I57_1:%.*]] = add i32 [[MUL_I56_1]], [[SUB104_1]]
+; THR15-NEXT:    [[XOR_I58_1:%.*]] = xor i32 [[ADD_I57_1]], [[TMP151]]
 ; THR15-NEXT:    [[ADD_I62_1:%.*]] = add i32 [[MUL_I61_1]], [[SUB106_1]]
 ; THR15-NEXT:    [[XOR_I63_1:%.*]] = xor i32 [[ADD_I62_1]], [[TMP122]]
 ; THR15-NEXT:    [[ADD108_1:%.*]] = add i32 [[XOR_I53_1]], [[ADD113]]
-; THR15-NEXT:    [[TMP142:%.*]] = extractelement <2 x i32> [[TMP141]], i32 0
-; THR15-NEXT:    [[ADD110_1:%.*]] = add i32 [[ADD108_1]], [[TMP142]]
-; THR15-NEXT:    [[TMP143:%.*]] = extractelement <2 x i32> [[TMP141]], i32 1
-; THR15-NEXT:    [[ADD112_1:%.*]] = add i32 [[ADD110_1]], [[TMP143]]
+; THR15-NEXT:    [[ADD110_1:%.*]] = add i32 [[ADD108_1]], [[XOR_I_1]]
+; THR15-NEXT:    [[ADD112_1:%.*]] = add i32 [[ADD110_1]], [[XOR_I58_1]]
 ; THR15-NEXT:    [[ADD113_1:%.*]] = add i32 [[ADD112_1]], [[XOR_I63_1]]
-; THR15-NEXT:    [[ADD78_2:%.*]] = add i32 [[SUB51_1]], [[SUB51]]
-; THR15-NEXT:    [[SUB86_2:%.*]] = sub i32 [[SUB51]], [[SUB51_1]]
-; THR15-NEXT:    [[TMP196:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_2]], i32 0
-; THR15-NEXT:    [[TMP160:%.*]] = shufflevector <2 x i32> [[TMP196]], <2 x i32> poison, <2 x i32> zeroinitializer
+; THR15-NEXT:    [[ADD78_2:%.*]] = add i32 [[SUB45_1]], [[SUB51]]
+; THR15-NEXT:    [[SUB86_2:%.*]] = sub i32 [[SUB51]], [[SUB45_1]]
+; THR15-NEXT:    [[TMP159:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_2]], i32 0
+; THR15-NEXT:    [[TMP160:%.*]] = shufflevector <2 x i32> [[TMP159]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; THR15-NEXT:    [[TMP161:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_2]], i32 0
 ; THR15-NEXT:    [[TMP162:%.*]] = shufflevector <2 x i32> [[TMP161]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; THR15-NEXT:    [[TMP163:%.*]] = add <2 x i32> [[TMP160]], [[TMP162]]
@@ -619,10 +619,10 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP165:%.*]] = shufflevector <2 x i32> [[TMP163]], <2 x i32> [[TMP164]], <2 x i32> <i32 0, i32 3>
 ; THR15-NEXT:    [[ADD105_2:%.*]] = add i32 [[SUB102_2]], [[SUB86_2]]
 ; THR15-NEXT:    [[SUB106_2:%.*]] = sub i32 [[SUB86_2]], [[SUB102_2]]
-; THR15-NEXT:    [[ADD_I52_2:%.*]] = add i32 [[MUL_I51_4]], [[ADD105_2]]
+; THR15-NEXT:    [[ADD_I52_2:%.*]] = add i32 [[MUL_I51_2]], [[ADD105_2]]
 ; THR15-NEXT:    [[XOR_I53_2:%.*]] = xor i32 [[ADD_I52_2]], [[CONV_1]]
 ; THR15-NEXT:    [[TMP166:%.*]] = add <2 x i32> [[TMP158]], [[TMP165]]
-; THR15-NEXT:    [[TMP167:%.*]] = xor <2 x i32> [[TMP166]], [[TMP136]]
+; THR15-NEXT:    [[TMP167:%.*]] = xor <2 x i32> [[TMP166]], [[TMP124]]
 ; THR15-NEXT:    [[SHR_I59_2:%.*]] = lshr i32 [[TMP119]], 15
 ; THR15-NEXT:    [[AND_I60_2:%.*]] = and i32 [[SHR_I59_2]], 65537
 ; THR15-NEXT:    [[MUL_I61_2:%.*]] = mul i32 [[AND_I60_2]], 65535
@@ -634,8 +634,8 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP169:%.*]] = extractelement <2 x i32> [[TMP167]], i32 1
 ; THR15-NEXT:    [[ADD112_2:%.*]] = add i32 [[ADD110_2]], [[TMP169]]
 ; THR15-NEXT:    [[ADD113_2:%.*]] = add i32 [[ADD112_2]], [[XOR_I63_2]]
-; THR15-NEXT:    [[ADD78_3:%.*]] = add i32 [[TMP170]], [[TMP171]]
-; THR15-NEXT:    [[SUB86_3:%.*]] = sub i32 [[TMP171]], [[TMP170]]
+; THR15-NEXT:    [[ADD78_3:%.*]] = add i32 [[SUB51_1]], [[SUB59]]
+; THR15-NEXT:    [[SUB86_3:%.*]] = sub i32 [[SUB59]], [[SUB51_1]]
 ; THR15-NEXT:    [[TMP172:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_3]], i32 0
 ; THR15-NEXT:    [[TMP173:%.*]] = shufflevector <2 x i32> [[TMP172]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; THR15-NEXT:    [[TMP174:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_3]], i32 0
@@ -643,9 +643,9 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP176:%.*]] = add <2 x i32> [[TMP173]], [[TMP175]]
 ; THR15-NEXT:    [[TMP177:%.*]] = sub <2 x i32> [[TMP173]], [[TMP175]]
 ; THR15-NEXT:    [[TMP178:%.*]] = shufflevector <2 x i32> [[TMP176]], <2 x i32> [[TMP177]], <2 x i32> <i32 0, i32 3>
-; THR15-NEXT:    [[ADD105_3:%.*]] = add i32 [[SUB102_4]], [[SUB86_3]]
-; THR15-NEXT:    [[SUB106_3:%.*]] = sub i32 [[SUB86_3]], [[SUB102_4]]
-; THR15-NEXT:    [[ADD_I52_3:%.*]] = add i32 [[MUL_I51_5]], [[ADD105_3]]
+; THR15-NEXT:    [[ADD105_3:%.*]] = add i32 [[SUB102_3]], [[SUB86_3]]
+; THR15-NEXT:    [[SUB106_3:%.*]] = sub i32 [[SUB86_3]], [[SUB102_3]]
+; THR15-NEXT:    [[ADD_I52_3:%.*]] = add i32 [[MUL_I51_3]], [[ADD105_3]]
 ; THR15-NEXT:    [[XOR_I53_3:%.*]] = xor i32 [[ADD_I52_3]], [[CONV]]
 ; THR15-NEXT:    [[TMP179:%.*]] = lshr <2 x i32> [[TMP93]], <i32 15, i32 15>
 ; THR15-NEXT:    [[TMP180:%.*]] = and <2 x i32> [[TMP179]], <i32 65537, i32 65537>
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads.ll
index 54eb564768318b8b69184c3f49dd102ee853bcb5..ce26bd3b89392d475bc7730684c73d2dbbb0cf21 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/segmented-loads.ll
@@ -6,8 +6,9 @@
 
 define void @test() {
 ; CHECK-LABEL: @test(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.experimental.vp.strided.load.v4f64.p0.i64(ptr align 8 @src, i64 16, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 4)
-; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.experimental.vp.strided.load.v4f64.p0.i64(ptr align 8 getelementptr inbounds ([8 x double], ptr @src, i32 0, i64 1), i64 16, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 4)
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x double>, ptr @src, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[TMP4]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[TMP4]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fsub fast <4 x double> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    store <4 x double> [[TMP3]], ptr @dst, align 8
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
index 413aedefe9b6ada1481b90ef3e8c335a12680e68..9cdb7c228e382780471996f7c627e916767e5c5e 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
@@ -4,7 +4,7 @@
 @bar = external global [4 x [4 x i32]], align 4
 @dct_luma = external global [4 x [4 x i32]], align 4
 
-define void @foo() local_unnamed_addr {
+define void @foo() {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD277:%.*]] = add nsw i32 undef, undef
@@ -18,7 +18,7 @@ define void @foo() local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = ashr <4 x i32> [[TMP5]], <i32 6, i32 6, i32 6, i32 6>
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr [[ARRAYIDX372]], align 4
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ;
 entry:
   %add277 = add nsw i32 undef, undef
@@ -42,5 +42,5 @@ entry:
   %shr.i.3 = ashr i32 %sub355.3, 6
   %arrayidx372.3 = getelementptr inbounds [4 x [4 x i32]], ptr @dct_luma, i64 0, i64 3, i64 3
   store i32 %shr.i.3, ptr %arrayidx372.3, align 4
-  unreachable
+  ret void
 }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
index 837b4ffa68260a093b03f41091fd1efc8912687f..1f33676c19c14825fb5a682e886af6528d49bbf5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
@@ -32,7 +32,7 @@ define void @main() {
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr undef, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> <double 2.000000e-01, double 3.000000e-01>, [[TMP1]]
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[AGG_TMP101211_SROA_0_0_IDX]], align 8
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ; CHECK:       cond.true63.us:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       for.body42.lr.ph.us:
@@ -81,7 +81,7 @@ cond.false66.us:
   store double %add4.i246.us, ptr %agg.tmp99208.sroa.1.8.idx388, align 8
   store double %mul.i.i.us, ptr %agg.tmp101211.sroa.0.0.idx, align 8
   store double %mul2.i.i.us, ptr %agg.tmp101211.sroa.1.8.idx390, align 8
-  unreachable
+  ret void
 
 cond.true63.us:
   unreachable
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
index aba45fe6bd5199663d9c876ad2541c477f2a3632..28bab3276c47dd2abc2cf57dea7d6d3630928506 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
@@ -41,7 +41,7 @@ define void @foo(double %i) {
 ; CHECK-NEXT:    [[OP_RDX1:%.*]] = select i1 [[OP_RDX]], i32 [[TMP31]], i32 32000
 ; CHECK-NEXT:    [[I163:%.*]] = fcmp ogt double [[I118]], 0.000000e+00
 ; CHECK-NEXT:    [[I164:%.*]] = icmp slt i32 0, [[OP_RDX1]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ; CHECK:       bb58:
 ; CHECK-NEXT:    ret void
 ;
@@ -134,7 +134,7 @@ bb115:
   %i159 = select i1 %i158, i32 %i157, i32 %i152
   %i163 = fcmp ogt double %i118, 0.000000e+00
   %i164 = icmp slt i32 0, %i159
-  unreachable
+  ret void
 
 bb58:
   ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insertelements-with-reused-indices.ll b/llvm/test/Transforms/SLPVectorizer/X86/insertelements-with-reused-indices.ll
index e46c3b94383ffcb35ab7e74a91538333969c5f1e..a8160d77619c53cce60a7931cfaa5fac47b6fd11 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insertelements-with-reused-indices.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insertelements-with-reused-indices.ll
@@ -10,7 +10,7 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float 0.000000e+00, i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub float 0.000000e+00, 0.000000e+00
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP4]], i32 0
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ;
 entry:
   %0 = fsub float 0.000000e+00, 0.000000e+00
@@ -23,5 +23,5 @@ entry:
   %7 = fsub float 0.000000e+00, %0
   %8 = fsub float %7, %4
   %9 = insertelement <2 x float> zeroinitializer, float %8, i32 0
-  unreachable
+  ret void
 }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/partail.ll b/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
index 0b9ed47ce0f17808599bb4073f25671f6b143025..20ad09a63282688f9fc3c865910f7076aaa3a63c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
@@ -37,7 +37,7 @@ define void @get_block(i32 %y_pos) local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[COND_I5_I_3:%.*]] = select i1 [[CMP_I4_I_3]], i32 [[COND_I_I_3]], i32 undef
 ; CHECK-NEXT:    [[IDXPROM30_3:%.*]] = sext i32 [[COND_I5_I_3]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX31_3:%.*]] = getelementptr inbounds ptr, ptr undef, i64 [[IDXPROM30_3]]
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %land.lhs.true
@@ -75,5 +75,5 @@ if.end:                                           ; preds = %land.lhs.true
   %cond.i5.i.3 = select i1 %cmp.i4.i.3, i32 %cond.i.i.3, i32 undef
   %idxprom30.3 = sext i32 %cond.i5.i.3 to i64
   %arrayidx31.3 = getelementptr inbounds ptr, ptr undef, i64 %idxprom30.3
-  unreachable
+  ret void
 }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
index b79ba458ef70635a9415605bed9ba0b063ea1805..042fb88297e2ba3e61a139255aa3cc0c516f0731 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
@@ -31,7 +31,7 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv()
 ; CHECK-NEXT:    [[TMP16:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v2i8(<16 x i8> [[TMP14]], <2 x i8> [[TMP15]], i64 2)
 ; CHECK-NEXT:    [[TMP17:%.*]] = and <16 x i8> [[TMP16]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
 ; CHECK-NEXT:    store <16 x i8> [[TMP17]], ptr undef, align 1
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ; CHECK:       if.end50.i:
 ; CHECK-NEXT:    ret void
 ;
@@ -119,7 +119,7 @@ if.then22.i:                                      ; preds = %entry
   %conv.15.i.i = and i8 %15, 1
   %arrayidx.i.i7.15.i.i = getelementptr inbounds %"struct.std::array", ptr undef, i64 0, i32 0, i64 15
   store i8 %conv.15.i.i, ptr %arrayidx.i.i7.15.i.i, align 1
-  unreachable
+  ret void
 
 if.end50.i:                                       ; preds = %entry
   ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
index d9496a3e3e343a75264def0cf24d3c65fb30ce01..1c4de256468c7ee9450dc00215f9636ec4b3d2dd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
 
-define dso_local void @rftbsub(ptr %a) local_unnamed_addr #0 {
+define void @rftbsub(ptr %a) {
 ; CHECK-LABEL: @rftbsub(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 2
@@ -17,7 +17,7 @@ define dso_local void @rftbsub(ptr %a) local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[SUB22]], i32 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    store <2 x double> [[TMP5]], ptr [[ARRAYIDX6]], align 8
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ;
 entry:
   %arrayidx6 = getelementptr inbounds double, ptr %a, i64 2
@@ -33,5 +33,5 @@ entry:
   store double %sub25, ptr %arrayidx6, align 8
   %sub29 = fsub double %2, %sub22
   store double %sub29, ptr %arrayidx12, align 8
-  unreachable
+  ret void
 }
diff --git a/llvm/test/Transforms/SLPVectorizer/scalarization-overhead.ll b/llvm/test/Transforms/SLPVectorizer/scalarization-overhead.ll
index 9f6b285f1ab90aff543f43dd8bd693d2326f7524..91c0c7ab42e7797136bc371584d3e4bd8d06e6aa 100644
--- a/llvm/test/Transforms/SLPVectorizer/scalarization-overhead.ll
+++ b/llvm/test/Transforms/SLPVectorizer/scalarization-overhead.ll
@@ -4,7 +4,7 @@
 
 ; Crash Test case reported on D134605
 
-define i16 @D134605() {
+define void @D134605() {
 ; CHECK-LABEL: @D134605(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX81:%.*]] = getelementptr inbounds [32 x i16], ptr poison, i16 0, i16 3
@@ -16,7 +16,7 @@ define i16 @D134605() {
 ; CHECK-NEXT:    [[OP_RDX:%.*]] = add i16 [[TMP3]], poison
 ; CHECK-NEXT:    [[REASS_MUL24:%.*]] = shl i16 [[OP_RDX]], 2
 ; CHECK-NEXT:    [[CALL:%.*]] = call i16 @check_i16(i16 noundef 1, i16 noundef [[REASS_MUL24]], i16 noundef 5120)
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ;
 entry:
   %arrayidx81 = getelementptr inbounds [32 x i16], ptr poison, i16 0, i16 3
@@ -38,7 +38,7 @@ entry:
   %add15935 = add i16 %add15534, poison
   %reass.mul24 = shl i16 %add15935, 2
   %call = call i16 @check_i16(i16 noundef 1, i16 noundef %reass.mul24, i16 noundef 5120)
-  unreachable
+  ret void
 }
 declare i16 @check_i16(i16, i16, i16)
 
@@ -53,7 +53,7 @@ define void @PR58054() {
 ; CHECK-NEXT:    [[VAL7:%.*]] = add i64 [[VAL]], [[VAL5]]
 ; CHECK-NEXT:    [[VAL8:%.*]] = sitofp i64 [[VAL7]] to double
 ; CHECK-NEXT:    call void @wibble(i32 poison, double [[VAL8]], i64 poison)
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    ret void
 ;
   %val = add i64 poison, poison
   %val2 = add i64 poison, poison
@@ -63,6 +63,6 @@ define void @PR58054() {
   %val7 = add i64 %val, %val5
   %val8 = sitofp i64 %val7 to double
   call void @wibble(i32 poison, double %val8, i64 poison)
-  unreachable
+  ret void
 }
 declare void @wibble(i32, double, i64)
diff --git a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
index e6a255a4b8f08617f8727cb6b6ce39ee7f45b713..fe0b48028a3b62c8b918cbac6452d952d35ae713 100644
--- a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
+++ b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
@@ -275,3 +275,33 @@ F:
   %gep2 = getelementptr nuw i8, ptr %p, i64 1
   ret ptr %gep2
 }
+
+define i1 @hoist_icmp_flags_preserve(i1 %C, i32 %x, i32 %y) {
+; CHECK-LABEL: @hoist_icmp_flags_preserve(
+; CHECK-NEXT:  common.ret:
+; CHECK-NEXT:    [[Z1:%.*]] = icmp samesign ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[Z1]]
+;
+  br i1 %C, label %T, label %F
+T:
+  %z1 = icmp samesign ult i32 %x, %y
+  ret i1 %z1
+F:
+  %z2 = icmp samesign ult i32 %x, %y
+  ret i1 %z2
+}
+
+define i1 @hoist_icmp_flags_drop(i1 %C, i32 %x, i32 %y) {
+; CHECK-LABEL: @hoist_icmp_flags_drop(
+; CHECK-NEXT:  common.ret:
+; CHECK-NEXT:    [[Z1:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[Z1]]
+;
+  br i1 %C, label %T, label %F
+T:
+  %z1 = icmp ult i32 %x, %y
+  ret i1 %z1
+F:
+  %z2 = icmp samesign ult i32 %x, %y
+  ret i1 %z2
+}
diff --git a/llvm/test/Verifier/denormal-fp-math.ll b/llvm/test/Verifier/denormal-fp-math.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f79f28a36694452cd46f991d1cf32ecdb6d3f95d
--- /dev/null
+++ b/llvm/test/Verifier/denormal-fp-math.ll
@@ -0,0 +1,20 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck --implicit-check-not="invalid value" %s
+
+define float @test_denormal_fp_math_valid() "denormal-fp-math"="ieee,ieee" {
+  ret float 1.0
+}
+
+; CHECK: invalid value for 'denormal-fp-math' attribute: foo,ieee
+define float @test_denormal_fp_math_invalid1() "denormal-fp-math"="foo,ieee" {
+  ret float 1.0
+}
+
+; CHECK: invalid value for 'denormal-fp-math' attribute: ieee,ieee,ieee
+define float @test_denormal_fp_math_invalid2() "denormal-fp-math"="ieee,ieee,ieee" {
+  ret float 1.0
+}
+
+; CHECK: invalid value for 'denormal-fp-math-f32' attribute: foo,ieee
+define float @test_denormal_fp_math_f32_invalid() "denormal-fp-math-f32"="foo,ieee" {
+  ret float 1.0
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
index 3ac63d070933ddb2f4b2ca8c2bc497590b95b973..5c9af3bb44da2ae885c4a1d11e67c8f6600349b6 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
@@ -9,43 +9,38 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct
 ; CHECK-NEXT:    .local .align 8 .b8 __local_depot0[32];
 ; CHECK-NEXT:    .reg .b32 %SP;
 ; CHECK-NEXT:    .reg .b32 %SPL;
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<17>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<13>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    mov.u32 %SPL, __local_depot0;
 ; CHECK-NEXT:    cvta.local.u32 %SP, %SPL;
 ; CHECK-NEXT:    ld.param.u32 %r1, [caller_St8x4_param_1];
-; CHECK-NEXT:    add.u32 %r3, %SPL, 0;
 ; CHECK-NEXT:    ld.param.u64 %rd1, [caller_St8x4_param_0+24];
+; CHECK-NEXT:    st.u64 [%SP+24], %rd1;
 ; CHECK-NEXT:    ld.param.u64 %rd2, [caller_St8x4_param_0+16];
+; CHECK-NEXT:    st.u64 [%SP+16], %rd2;
 ; CHECK-NEXT:    ld.param.u64 %rd3, [caller_St8x4_param_0+8];
+; CHECK-NEXT:    st.u64 [%SP+8], %rd3;
 ; CHECK-NEXT:    ld.param.u64 %rd4, [caller_St8x4_param_0];
-; CHECK-NEXT:    st.local.u64 [%r3], %rd4;
-; CHECK-NEXT:    st.local.u64 [%r3+8], %rd3;
-; CHECK-NEXT:    st.local.u64 [%r3+16], %rd2;
-; CHECK-NEXT:    st.local.u64 [%r3+24], %rd1;
-; CHECK-NEXT:    ld.u64 %rd5, [%SP+8];
-; CHECK-NEXT:    ld.u64 %rd6, [%SP+0];
-; CHECK-NEXT:    ld.u64 %rd7, [%SP+24];
-; CHECK-NEXT:    ld.u64 %rd8, [%SP+16];
+; CHECK-NEXT:    st.u64 [%SP+0], %rd4;
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[32];
-; CHECK-NEXT:    st.param.v2.b64 [param0+0], {%rd6, %rd5};
-; CHECK-NEXT:    st.param.v2.b64 [param0+16], {%rd8, %rd7};
+; CHECK-NEXT:    st.param.v2.b64 [param0+0], {%rd4, %rd3};
+; CHECK-NEXT:    st.param.v2.b64 [param0+16], {%rd2, %rd1};
 ; CHECK-NEXT:    .param .align 16 .b8 retval0[32];
 ; CHECK-NEXT:    call.uni (retval0),
 ; CHECK-NEXT:    callee_St8x4,
 ; CHECK-NEXT:    (
 ; CHECK-NEXT:    param0
 ; CHECK-NEXT:    );
-; CHECK-NEXT:    ld.param.v2.b64 {%rd9, %rd10}, [retval0+0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd11, %rd12}, [retval0+16];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [retval0+0];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd7, %rd8}, [retval0+16];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    st.u64 [%r1], %rd9;
-; CHECK-NEXT:    st.u64 [%r1+8], %rd10;
-; CHECK-NEXT:    st.u64 [%r1+16], %rd11;
-; CHECK-NEXT:    st.u64 [%r1+24], %rd12;
+; CHECK-NEXT:    st.u64 [%r1], %rd5;
+; CHECK-NEXT:    st.u64 [%r1+8], %rd6;
+; CHECK-NEXT:    st.u64 [%r1+16], %rd7;
+; CHECK-NEXT:    st.u64 [%r1+24], %rd8;
 ; CHECK-NEXT:    ret;
   %call = tail call fastcc [4 x i64] @callee_St8x4(ptr noundef nonnull byval(%struct.St8x4) align 8 %in) #2
   %.fca.0.extract = extractvalue [4 x i64] %call, 0
diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.v6.wasm32 b/llvm/test/tools/llvm-cov/Inputs/binary-formats.v6.wasm32
new file mode 100755
index 0000000000000000000000000000000000000000..5a606d5a2f69fd150690bd17ebb9d922cc17c383
Binary files /dev/null and b/llvm/test/tools/llvm-cov/Inputs/binary-formats.v6.wasm32 differ
diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.wasm.proftext b/llvm/test/tools/llvm-cov/Inputs/binary-formats.wasm.proftext
new file mode 100644
index 0000000000000000000000000000000000000000..20fc3816c2255a503ed59897ba1f54a7a5e3c5a0
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/binary-formats.wasm.proftext
@@ -0,0 +1,4 @@
+__main_argc_argv
+0x0
+1
+100
diff --git a/llvm/test/tools/llvm-cov/binary-formats.c b/llvm/test/tools/llvm-cov/binary-formats.c
index a5bfc012860ec3ae4f14756f0605f914c5b92ba6..bb61b288cfc62e620768ff6df350100ad5622e5a 100644
--- a/llvm/test/tools/llvm-cov/binary-formats.c
+++ b/llvm/test/tools/llvm-cov/binary-formats.c
@@ -10,4 +10,11 @@ int main(int argc, const char *argv[]) {}
 // RUN: llvm-cov show %S/Inputs/binary-formats.v3.macho64l -instr-profile %t.profdata -path-equivalence=/tmp,%S %s | FileCheck %s
 // RUN: llvm-cov show %S/Inputs/binary-formats.v6.linux64l -instr-profile %t.profdata -path-equivalence=/tmp,%S %s | FileCheck %s
 
+// RUN: llvm-profdata merge %S/Inputs/binary-formats.wasm.proftext -o %t.wasm.profdata
+// NOTE: The wasm binary is built with the following command:
+//   clang -target wasm32-unknown-wasi %s -o %S/Inputs/binary-formats.v6.wasm32 \
+//     -mllvm -enable-name-compression=false \
+//     -fprofile-instr-generate -fcoverage-mapping -lwasi-emulated-getpid -lwasi-emulated-mman
+// RUN: llvm-cov show %S/Inputs/binary-formats.v6.wasm32 -instr-profile %t.wasm.profdata -path-equivalence=/tmp,%S %s | FileCheck %s
+
 // RUN: llvm-cov export %S/Inputs/binary-formats.macho64l -instr-profile %t.profdata | FileCheck %S/Inputs/binary-formats.canonical.json
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-forwarding.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-forwarding.s
new file mode 100644
index 0000000000000000000000000000000000000000..4de37f960005202c68616b08088cbf6251459167
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-forwarding.s
@@ -0,0 +1,1421 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=neoverse-v1 -mattr=+sve --instruction-info=0 --resource-pressure=0 --timeline --timeline-max-iterations=2 < %s | FileCheck %s
+
+# LLVM-MCA-BEGIN madd
+mul  x0, x0, x0
+madd x0, x1, x2, x0
+madd x0, x1, x2, x0
+madd x0, x0, x0, x0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN smaddl
+mul    x0, x0, x0
+smaddl x0, w1, w2, x0
+smaddl x0, w1, w2, x0
+smaddl x0, w0, w0, x0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN fmadd
+fadd  d0, d0, d0
+fmadd d0, d1, d2, d0
+fmul  d0, d0, d0
+fmadd d0, d1, d2, d0
+fmadd d0, d1, d2, d0
+fmadd d0, d0, d1, d2
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN saba
+mul  v0.4s, v0.4s, v0.4s
+saba v0.4s, v1.4s, v2.4s
+saba v0.4s, v1.4s, v2.4s
+saba v0.4s, v0.4s, v1.4s
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN sadalp
+mul    v0.4s, v0.4s, v0.4s
+sadalp v0.2d, v1.4s
+sadalp v0.2d, v1.4s
+sadalp v0.2d, v0.4s
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN sdot
+mul  v0.4s, v0.4s,  v0.4s
+sdot v0.4s, v1.16b, v2.16b
+sdot v0.4s, v1.16b, v2.16b
+sdot v0.4s, v0.16b, v1.16b
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN smmla
+mul   v0.4s, v0.4s,  v0.4s
+smmla v0.4s, v1.16b, v2.16b
+smmla v0.4s, v1.16b, v2.16b
+smmla v0.4s, v0.16b, v1.16b
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN mla
+mul v0.4s, v0.4s, v0.4s
+mla v0.4s, v1.4s, v2.4s
+mla v0.4s, v1.4s, v2.4s
+mla v0.4s, v0.4s, v1.4s
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN smlal2
+mul    v0.4s, v0.4s, v0.4s
+smlal2 v0.4s, v1.8h, v2.8h
+smlal2 v0.4s, v1.8h, v2.8h
+smlal2 v0.4s, v0.8h, v1.8h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN ssra
+mul  v0.4s, v0.4s, v0.4s
+ssra v0.2d, v1.2d, #1
+ssra v0.2d, v1.2d, #1
+ssra v0.2d, v0.2d, #1
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN fcmla
+fmul  v0.4s, v0.4s, v0.4s
+fcmla v0.2d, v1.2d, v2.2d, #90
+fcmla v0.2d, v1.2d, v2.2d, #90
+fcmla v0.2d, v0.2d, v1.2d, #90
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN fmla
+fmul v0.2d, v0.2d, v0.2d
+fmla v0.2d, v1.2d, v2.2d
+fadd v0.2d, v0.2d, v0.2d
+fmla v0.2d, v1.2d, v2.2d
+fmla v0.2d, v1.2d, v2.2d
+fmla v0.2d, v0.2d, v1.2d
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN fmlal
+fmul  v0.2d, v0.2d, v0.2d
+fmlal v0.4s, v1.4h, v2.4h
+fadd  v0.2d, v0.2d, v0.2d
+fmlal v0.4s, v1.4h, v2.4h
+fmlal v0.4s, v1.4h, v2.4h
+fmlal v0.4s, v0.4h, v1.4h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN bfdot
+fmul  v0.2d, v0.2d, v0.2d
+bfdot v0.4s, v1.8h, v2.8h
+bfdot v0.4s, v1.8h, v2.8h
+bfdot v0.4s, v0.8h, v1.8h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN bfmmla
+fmul   v0.2d, v0.2d, v0.2d
+bfmmla v0.4s, v1.8h, v2.8h
+bfmmla v0.4s, v1.8h, v2.8h
+bfmmla v0.4s, v0.8h, v1.8h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN bfmlalb
+fmul    v0.2d, v0.2d, v0.2d
+bfmlalb v0.4s, v1.8h, v2.8h
+bfmlalb v0.4s, v1.8h, v2.8h
+bfmlalb v0.4s, v0.8h, v1.8h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN crc32cb
+mul    w0, w0, w0
+crc32cb w0, w0, w1
+crc32cb w0, w0, w1
+crc32cb w0, w0, w0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z sdot.s
+mul z0.d, p0/m, z0.d, z0.d
+sdot z0.s, z1.b, z2.b
+sdot z0.s, z1.b, z2.b
+sdot z0.s, z0.b, z1.b
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z sudot
+mul z0.d, p0/m, z0.d, z0.d
+sdot z0.s, z1.b, z2.b[1]
+sdot z0.s, z1.b, z2.b[1]
+sdot z0.s, z0.b, z1.b[1]
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z sdot.d
+mul z0.d, p0/m, z0.d, z0.d
+sdot z0.d, z1.h, z2.h
+sdot z0.d, z1.h, z2.h
+sdot z0.d, z0.h, z1.h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z smmla
+mul z0.d, p0/m, z0.d, z0.d
+smmla z0.s, z1.b, z2.b
+smmla z0.s, z1.b, z2.b
+smmla z0.s, z0.b, z1.b
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z mla.d
+mul z0.d, p0/m, z0.d, z0.d
+mla z0.d, p0/m, z1.d, z2.d
+mla z0.d, p0/m, z1.d, z2.d
+mla z0.d, p0/m, z0.d, z1.d
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z mad.d
+mul z0.d, p0/m, z0.d, z0.d
+mad z0.d, p0/m, z1.d, z2.d
+mad z0.d, p0/m, z1.d, z2.d
+mad z0.d, p0/m, z0.d, z1.d
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z msb.d
+mul z0.d, p0/m, z0.d, z0.d
+msb z0.d, p0/m, z1.d, z2.d
+msb z0.d, p0/m, z1.d, z2.d
+msb z0.d, p0/m, z0.d, z1.d
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z fcmla ZPmZZ
+fmul  z0.d, z0.d, z0.d
+fcmla z0.d, p0/m, z1.d, z2.d, 90
+fcmla z0.d, p0/m, z1.d, z2.d, 90
+fcmla z0.d, p0/m, z0.d, z1.d, 90
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z fcmla ZZZI
+fmul  z0.d, z0.d, z0.d
+fcmla z0.s, z1.s, z2.s[1], 90
+fcmla z0.s, z1.s, z2.s[1], 90
+fcmla z0.s, z0.s, z1.s[1], 90
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z fmla ZPmZZ
+fmul z0.d, z0.d, z0.d
+fmla z0.d, p0/m, z1.d, z2.d
+fmla z0.d, p0/m, z1.d, z2.d
+fmla z0.d, p0/m, z0.d, z1.d
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z fmla ZZZI
+fmul z0.d, z0.d, z0.d
+fmla z0.d, z1.d, z2.d[1]
+fmla z0.d, z1.d, z2.d[1]
+fmla z0.d, z0.d, z1.d[1]
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z bfdot
+fmul  z0.d, z0.d, z0.d
+bfdot z0.s, z1.h, z2.h
+bfdot z0.s, z1.h, z2.h
+bfdot z0.s, z0.h, z1.h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN Z bfmmla
+fmul   z0.d, z0.d, z0.d
+bfmmla z0.s, z1.h, z2.h
+bfmmla z0.s, z1.h, z2.h
+bfmmla z0.s, z0.h, z1.h
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN bfmlalb
+fmul    z0.d, z0.d, z0.d
+bfmlalb z0.s, z1.h, z2.h
+bfmlalb z0.s, z1.h, z2.h
+bfmlalb z0.s, z0.h, z1.h
+# LLVM-MCA-END
+
+# CHECK:      [0] Code Region - madd
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.57
+# CHECK-NEXT: IPC:               0.57
+# CHECK-NEXT: Block RThroughput: 3.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeER.    .    ..   mul	x0, x0, x0
+# CHECK-NEXT: [0,1]     D==eeER   .    ..   madd	x0, x1, x2, x0
+# CHECK-NEXT: [0,2]     D===eeER  .    ..   madd	x0, x1, x2, x0
+# CHECK-NEXT: [0,3]     D=====eeER.    ..   madd	x0, x0, x0, x0
+# CHECK-NEXT: [1,0]     D=======eeER   ..   mul	x0, x0, x0
+# CHECK-NEXT: [1,1]     D=========eeER ..   madd	x0, x1, x2, x0
+# CHECK-NEXT: [1,2]     D==========eeER..   madd	x0, x1, x2, x0
+# CHECK-NEXT: [1,3]     D============eeER   madd	x0, x0, x0, x0
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     4.5    0.5    0.0       mul	x0, x0, x0
+# CHECK-NEXT: 1.     2     6.5    0.0    0.0       madd	x0, x1, x2, x0
+# CHECK-NEXT: 2.     2     7.5    0.0    0.0       madd	x0, x1, x2, x0
+# CHECK-NEXT: 3.     2     9.5    0.0    0.0       madd	x0, x0, x0, x0
+# CHECK-NEXT:        2     7.0    0.1    0.0       <total>
+
+# CHECK:      [1] Code Region - smaddl
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.57
+# CHECK-NEXT: IPC:               0.57
+# CHECK-NEXT: Block RThroughput: 3.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeER.    .    ..   mul	x0, x0, x0
+# CHECK-NEXT: [0,1]     D==eeER   .    ..   smaddl	x0, w1, w2, x0
+# CHECK-NEXT: [0,2]     D===eeER  .    ..   smaddl	x0, w1, w2, x0
+# CHECK-NEXT: [0,3]     D=====eeER.    ..   smaddl	x0, w0, w0, x0
+# CHECK-NEXT: [1,0]     D=======eeER   ..   mul	x0, x0, x0
+# CHECK-NEXT: [1,1]     D=========eeER ..   smaddl	x0, w1, w2, x0
+# CHECK-NEXT: [1,2]     D==========eeER..   smaddl	x0, w1, w2, x0
+# CHECK-NEXT: [1,3]     D============eeER   smaddl	x0, w0, w0, x0
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     4.5    0.5    0.0       mul	x0, x0, x0
+# CHECK-NEXT: 1.     2     6.5    0.0    0.0       smaddl	x0, w1, w2, x0
+# CHECK-NEXT: 2.     2     7.5    0.0    0.0       smaddl	x0, w1, w2, x0
+# CHECK-NEXT: 3.     2     9.5    0.0    0.0       smaddl	x0, w0, w0, x0
+# CHECK-NEXT:        2     7.0    0.1    0.0       <total>
+
+# CHECK:      [2] Code Region - fmadd
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      600
+# CHECK-NEXT: Total Cycles:      1703
+# CHECK-NEXT: Total uOps:        600
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.35
+# CHECK-NEXT: IPC:               0.35
+# CHECK-NEXT: Block RThroughput: 1.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeER.    .    .    .    .    .    ..   fadd	d0, d0, d0
+# CHECK-NEXT: [0,1]     D==eeeeER .    .    .    .    .    ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [0,2]     D======eeeER   .    .    .    .    ..   fmul	d0, d0, d0
+# CHECK-NEXT: [0,3]     D=======eeeeER .    .    .    .    ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [0,4]     D=========eeeeER    .    .    .    ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [0,5]     D=============eeeeER.    .    .    ..   fmadd	d0, d0, d1, d2
+# CHECK-NEXT: [1,0]     D=================eeER   .    .    ..   fadd	d0, d0, d0
+# CHECK-NEXT: [1,1]     D===================eeeeER    .    ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [1,2]     D=======================eeeER .    ..   fmul	d0, d0, d0
+# CHECK-NEXT: [1,3]     D========================eeeeER    ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [1,4]     D==========================eeeeER  ..   fmadd	d0, d1, d2, d0
+# CHECK-NEXT: [1,5]     D==============================eeeeER   fmadd	d0, d0, d1, d2
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       fadd	d0, d0, d0
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       fmadd	d0, d1, d2, d0
+# CHECK-NEXT: 2.     2     15.5   0.0    0.0       fmul	d0, d0, d0
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmadd	d0, d1, d2, d0
+# CHECK-NEXT: 4.     2     18.5   0.0    0.0       fmadd	d0, d1, d2, d0
+# CHECK-NEXT: 5.     2     22.5   0.0    0.0       fmadd	d0, d0, d1, d2
+# CHECK-NEXT:        2     15.7   0.1    0.0       <total>
+
+# CHECK:      [3] Code Region - saba
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 1.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeER   .    .    .    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D====eeeeER    .    .    .  .   saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   saba	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D=================eeeeER .  .   saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [1,3]     D======================eeeeER   saba	v0.4s, v0.4s, v1.4s
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       saba	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       saba	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [4] Code Region - sadalp
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 1.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeER   .    .    .    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D====eeeeER    .    .    .  .   sadalp	v0.2d, v1.4s
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   sadalp	v0.2d, v1.4s
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   sadalp	v0.2d, v0.4s
+# CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D=================eeeeER .  .   sadalp	v0.2d, v1.4s
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   sadalp	v0.2d, v1.4s
+# CHECK-NEXT: [1,3]     D======================eeeeER   sadalp	v0.2d, v0.4s
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       sadalp	v0.2d, v1.4s
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       sadalp	v0.2d, v1.4s
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       sadalp	v0.2d, v0.4s
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [5] Code Region - sdot
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1103
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.36
+# CHECK-NEXT: IPC:               0.36
+# CHECK-NEXT: Block RThroughput: 0.8
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01234
+
+# CHECK:      [0,0]     DeeeeER   .    .    .   .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D====eeeER.    .    .   .   sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [0,2]     D=====eeeER    .    .   .   sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [0,3]     D========eeeER .    .   .   sdot	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT: [1,0]     D===========eeeeER  .   .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D===============eeeER   .   sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [1,2]     D================eeeER  .   sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [1,3]     D===================eeeER   sdot	v0.4s, v0.16b, v1.16b
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: 2.     2     11.5   0.0    0.0       sdot	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: 3.     2     14.5   0.0    0.0       sdot	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT:        2     10.8   0.1    0.0       <total>
+
+# CHECK:      [6] Code Region - smmla
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1103
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.36
+# CHECK-NEXT: IPC:               0.36
+# CHECK-NEXT: Block RThroughput: 0.8
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01234
+
+# CHECK:      [0,0]     DeeeeER   .    .    .   .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D====eeeER.    .    .   .   smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [0,2]     D=====eeeER    .    .   .   smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [0,3]     D========eeeER .    .   .   smmla	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT: [1,0]     D===========eeeeER  .   .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D===============eeeER   .   smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [1,2]     D================eeeER  .   smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: [1,3]     D===================eeeER   smmla	v0.4s, v0.16b, v1.16b
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     6.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: 2.     2     11.5   0.0    0.0       smmla	v0.4s, v1.16b, v2.16b
+# CHECK-NEXT: 3.     2     14.5   0.0    0.0       smmla	v0.4s, v0.16b, v1.16b
+# CHECK-NEXT:        2     10.8   0.1    0.0       <total>
+
+# CHECK:      [7] Code Region - mla
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeER   .    .    .    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D====eeeeER    .    .    .  .   mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   mla	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D=================eeeeER .  .   mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: [1,3]     D======================eeeeER   mla	v0.4s, v0.4s, v1.4s
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       mla	v0.4s, v1.4s, v2.4s
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       mla	v0.4s, v0.4s, v1.4s
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [8] Code Region - smlal2
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeER   .    .    .    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D====eeeeER    .    .    .  .   smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   smlal2	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D=================eeeeER .  .   smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     D======================eeeeER   smlal2	v0.4s, v0.8h, v1.8h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       smlal2	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       smlal2	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [9] Code Region - ssra
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 1.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeER   .    .    .    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D====eeeeER    .    .    .  .   ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   ssra	v0.2d, v0.2d, #1
+# CHECK-NEXT: [1,0]     D=============eeeeER.    .  .   mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D=================eeeeER .  .   ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: [1,3]     D======================eeeeER   ssra	v0.2d, v0.2d, #1
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       mul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       ssra	v0.2d, v1.2d, #1
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       ssra	v0.2d, v0.2d, #1
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [10] Code Region - fcmla
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .  .   fmul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [0,1]     D===eeeeER.    .    .    .  .   fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   fcmla	v0.2d, v0.2d, v1.2d, #90
+# CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: [1,1]     D================eeeeER  .  .   fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: [1,3]     D======================eeeeER   fcmla	v0.2d, v0.2d, v1.2d, #90
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       fcmla	v0.2d, v1.2d, v2.2d, #90
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fcmla	v0.2d, v0.2d, v1.2d, #90
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+
+# CHECK:      [11] Code Region - fmla
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      600
+# CHECK-NEXT: Total Cycles:      1703
+# CHECK-NEXT: Total uOps:        600
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.35
+# CHECK-NEXT: IPC:               0.35
+# CHECK-NEXT: Block RThroughput: 1.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .    .    ..   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0,1]     D=eeeeER  .    .    .    .    .    ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [0,2]     D=====eeER.    .    .    .    .    ..   fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0,3]     D=======eeeeER .    .    .    .    ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [0,4]     D=========eeeeER    .    .    .    ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [0,5]     D=============eeeeER.    .    .    ..   fmla	v0.2d, v0.2d, v1.2d
+# CHECK-NEXT: [1,0]     D=================eeeER  .    .    ..   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,1]     D==================eeeeER.    .    ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [1,2]     D======================eeER   .    ..   fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,3]     D========================eeeeER    ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [1,4]     D==========================eeeeER  ..   fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: [1,5]     D==============================eeeeER   fmla	v0.2d, v0.2d, v1.2d
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: 2.     2     14.5   0.0    0.0       fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: 4.     2     18.5   0.0    0.0       fmla	v0.2d, v1.2d, v2.2d
+# CHECK-NEXT: 5.     2     22.5   0.0    0.0       fmla	v0.2d, v0.2d, v1.2d
+# CHECK-NEXT:        2     15.3   0.1    0.0       <total>
+
+# CHECK:      [12] Code Region - fmlal
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      600
+# CHECK-NEXT: Total Cycles:      2203
+# CHECK-NEXT: Total uOps:        600
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.27
+# CHECK-NEXT: IPC:               0.27
+# CHECK-NEXT: Block RThroughput: 1.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789
+# CHECK-NEXT: Index     0123456789          0123456789          0123456
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .    .    .    .    ..   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0,1]     D===eeeeeER    .    .    .    .    .    .    ..   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [0,2]     D========eeER  .    .    .    .    .    .    ..   fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    .    .    .    ..   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [0,4]     D============eeeeeER.    .    .    .    .    ..   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [0,5]     D=================eeeeeER.    .    .    .    ..   fmlal	v0.4s, v0.4h, v1.4h
+# CHECK-NEXT: [1,0]     D======================eeeER  .    .    .    ..   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,1]     D=========================eeeeeER  .    .    ..   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [1,2]     D==============================eeER.    .    ..   fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,3]     D================================eeeeeER.    ..   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [1,4]     D==================================eeeeeER   ..   fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: [1,5]     D=======================================eeeeeER   fmlal	v0.4s, v0.4h, v1.4h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     12.0   0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 1.     2     15.0   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: 2.     2     20.0   0.0    0.0       fadd	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 3.     2     22.0   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: 4.     2     24.0   0.0    0.0       fmlal	v0.4s, v1.4h, v2.4h
+# CHECK-NEXT: 5.     2     29.0   0.0    0.0       fmlal	v0.4s, v0.4h, v1.4h
+# CHECK-NEXT:        2     20.3   0.1    0.0       <total>
+
+# CHECK:      [13] Code Region - bfdot
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .  .   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0,1]     D===eeeeER.    .    .    .  .   bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   bfdot	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,1]     D================eeeeER  .  .   bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     D======================eeeeER   bfdot	v0.4s, v0.8h, v1.8h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       bfdot	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       bfdot	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+
+# CHECK:      [14] Code Region - bfmmla
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1603
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.25
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          01234
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .    .   .   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0,1]     D===eeeeeER    .    .    .    .   .   bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,2]     D======eeeeeER .    .    .    .   .   bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,3]     D===========eeeeeER .    .    .   .   bfmmla	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,0]     D================eeeER   .    .   .   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,1]     D===================eeeeeER   .   .   bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,2]     D======================eeeeeER.   .   bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     D===========================eeeeeER   bfmmla	v0.4s, v0.8h, v1.8h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     9.0    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 1.     2     12.0   0.0    0.0       bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 2.     2     15.0   0.0    0.0       bfmmla	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     20.0   0.0    0.0       bfmmla	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     14.0   0.1    0.0       <total>
+
+# CHECK:      [15] Code Region - bfmlalb
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .  .   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [0,1]     D===eeeeER.    .    .    .  .   bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   bfmlalb	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: [1,1]     D================eeeeER  .  .   bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: [1,3]     D======================eeeeER   bfmlalb	v0.4s, v0.8h, v1.8h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       bfmlalb	v0.4s, v1.8h, v2.8h
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       bfmlalb	v0.4s, v0.8h, v1.8h
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+
+# CHECK:      [16] Code Region - crc32cb
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.57
+# CHECK-NEXT: IPC:               0.57
+# CHECK-NEXT: Block RThroughput: 3.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeER.    .    ..   mul	w0, w0, w0
+# CHECK-NEXT: [0,1]     D==eeER   .    ..   crc32cb	w0, w0, w1
+# CHECK-NEXT: [0,2]     D===eeER  .    ..   crc32cb	w0, w0, w1
+# CHECK-NEXT: [0,3]     D=====eeER.    ..   crc32cb	w0, w0, w0
+# CHECK-NEXT: [1,0]     D=======eeER   ..   mul	w0, w0, w0
+# CHECK-NEXT: [1,1]     D=========eeER ..   crc32cb	w0, w0, w1
+# CHECK-NEXT: [1,2]     D==========eeER..   crc32cb	w0, w0, w1
+# CHECK-NEXT: [1,3]     D============eeER   crc32cb	w0, w0, w0
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     4.5    0.5    0.0       mul	w0, w0, w0
+# CHECK-NEXT: 1.     2     6.5    0.0    0.0       crc32cb	w0, w0, w1
+# CHECK-NEXT: 2.     2     7.5    0.0    0.0       crc32cb	w0, w0, w1
+# CHECK-NEXT: 3.     2     9.5    0.0    0.0       crc32cb	w0, w0, w0
+# CHECK-NEXT:        2     7.0    0.1    0.0       <total>
+
+# CHECK:      [17] Code Region - Z sdot.s
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1203
+# CHECK-NEXT: Total uOps:        500
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.42
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123456
+
+# CHECK:      [0,0]     DeeeeeER  .    .    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D=====eeeER    .    .    ..   sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: [0,2]     D======eeeER   .    .    ..   sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: [0,3]     D=========eeeER.    .    ..   sdot	z0.s, z0.b, z1.b
+# CHECK-NEXT: [1,0]     D============eeeeeER.    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D=================eeeER  ..   sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: [1,2]     D==================eeeER ..   sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: [1,3]     D=====================eeeER   sdot	z0.s, z0.b, z1.b
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.0    0.5    0.0       mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.0   0.0    0.0       sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: 2.     2     13.0   0.0    0.0       sdot	z0.s, z1.b, z2.b
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       sdot	z0.s, z0.b, z1.b
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [18] Code Region - Z sudot
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1203
+# CHECK-NEXT: Total uOps:        500
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.42
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123456
+
+# CHECK:      [0,0]     DeeeeeER  .    .    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D=====eeeER    .    .    ..   sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: [0,2]     D======eeeER   .    .    ..   sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: [0,3]     D=========eeeER.    .    ..   sdot	z0.s, z0.b, z1.b[1]
+# CHECK-NEXT: [1,0]     D============eeeeeER.    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D=================eeeER  ..   sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: [1,2]     D==================eeeER ..   sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: [1,3]     D=====================eeeER   sdot	z0.s, z0.b, z1.b[1]
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.0    0.5    0.0       mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.0   0.0    0.0       sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: 2.     2     13.0   0.0    0.0       sdot	z0.s, z1.b, z2.b[1]
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       sdot	z0.s, z0.b, z1.b[1]
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [19] Code Region - Z sdot.d
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1403
+# CHECK-NEXT: Total uOps:        500
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.36
+# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D=====eeeeER   .    .    .    .   sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: [0,2]     D======eeeeER  .    .    .    .   sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: [0,3]     D==========eeeeER   .    .    .   sdot	z0.d, z0.h, z1.h
+# CHECK-NEXT: [1,0]     D==============eeeeeER   .    .   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D===================eeeeER    .   sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: [1,2]     D====================eeeeER   .   sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: [1,3]     D========================eeeeER   sdot	z0.d, z0.h, z1.h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: 1.     2     13.0   0.0    0.0       sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: 2.     2     14.0   0.0    0.0       sdot	z0.d, z1.h, z2.h
+# CHECK-NEXT: 3.     2     18.0   0.0    0.0       sdot	z0.d, z0.h, z1.h
+# CHECK-NEXT:        2     13.3   0.1    0.0       <total>
+
+# CHECK:      [20] Code Region - Z smmla
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1203
+# CHECK-NEXT: Total uOps:        500
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.42
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123456
+
+# CHECK:      [0,0]     DeeeeeER  .    .    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D=====eeeER    .    .    ..   smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: [0,2]     D======eeeER   .    .    ..   smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: [0,3]     D=========eeeER.    .    ..   smmla	z0.s, z0.b, z1.b
+# CHECK-NEXT: [1,0]     D============eeeeeER.    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D=================eeeER  ..   smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: [1,2]     D==================eeeER ..   smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: [1,3]     D=====================eeeER   smmla	z0.s, z0.b, z1.b
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.0    0.5    0.0       mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.0   0.0    0.0       smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: 2.     2     13.0   0.0    0.0       smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT: 3.     2     16.0   0.0    0.0       smmla	z0.s, z0.b, z1.b
+# CHECK-NEXT:        2     12.0   0.1    0.0       <total>
+
+# CHECK:      [21] Code Region - Z mla.d
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1703
+# CHECK-NEXT: Total uOps:        800
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.47
+# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: Block RThroughput: 8.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D=====eeeeeER  .    .    .    .    ..   mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,2]     D=======eeeeeER.    .    .    .    ..   mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,3]     D============eeeeeER.    .    .    ..   mla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT: [1,0]     D=================eeeeeER.    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D======================eeeeeER.    ..   mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,2]     D========================eeeeeER   ..   mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,3]     .D============================eeeeeER   mla	z0.d, p0/m, z0.d, z1.d
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: 1.     2     14.5   0.0    0.0       mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 2.     2     16.5   0.0    0.0       mla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 3.     2     21.0   0.0    0.0       mla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT:        2     15.4   0.1    0.0       <total>
+
+# CHECK:      [22] Code Region - Z mad.d
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1703
+# CHECK-NEXT: Total uOps:        800
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.47
+# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: Block RThroughput: 8.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D=====eeeeeER  .    .    .    .    ..   mad	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,2]     D=======eeeeeER.    .    .    .    ..   mad	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,3]     D============eeeeeER.    .    .    ..   mad	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT: [1,0]     D=================eeeeeER.    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D======================eeeeeER.    ..   mad	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,2]     D========================eeeeeER   ..   mad	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,3]     .D============================eeeeeER   mad	z0.d, p0/m, z0.d, z1.d
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: 1.     2     14.5   0.0    0.0       mad	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 2.     2     16.5   0.0    0.0       mad	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 3.     2     21.0   0.0    0.0       mad	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT:        2     15.4   0.1    0.0       <total>
+
+# CHECK:      [23] Code Region - Z msb.d
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1703
+# CHECK-NEXT: Total uOps:        800
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.47
+# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: Block RThroughput: 8.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D=====eeeeeER  .    .    .    .    ..   msb	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,2]     D=======eeeeeER.    .    .    .    ..   msb	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,3]     D============eeeeeER.    .    .    ..   msb	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT: [1,0]     D=================eeeeeER.    .    ..   mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D======================eeeeeER.    ..   msb	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,2]     D========================eeeeeER   ..   msb	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,3]     .D============================eeeeeER   msb	z0.d, p0/m, z0.d, z1.d
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     9.5    0.5    0.0       mul	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT: 1.     2     14.5   0.0    0.0       msb	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 2.     2     16.5   0.0    0.0       msb	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 3.     2     21.0   0.0    0.0       msb	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT:        2     15.4   0.1    0.0       <total>
+
+# CHECK:      [24] Code Region - Z fcmla ZPmZZ
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1503
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.27
+# CHECK-NEXT: IPC:               0.27
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          012
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .    . .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D===eeeeeER    .    .    .    . .   fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .    . .   fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    . .   fcmla	z0.d, p0/m, z0.d, z1.d, #90
+# CHECK-NEXT: [1,0]     D===============eeeER    .    . .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D==================eeeeeER    . .   fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: [1,2]     D====================eeeeeER  . .   fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: [1,3]     D=========================eeeeeER   fcmla	z0.d, p0/m, z0.d, z1.d, #90
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     8.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT: 3.     2     18.5   0.0    0.0       fcmla	z0.d, p0/m, z0.d, z1.d, #90
+# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
+
+# CHECK:      [25] Code Region - Z fcmla ZZZI
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1503
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.27
+# CHECK-NEXT: IPC:               0.27
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          012
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .    . .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D===eeeeeER    .    .    .    . .   fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .    . .   fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    . .   fcmla	z0.s, z0.s, z1.s[1], #90
+# CHECK-NEXT: [1,0]     D===============eeeER    .    . .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D==================eeeeeER    . .   fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: [1,2]     D====================eeeeeER  . .   fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: [1,3]     D=========================eeeeeER   fcmla	z0.s, z0.s, z1.s[1], #90
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     8.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       fcmla	z0.s, z1.s, z2.s[1], #90
+# CHECK-NEXT: 3.     2     18.5   0.0    0.0       fcmla	z0.s, z0.s, z1.s[1], #90
+# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
+
+# CHECK:      [26] Code Region - Z fmla ZPmZZ
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .  .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D===eeeeER.    .    .    .  .   fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   fmla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D================eeeeER  .  .   fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: [1,3]     D======================eeeeER   fmla	z0.d, p0/m, z0.d, z1.d
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       fmla	z0.d, p0/m, z1.d, z2.d
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmla	z0.d, p0/m, z0.d, z1.d
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+
+# CHECK:      [27] Code Region - Z fmla ZZZI
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .  .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D===eeeeER.    .    .    .  .   fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   fmla	z0.d, z0.d, z1.d[1]
+# CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D================eeeeER  .  .   fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: [1,3]     D======================eeeeER   fmla	z0.d, z0.d, z1.d[1]
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       fmla	z0.d, z1.d, z2.d[1]
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       fmla	z0.d, z0.d, z1.d[1]
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+
+# CHECK:      [28] Code Region - Z bfdot
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1303
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .  .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D===eeeeER.    .    .    .  .   bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    .  .   bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: [0,3]     D=========eeeeER    .    .  .   bfdot	z0.s, z0.h, z1.h
+# CHECK-NEXT: [1,0]     D=============eeeER .    .  .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D================eeeeER  .  .   bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,2]     D==================eeeeER.  .   bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,3]     D======================eeeeER   bfdot	z0.s, z0.h, z1.h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     7.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     10.5   0.0    0.0       bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: 2.     2     12.5   0.0    0.0       bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT: 3.     2     16.5   0.0    0.0       bfdot	z0.s, z0.h, z1.h
+# CHECK-NEXT:        2     11.8   0.1    0.0       <total>
+
+# CHECK:      [29] Code Region - Z bfmmla
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1603
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.25
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          01234
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .    .   .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D===eeeeeER    .    .    .    .   .   bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: [0,2]     D======eeeeeER .    .    .    .   .   bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: [0,3]     D===========eeeeeER .    .    .   .   bfmmla	z0.s, z0.h, z1.h
+# CHECK-NEXT: [1,0]     D================eeeER   .    .   .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D===================eeeeeER   .   .   bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,2]     D======================eeeeeER.   .   bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,3]     D===========================eeeeeER   bfmmla	z0.s, z0.h, z1.h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     9.0    0.5    0.0       fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     12.0   0.0    0.0       bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: 2.     2     15.0   0.0    0.0       bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT: 3.     2     20.0   0.0    0.0       bfmmla	z0.s, z0.h, z1.h
+# CHECK-NEXT:        2     14.0   0.1    0.0       <total>
+
+# CHECK:      [30] Code Region - bfmlalb
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1503
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    15
+# CHECK-NEXT: uOps Per Cycle:    0.27
+# CHECK-NEXT: IPC:               0.27
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          012
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeER    .    .    .    .    . .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [0,1]     D===eeeeeER    .    .    .    . .   bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .    . .   bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: [0,3]     D==========eeeeeER  .    .    . .   bfmlalb	z0.s, z0.h, z1.h
+# CHECK-NEXT: [1,0]     D===============eeeER    .    . .   fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: [1,1]     D==================eeeeeER    . .   bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,2]     D====================eeeeeER  . .   bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: [1,3]     D=========================eeeeeER   bfmlalb	z0.s, z0.h, z1.h
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     8.5    0.5    0.0       fmul	z0.d, z0.d, z0.d
+# CHECK-NEXT: 1.     2     11.5   0.0    0.0       bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: 2.     2     13.5   0.0    0.0       bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT: 3.     2     18.5   0.0    0.0       bfmlalb	z0.s, z0.h, z1.h
+# CHECK-NEXT:        2     13.0   0.1    0.0       <total>
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
index 1e8df4770d7950039f2bad01477c7be462f7481b..65b73177c7b70adbeedebdcd11e13c120bb8053b 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
@@ -1365,8 +1365,8 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        fcmgt	s10, s11, s12
 # CHECK-NEXT:  1      2     0.25                        fcmgt	v0.4s, v0.4s, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmgt	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  1      2     0.25                        fcmla	v0.2s, v0.2s, v0.2s, #90
-# CHECK-NEXT:  1      2     0.25                        fcmla	v0.4s, v0.4s, v0.s[1], #0
+# CHECK-NEXT:  1      4     0.25                        fcmla	v0.2s, v0.2s, v0.2s, #90
+# CHECK-NEXT:  1      4     0.25                        fcmla	v0.4s, v0.4s, v0.s[1], #0
 # CHECK-NEXT:  1      2     0.25                        fcmle	d20, d21, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmle	s10, s11, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmle	v0.2d, v0.2d, #0.0
@@ -1651,7 +1651,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v0.2d, v1.2d, v2.2d, v3.2d }, [sp]
 # CHECK-NEXT:  8      8     1.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], #16
 # CHECK-NEXT:  8      8     1.00    *                   ld4r	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x8
-# CHECK-NEXT:  1      2     0.25                        mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        mla	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  1      4     0.50                        mls	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        mov	b0, v0.b[15]
 # CHECK-NEXT:  1      2     0.25                        mov	d6, v0.d[1]
@@ -1673,7 +1673,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        movi	v0.2s, #8, msl #8
 # CHECK-NEXT:  1      2     0.25                        movi	v0.4s, #255, lsl #24
 # CHECK-NEXT:  1      2     0.25                        movi	v0.8b, #255
-# CHECK-NEXT:  1      2     0.25                        mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        mul	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        mvni	v0.2s, #0
 # CHECK-NEXT:  1      2     0.25                        mvni	v0.4s, #16, msl #16
 # CHECK-NEXT:  1      2     0.25                        neg	d29, d24
@@ -1780,10 +1780,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  2      4     1.00                        scvtf	v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        scvtf	v0.4s, v0.4s, #3
 # CHECK-NEXT:  4      6     1.00                        scvtf	v0.8h, v0.8h
-# CHECK-NEXT:  1      2     0.25                        sdot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  1      2     0.25                        sdot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  1      2     0.25                        sdot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  1      2     0.25                        sdot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        sdot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        sdot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.25                        sdot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.25                        sdot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.25                        shadd	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        shl	d7, d10, #12
 # CHECK-NEXT:  1      2     0.50                        shl	v0.16b, v0.16b, #3
@@ -1873,26 +1873,26 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        sqadd	b20, b11, b15
 # CHECK-NEXT:  1      2     0.25                        sqadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        sqadd	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  1      2     0.25                        sqdmlal	d19, s24, s12
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	d19, s24, s12
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	d8, s9, v0.s[1]
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	s0, h0, v0.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqdmlal	s17, h27, h12
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	s17, h27, h12
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        sqdmlal2	v0.2d, v0.4s, v0.4s
 # CHECK-NEXT:  1      4     0.50                        sqdmlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  1      2     0.25                        sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	d12, s23, s13
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	d8, s9, v0.s[1]
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	s0, h0, v0.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	s14, h12, h25
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v0.2d, v0.4s, v0.4s
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  1      2     0.25                        sqdmulh	h10, h11, h12
-# CHECK-NEXT:  1      2     0.25                        sqdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  1      2     0.25                        sqdmulh	s20, s21, s2
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	h10, h11, h12
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	s20, s21, s2
 # CHECK-NEXT:  1      4     0.50                        sqdmulh	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      4     0.50                        sqdmulh	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      3     0.50                        sqdmull	d1, s1, v0.s[1]
@@ -1914,34 +1914,34 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        sqneg	v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        sqneg	v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        sqneg	v0.8h, v0.8h
-# CHECK-NEXT:  1      2     0.25                        sqrdmlah	h0, h1, v2.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlah	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlah	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlah	s0, s1, v2.s[1]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlah	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlah	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	h0, h1, v2.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	s0, s1, v2.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.4s, v1.4s, v2.s[1]
 # CHECK-NEXT:  1      4     0.50                        sqrdmlah	h0, h1, h2
 # CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.4h, v1.4h, v2.4h
 # CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.8h, v1.8h, v2.8h
 # CHECK-NEXT:  1      4     0.50                        sqrdmlah	s0, s1, s2
 # CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.2s, v1.2s, v2.2s
 # CHECK-NEXT:  1      4     0.50                        sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  1      2     0.25                        sqrdmlsh	h0, h1, v2.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlsh	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlsh	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlsh	s0, s1, v2.s[1]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlsh	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  1      2     0.25                        sqrdmlsh	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	h0, h1, v2.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	s0, s1, v2.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.4s, v1.4s, v2.s[1]
 # CHECK-NEXT:  1      4     0.50                        sqrdmlsh	h0, h1, h2
 # CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.4h, v1.4h, v2.4h
 # CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.8h, v1.8h, v2.8h
 # CHECK-NEXT:  1      4     0.50                        sqrdmlsh	s0, s1, s2
 # CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.2s, v1.2s, v2.2s
 # CHECK-NEXT:  1      4     0.50                        sqrdmlsh	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  1      2     0.25                        sqrdmulh	h10, h11, h12
-# CHECK-NEXT:  1      2     0.25                        sqrdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  1      2     0.25                        sqrdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  1      2     0.25                        sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	s20, s21, s2
 # CHECK-NEXT:  1      4     0.50                        sqrdmulh	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        sqrdmulh	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        sqrshl	d31, d31, d31
@@ -2124,8 +2124,8 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  5      4     1.00           *            st4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
 # CHECK-NEXT:  1      2     0.25                        sub	d15, d5, d16
 # CHECK-NEXT:  1      2     0.25                        sub	v0.2d, v0.2d, v0.2d
-# CHECK-NEXT:  1      2     0.25                        sudot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  1      2     0.25                        sudot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        sudot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        sudot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.25                        suqadd	b19, b14
 # CHECK-NEXT:  1      2     0.25                        suqadd	d18, d22
 # CHECK-NEXT:  1      2     0.25                        suqadd	h20, h15
@@ -2222,10 +2222,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  2      4     1.00                        ucvtf	v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        ucvtf	v0.4s, v0.4s, #3
 # CHECK-NEXT:  4      6     1.00                        ucvtf	v0.8h, v0.8h
-# CHECK-NEXT:  1      2     0.25                        udot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  1      2     0.25                        udot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  1      2     0.25                        udot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  1      2     0.25                        udot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        udot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        udot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.25                        udot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.25                        udot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.25                        uhadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        uhadd	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        uhsub	v0.4s, v0.4s, v0.4s
@@ -2356,10 +2356,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        ursra	v0.4s, v0.4s, #3
 # CHECK-NEXT:  1      4     0.50                        ursra	v0.8b, v0.8b, #3
 # CHECK-NEXT:  1      4     0.50                        ursra	v0.8h, v0.8h, #3
-# CHECK-NEXT:  1      2     0.25                        usdot	v0.2s, v0.8b, v0.4b[2]
-# CHECK-NEXT:  1      2     0.25                        usdot	v0.2s, v0.8b, v0.8b
-# CHECK-NEXT:  1      2     0.25                        usdot	v0.4s, v0.16b, v0.16b
-# CHECK-NEXT:  1      2     0.25                        usdot	v0.4s, v0.16b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        usdot	v0.2s, v0.8b, v0.4b[2]
+# CHECK-NEXT:  1      3     0.25                        usdot	v0.2s, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.25                        usdot	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.25                        usdot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.50                        ushl	d0, d0, d0
 # CHECK-NEXT:  1      2     0.50                        ushl	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.50                        ushl	v0.4s, v0.4s, v0.4s
@@ -2465,7 +2465,7 @@ zip2 v0.8h, v0.8h, v0.8h
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     26.67  49.17  49.17  18.75  7.75   7.75   7.75   394.50 377.00 349.00 331.50
+# CHECK-NEXT:  -      -      -      -     26.67  49.17  49.17  18.75  7.75   7.75   7.75   401.00 370.50 355.50 325.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
@@ -2892,7 +2892,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.2d, v1.2d, v2.2d, v3.2d }, [sp]
 # CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], #16
 # CHECK-NEXT:  -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[15]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d6, v0.d[1]
@@ -2914,7 +2914,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2s, #8, msl #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.4s, #255, lsl #24
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.8b, #255
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.2s, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.4s, #16, msl #16
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	d29, d24
@@ -3114,26 +3114,26 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	b20, b11, b15
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.2s, v0.2s, v0.2s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmlal	d19, s24, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d19, s24, s12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d8, s9, v0.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s0, h0, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmlal	s17, h27, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s17, h27, h12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.2d, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d12, s23, s13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d8, s9, v0.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s0, h0, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s14, h12, h25
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.2d, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.4s, v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmulh	h10, h11, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s20, s21, s2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	d1, s1, v0.s[1]
@@ -3155,34 +3155,34 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqneg	v0.8h, v0.8h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlah	h0, h1, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlah	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlah	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlah	s0, s1, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlah	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlah	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	h0, h1, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	s0, s1, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4s, v1.4s, v2.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	h0, h1, h2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4h, v1.4h, v2.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.8h, v1.8h, v2.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	s0, s1, s2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.2s, v1.2s, v2.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlah	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlsh	h0, h1, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlsh	v0.4h, v1.4h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlsh	v0.8h, v1.8h, v2.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlsh	s0, s1, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlsh	v0.2s, v1.2s, v2.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmlsh	v0.4s, v1.4s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	h0, h1, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4h, v1.4h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.8h, v1.8h, v2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	s0, s1, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.2s, v1.2s, v2.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4s, v1.4s, v2.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	h0, h1, h2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4h, v1.4h, v2.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.8h, v1.8h, v2.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	s0, s1, s2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.2s, v1.2s, v2.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmlsh	v0.4s, v1.4s, v2.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmulh	h10, h11, h12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmulh	h7, h15, v0.h[3]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmulh	s15, s14, v0.s[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	s20, s21, s2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	d31, d31, d31
diff --git a/llvm/test/tools/llvm-profdata/memprof-merge-versions.test b/llvm/test/tools/llvm-profdata/memprof-merge-versions.test
index aa7d0329425dc51dbee038927d58cffba4ddfd97..144afdd92a1149eaf29cd4ac18edbeac4f6bee80 100644
--- a/llvm/test/tools/llvm-profdata/memprof-merge-versions.test
+++ b/llvm/test/tools/llvm-profdata/memprof-merge-versions.test
@@ -25,6 +25,10 @@ RUN: llvm-profdata show %t.prof.v3 | FileCheck %s
 RUN: llvm-profdata merge %t.proftext %p/Inputs/basic.memprofraw --memprof-version=3 --memprof-full-schema --profiled-binary %p/Inputs/basic.memprofexe -o %t.prof.v3
 RUN: llvm-profdata show %t.prof.v3 | FileCheck %s
 
+Check to see if llvm-profdata produces V3 by default.
+RUN: llvm-profdata merge %t.proftext %p/Inputs/basic.memprofraw --memprof-full-schema --profiled-binary %p/Inputs/basic.memprofexe -o %t.prof.default
+RUN: cmp %t.prof.default %t.prof.v3
+
 For now we only check the validity of the instrumented profile since we don't
 have a way to display the contents of the memprof indexed format yet.
 
diff --git a/llvm/test/tools/llvm-reduce/mir/preserve-reg-hints.mir b/llvm/test/tools/llvm-reduce/mir/preserve-reg-hints.mir
index ae55990c1ad87a7cdde5a83893b2bca367717ac1..8ba07c31dcce11c94ffdd345c05273ba7f0ba7ed 100644
--- a/llvm/test/tools/llvm-reduce/mir/preserve-reg-hints.mir
+++ b/llvm/test/tools/llvm-reduce/mir/preserve-reg-hints.mir
@@ -7,11 +7,11 @@
 # Make sure that register hints are preserved in the cloned function.
 
 # RESULT: registers:
-# RESULT-NEXT: - { id: 0, class: vgpr_32, preferred-register: '$vgpr0' }
-# RESULT-NEXT: - { id: 1, class: vgpr_32, preferred-register: '' }
-# RESULT-NEXT: - { id: 2, class: vgpr_32, preferred-register: '%1' }
-# RESULT-NEXT: - { id: 3, class: vgpr_32, preferred-register: '%4' }
-# RESULT-NEXT: - { id: 4, class: vgpr_32, preferred-register: '%3' }
+# RESULT-NEXT: - { id: 0, class: vgpr_32, preferred-register: '$vgpr0', flags: [   ] }
+# RESULT-NEXT: - { id: 1, class: vgpr_32, preferred-register: '', flags: [   ] }
+# RESULT-NEXT: - { id: 2, class: vgpr_32, preferred-register: '%1', flags: [   ] }
+# RESULT-NEXT: - { id: 3, class: vgpr_32, preferred-register: '%4', flags: [   ] }
+# RESULT-NEXT: - { id: 4, class: vgpr_32, preferred-register: '%3', flags: [   ] }
 ---
 name: register_hints
 tracksRegLiveness: true
diff --git a/llvm/test/tools/llvm-split/AMDGPU/kernels-dependency-indirect-callee-md.ll b/llvm/test/tools/llvm-split/AMDGPU/kernels-dependency-indirect-callee-md.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d2cfa570092ba61764479ca5eb5fcfd1ede6a9d3
--- /dev/null
+++ b/llvm/test/tools/llvm-split/AMDGPU/kernels-dependency-indirect-callee-md.ll
@@ -0,0 +1,69 @@
+; RUN: sed 's/_MD_/, !callees !{ptr @CallCandidate0}/g' %s | llvm-split -o %t -j 3 -mtriple amdgcn-amd-amdhsa
+; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t2 | FileCheck --check-prefix=CHECK2 --implicit-check-not=define %s
+
+; RUN: sed 's/_MD_//g' %s | llvm-split -o %t-nomd -j 3 -mtriple amdgcn-amd-amdhsa
+; RUN: llvm-dis -o - %t-nomd0 | FileCheck --check-prefix=CHECK-NOMD0 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t-nomd1 | FileCheck --check-prefix=CHECK-NOMD1 --implicit-check-not=define %s
+; RUN: llvm-dis -o - %t-nomd2 | FileCheck --check-prefix=CHECK-NOMD2 --implicit-check-not=define %s
+
+; CHECK0: define internal void @HelperC
+; CHECK0: define amdgpu_kernel void @C
+
+; CHECK1: define hidden void @CallCandidate1
+; CHECK1: define internal void @HelperB
+; CHECK1: define amdgpu_kernel void @B
+
+; CHECK2: define internal void @HelperA
+; CHECK2: define hidden void @CallCandidate0
+; CHECK2: define amdgpu_kernel void @A
+
+; CHECK-NOMD0: define internal void @HelperC
+; CHECK-NOMD0: define amdgpu_kernel void @C
+
+; CHECK-NOMD1: define internal void @HelperB
+; CHECK-NOMD1: define amdgpu_kernel void @B
+
+; CHECK-NOMD2: define internal void @HelperA
+; CHECK-NOMD2: define hidden void @CallCandidate0
+; CHECK-NOMD2: define hidden void @CallCandidate1
+; CHECK-NOMD2: define amdgpu_kernel void @A
+
+@addrthief = global [2 x ptr] [ptr @CallCandidate0, ptr @CallCandidate1]
+
+define internal void @HelperA(ptr %call) {
+  call void %call() _MD_
+  ret void
+}
+
+define internal void @CallCandidate0() {
+  ret void
+}
+
+define internal void @CallCandidate1() {
+  ret void
+}
+
+define internal void @HelperB() {
+  ret void
+}
+
+define internal void @HelperC() {
+  ret void
+}
+
+define amdgpu_kernel void @A(ptr %call) {
+  call void @HelperA(ptr %call)
+  ret void
+}
+
+define amdgpu_kernel void @B() {
+  call void @HelperB()
+  ret void
+}
+
+define amdgpu_kernel void @C() {
+  call void @HelperC()
+  ret void
+}
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index f21d9810cca6a9594ef0c5131ad66a534b8bdea6..4c023bcdd616582c75dbce7e5bb64b5c899c70b7 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -30,6 +30,7 @@
 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/IRPartitionLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
@@ -1060,7 +1061,7 @@ int runOrcJIT(const char *ProgName) {
   }
 
   if (PerModuleLazy)
-    J->setPartitionFunction(orc::CompileOnDemandLayer::compileWholeModule);
+    J->setPartitionFunction(orc::IRPartitionLayer::compileWholeModule);
 
   auto IRDump = createIRDebugDumper();
   J->getIRTransformLayer().setTransform(
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 17933e4500ead67065d32d9c8d7afa9bb855ddee..c235c3f2b105e15261684fd1a7bfb0811ad7349a 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -332,7 +332,7 @@ cl::opt<bool> DoWritePrevVersion(
 cl::opt<memprof::IndexedVersion> MemProfVersionRequested(
     "memprof-version", cl::Hidden, cl::sub(MergeSubcommand),
     cl::desc("Specify the version of the memprof format to use"),
-    cl::init(memprof::Version0),
+    cl::init(memprof::Version3),
     cl::values(clEnumValN(memprof::Version0, "0", "version 0"),
                clEnumValN(memprof::Version1, "1", "version 1"),
                clEnumValN(memprof::Version2, "2", "version 2"),
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceOpcodes.cpp b/llvm/tools/llvm-reduce/deltas/ReduceOpcodes.cpp
index fb8729c36a6f2d46656cf58a863d478e3a104f88..0e2a6decfbc9d5ffcb39f4f428ef2c7685c99ba8 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceOpcodes.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceOpcodes.cpp
@@ -30,7 +30,7 @@ static bool shouldIgnoreArgument(const Value *V) {
 static Value *replaceIntrinsic(Module &M, IntrinsicInst *II,
                                Intrinsic::ID NewIID,
                                ArrayRef<Type *> Tys = {}) {
-  Function *NewFunc = Intrinsic::getDeclaration(&M, NewIID, Tys);
+  Function *NewFunc = Intrinsic::getOrInsertDeclaration(&M, NewIID, Tys);
   II->setCalledFunction(NewFunc);
   return II;
 }
diff --git a/llvm/unittests/ADT/APFloatTest.cpp b/llvm/unittests/ADT/APFloatTest.cpp
index 6008f00e42a98923ac64f59cf699fe515a8e9dc2..665cff9c424594604ff7d351939d07425f019e4d 100644
--- a/llvm/unittests/ADT/APFloatTest.cpp
+++ b/llvm/unittests/ADT/APFloatTest.cpp
@@ -892,6 +892,13 @@ TEST(APFloatTest, Zero) {
   EXPECT_EQ(fcNegZero, APFloat(-0.0).classify());
 }
 
+TEST(APFloatTest, getOne) {
+  EXPECT_EQ(APFloat::getOne(APFloat::IEEEsingle(), false).convertToFloat(),
+            1.0f);
+  EXPECT_EQ(APFloat::getOne(APFloat::IEEEsingle(), true).convertToFloat(),
+            -1.0f);
+}
+
 TEST(APFloatTest, DecimalStringsWithoutNullTerminators) {
   // Make sure that we can parse strings without null terminators.
   // rdar://14323230.
diff --git a/llvm/unittests/ADT/APIntTest.cpp b/llvm/unittests/ADT/APIntTest.cpp
index 7a9ac5562e77224928a909410157f2be31378b03..4d5553fcbd1e3fbb02886c42fc40ac6898e4aa91 100644
--- a/llvm/unittests/ADT/APIntTest.cpp
+++ b/llvm/unittests/ADT/APIntTest.cpp
@@ -3423,6 +3423,7 @@ TEST(APIntTest, ZeroWidth) {
   EXPECT_EQ(0U, ZW.getBitWidth());
   EXPECT_EQ(0U, APInt(0, ArrayRef<uint64_t>({0, 1, 2})).getBitWidth());
   EXPECT_EQ(0U, APInt(0, "0", 10).getBitWidth());
+  EXPECT_EQ(0U, APInt::getAllOnes(0).getBitWidth());
 
   // Default constructor is single bit wide.
   EXPECT_EQ(1U, APInt().getBitWidth());
diff --git a/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp b/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
index 6437e0c9491f7f995f7f2c72858bd4e4a4a68608..8ad15ca41510f293d50d202556b6eddbdccebb00 100644
--- a/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
+++ b/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
@@ -430,7 +430,8 @@ static void RunRandTest(uint64_t Seed, int Size, int MinCount, int MaxCount,
   BB->insertInto(F);
   Instruction *Ret = ReturnInst::Create(C);
   Ret->insertInto(BB, BB->begin());
-  Function *FnAssume = Intrinsic::getDeclaration(Mod.get(), Intrinsic::assume);
+  Function *FnAssume =
+      Intrinsic::getOrInsertDeclaration(Mod.get(), Intrinsic::assume);
 
   std::vector<Argument *> ShuffledArgs;
   BitVector HasArg;
diff --git a/llvm/unittests/Analysis/MemorySSATest.cpp b/llvm/unittests/Analysis/MemorySSATest.cpp
index 9e6c517242a269b3761b31e0d39b4223c6aacd90..81784bb2360975dae0cebca7f7cb323f7cd892b8 100644
--- a/llvm/unittests/Analysis/MemorySSATest.cpp
+++ b/llvm/unittests/Analysis/MemorySSATest.cpp
@@ -1120,7 +1120,7 @@ TEST_F(MemorySSATest, LifetimeMarkersAreClobbers) {
   B.CreateStore(B.getInt8(0), Bar);
 
   auto GetLifetimeIntrinsic = [&](Intrinsic::ID ID) {
-    return Intrinsic::getDeclaration(&M, ID, {Foo->getType()});
+    return Intrinsic::getOrInsertDeclaration(&M, ID, {Foo->getType()});
   };
 
   B.CreateCall(GetLifetimeIntrinsic(Intrinsic::lifetime_end),
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index 77d966155dceff5c1d2bf9b037e6e1e5b4d76be0..0145ee70a14c17583d7155267c4af3cb17b87f70 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -2481,8 +2481,8 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsAddWithRange) {
 TEST_F(ComputeKnownBitsTest, ComputeKnownBitsUnknownVScale) {
   Module M("", Context);
   IRBuilder<> Builder(Context);
-  Function *TheFn =
-      Intrinsic::getDeclaration(&M, Intrinsic::vscale, {Builder.getInt32Ty()});
+  Function *TheFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::vscale,
+                                                      {Builder.getInt32Ty()});
   CallInst *CI = Builder.CreateCall(TheFn, {}, {}, "");
 
   KnownBits Known = computeKnownBits(CI, M.getDataLayout(), /* Depth */ 0);
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
index dd6edd35a8468b6c173107dc634fbd3aeb0f811b..2f3336e9085b636f8fe33237c130d9c8ec81d804 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
@@ -1548,3 +1548,48 @@ TEST_F(AArch64GISelMITest, TestNumSignBitsUAddoOverflow) {
   // Assert sign-extension from vector boolean
   EXPECT_EQ(32u, Info.computeNumSignBits(CopyOverflow));
 }
+
+TEST_F(AArch64GISelMITest, TestKnwonBitsUnmergeVectorScalar) {
+  StringRef MIRString = R"(
+   %copy_x0:_(<2 x s16>) = COPY $w0
+   %maskf:_(s16) = G_CONSTANT i16 15
+   %x0_x1:_(<2 x s16>) = G_BUILD_VECTOR %maskf, %maskf
+   %and:_(<2 x s16>) = G_AND %copy_x0, %x0_x1
+   %x0_0:_(s16), %x0_1:_(s16) = G_UNMERGE_VALUES %and
+   %result:_(s16) = COPY %x0_0
+)";
+
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyOverflow = Copies[Copies.size() - 1];
+
+  GISelKnownBits Info(*MF);
+
+  EXPECT_EQ(0xFFF0u, Info.getKnownBits(CopyOverflow).Zero.getZExtValue());
+}
+
+TEST_F(AArch64GISelMITest, TestKnwonBitsUnmergeVectorVector) {
+  StringRef MIRString = R"(
+   %copy_x0:_(<4 x s8>) = COPY $w0
+   %maskff:_(s8) = G_CONSTANT i8 255
+   %maskf:_(s8) = G_CONSTANT i8 15
+   %x0_x1:_(<4 x s8>) = G_BUILD_VECTOR %maskf, %maskf, %maskff, %maskff
+   %and:_(<4 x s8>) = G_AND %copy_x0, %x0_x1
+   %x0_0:_(<2 x s8>), %x0_1:_(<2 x s8>) = G_UNMERGE_VALUES %and
+   %result1:_(<2 x s8>) = COPY %x0_0
+   %result2:_(<2 x s8>) = COPY %x0_1
+)";
+
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  GISelKnownBits Info(*MF);
+
+  Register CopyOverflow1 = Copies[Copies.size() - 2];
+  EXPECT_EQ(0xF0u, Info.getKnownBits(CopyOverflow1).Zero.getZExtValue());
+  Register CopyOverflow2 = Copies[Copies.size() - 1];
+  EXPECT_EQ(0x00u, Info.getKnownBits(CopyOverflow2).Zero.getZExtValue());
+}
diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
index dc3380d35fda9df122b4a9359cd7f6133b2c6d31..a2bbb10039c9a03affbd9df22d3d265129857d0c 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
@@ -43,6 +43,8 @@ add_llvm_unittest(OrcJITTests
   TaskDispatchTest.cpp
   ThreadSafeModuleTest.cpp
   WrapperFunctionUtilsTest.cpp
+  JITLinkRedirectionManagerTest.cpp
+  ReOptimizeLayerTest.cpp
 
   EXPORT_SYMBOLS
   )
diff --git a/llvm/unittests/ExecutionEngine/Orc/JITLinkRedirectionManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/JITLinkRedirectionManagerTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b630b72ef7081699a709414dce69eb08ba50b9ea
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/Orc/JITLinkRedirectionManagerTest.cpp
@@ -0,0 +1,111 @@
+#include "OrcTestCommon.h"
+#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
+#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h"
+#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::jitlink;
+
+static int initialTarget() { return 42; }
+static int middleTarget() { return 13; }
+static int finalTarget() { return 53; }
+
+class JITLinkRedirectionManagerTest : public testing::Test {
+public:
+  ~JITLinkRedirectionManagerTest() {
+    if (ES)
+      if (auto Err = ES->endSession())
+        ES->reportError(std::move(Err));
+  }
+
+protected:
+  void SetUp() override {
+    auto JTMB = JITTargetMachineBuilder::detectHost();
+    // Bail out if we can not detect the host.
+    if (!JTMB) {
+      consumeError(JTMB.takeError());
+      GTEST_SKIP();
+    }
+    auto DLOrErr = JTMB->getDefaultDataLayoutForTarget();
+    if (!DLOrErr) {
+      consumeError(DLOrErr.takeError());
+      GTEST_SKIP();
+    }
+
+    // COFF-ARM64 is not supported yet
+    auto Triple = JTMB->getTargetTriple();
+    if (Triple.isOSBinFormatCOFF() && Triple.isAArch64())
+      GTEST_SKIP();
+
+    if (Triple.isPPC())
+      GTEST_SKIP();
+
+    ES = std::make_unique<ExecutionSession>(
+        std::make_unique<UnsupportedExecutorProcessControl>(
+            nullptr, nullptr, JTMB->getTargetTriple().getTriple()));
+    JD = &ES->createBareJITDylib("main");
+    ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(
+        *ES, std::make_unique<InProcessMemoryManager>(16384));
+    DL = std::make_unique<DataLayout>(std::move(*DLOrErr));
+  }
+  JITDylib *JD{nullptr};
+  std::unique_ptr<ExecutionSession> ES;
+  std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer;
+  std::unique_ptr<DataLayout> DL;
+};
+
+TEST_F(JITLinkRedirectionManagerTest, BasicRedirectionOperation) {
+  auto RM = JITLinkRedirectableSymbolManager::Create(*ObjLinkingLayer, *JD);
+  // Bail out if we can not create
+  if (!RM) {
+    consumeError(RM.takeError());
+    GTEST_SKIP();
+  }
+
+  auto DefineTarget = [&](StringRef TargetName, ExecutorAddr Addr) {
+    SymbolStringPtr Target = ES->intern(TargetName);
+    cantFail(JD->define(std::make_unique<SimpleMaterializationUnit>(
+        SymbolFlagsMap({{Target, JITSymbolFlags::Exported}}),
+        [&](std::unique_ptr<MaterializationResponsibility> R) -> void {
+          // No dependencies registered, can't fail.
+          cantFail(
+              R->notifyResolved({{Target, {Addr, JITSymbolFlags::Exported}}}));
+          cantFail(R->notifyEmitted({}));
+        })));
+    return cantFail(ES->lookup({JD}, TargetName));
+  };
+
+  auto InitialTarget =
+      DefineTarget("InitialTarget", ExecutorAddr::fromPtr(&initialTarget));
+  auto MiddleTarget =
+      DefineTarget("MiddleTarget", ExecutorAddr::fromPtr(&middleTarget));
+  auto FinalTarget =
+      DefineTarget("FinalTarget", ExecutorAddr::fromPtr(&finalTarget));
+
+  auto RedirectableSymbol = ES->intern("RedirectableTarget");
+  EXPECT_THAT_ERROR(
+      (*RM)->createRedirectableSymbols(JD->getDefaultResourceTracker(),
+                                       {{RedirectableSymbol, InitialTarget}}),
+      Succeeded());
+  auto RTDef = cantFail(ES->lookup({JD}, RedirectableSymbol));
+
+  auto RTPtr = RTDef.getAddress().toPtr<int (*)()>();
+  auto Result = RTPtr();
+  EXPECT_EQ(Result, 42) << "Failed to call initial target";
+
+  EXPECT_THAT_ERROR((*RM)->redirect(*JD, {{RedirectableSymbol, MiddleTarget}}),
+                    Succeeded());
+  Result = RTPtr();
+  EXPECT_EQ(Result, 13) << "Failed to call middle redirected target";
+
+  EXPECT_THAT_ERROR((*RM)->redirect(*JD, {{RedirectableSymbol, FinalTarget}}),
+                    Succeeded());
+  Result = RTPtr();
+  EXPECT_EQ(Result, 53) << "Failed to call redirected target";
+}
diff --git a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4f0850022cf1e9c2d4239423b52614858ad77ba1
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
@@ -0,0 +1,165 @@
+#include "llvm/ExecutionEngine/Orc/ReOptimizeLayer.h"
+#include "OrcTestCommon.h"
+#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
+#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
+#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/ExecutionEngine/Orc/IRPartitionLayer.h"
+#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
+#include "llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h"
+#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/Support/CodeGen.h"
+#include "llvm/TargetParser/Host.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::jitlink;
+
+class ReOptimizeLayerTest : public testing::Test {
+public:
+  ~ReOptimizeLayerTest() {
+    if (ES)
+      if (auto Err = ES->endSession())
+        ES->reportError(std::move(Err));
+  }
+
+protected:
+  void SetUp() override {
+    auto JTMB = JITTargetMachineBuilder::detectHost();
+    // Bail out if we can not detect the host.
+    if (!JTMB) {
+      consumeError(JTMB.takeError());
+      GTEST_SKIP();
+    }
+
+    // COFF-ARM64 is not supported yet
+    auto Triple = JTMB->getTargetTriple();
+    if (Triple.isOSBinFormatCOFF() && Triple.isAArch64())
+      GTEST_SKIP();
+
+    if (Triple.isPPC())
+      GTEST_SKIP();
+
+    auto EPC = SelfExecutorProcessControl::Create();
+    if (!EPC) {
+      consumeError(EPC.takeError());
+      GTEST_SKIP();
+    }
+
+    auto DLOrErr = JTMB->getDefaultDataLayoutForTarget();
+    if (!DLOrErr) {
+      consumeError(DLOrErr.takeError());
+      GTEST_SKIP();
+    }
+    ES = std::make_unique<ExecutionSession>(std::move(*EPC));
+    JD = &ES->createBareJITDylib("main");
+    ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(
+        *ES, std::make_unique<InProcessMemoryManager>(16384));
+    DL = std::make_unique<DataLayout>(std::move(*DLOrErr));
+
+    auto TM = JTMB->createTargetMachine();
+    if (!TM) {
+      consumeError(TM.takeError());
+      GTEST_SKIP();
+    }
+    auto CompileFunction =
+        std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
+    CompileLayer = std::make_unique<IRCompileLayer>(*ES, *ObjLinkingLayer,
+                                                    std::move(CompileFunction));
+  }
+
+  Error addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
+    assert(TSM && "Can not add null module");
+
+    TSM.withModuleDo([&](Module &M) { M.setDataLayout(*DL); });
+
+    return ROLayer->add(std::move(RT), std::move(TSM));
+  }
+
+  JITDylib *JD{nullptr};
+  std::unique_ptr<ExecutionSession> ES;
+  std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer;
+  std::unique_ptr<IRCompileLayer> CompileLayer;
+  std::unique_ptr<ReOptimizeLayer> ROLayer;
+  std::unique_ptr<DataLayout> DL;
+};
+
+static Function *createRetFunction(Module *M, StringRef Name,
+                                   uint32_t ReturnCode) {
+  Function *Result = Function::Create(
+      FunctionType::get(Type::getInt32Ty(M->getContext()), {}, false),
+      GlobalValue::ExternalLinkage, Name, M);
+
+  BasicBlock *BB = BasicBlock::Create(M->getContext(), Name, Result);
+  IRBuilder<> Builder(M->getContext());
+  Builder.SetInsertPoint(BB);
+
+  Value *RetValue = ConstantInt::get(M->getContext(), APInt(32, ReturnCode));
+  Builder.CreateRet(RetValue);
+  return Result;
+}
+
+TEST_F(ReOptimizeLayerTest, BasicReOptimization) {
+  MangleAndInterner Mangle(*ES, *DL);
+
+  auto &EPC = ES->getExecutorProcessControl();
+  EXPECT_THAT_ERROR(JD->define(absoluteSymbols(
+                        {{Mangle("__orc_rt_jit_dispatch"),
+                          {EPC.getJITDispatchInfo().JITDispatchFunction,
+                           JITSymbolFlags::Exported}},
+                         {Mangle("__orc_rt_jit_dispatch_ctx"),
+                          {EPC.getJITDispatchInfo().JITDispatchContext,
+                           JITSymbolFlags::Exported}},
+                         {Mangle("__orc_rt_reoptimize_tag"),
+                          {ExecutorAddr(), JITSymbolFlags::Exported}}})),
+                    Succeeded());
+
+  auto RM = JITLinkRedirectableSymbolManager::Create(*ObjLinkingLayer, *JD);
+  EXPECT_THAT_ERROR(RM.takeError(), Succeeded());
+
+  ROLayer = std::make_unique<ReOptimizeLayer>(*ES, *DL, *CompileLayer, **RM);
+  ROLayer->setReoptimizeFunc(
+      [&](ReOptimizeLayer &Parent,
+          ReOptimizeLayer::ReOptMaterializationUnitID MUID, unsigned CurVerison,
+          ResourceTrackerSP OldRT, ThreadSafeModule &TSM) {
+        TSM.withModuleDo([&](Module &M) {
+          for (auto &F : M) {
+            if (F.isDeclaration())
+              continue;
+            for (auto &B : F) {
+              for (auto &I : B) {
+                if (ReturnInst *Ret = dyn_cast<ReturnInst>(&I)) {
+                  Value *RetValue =
+                      ConstantInt::get(M.getContext(), APInt(32, 53));
+                  Ret->setOperand(0, RetValue);
+                }
+              }
+            }
+          }
+        });
+        return Error::success();
+      });
+  EXPECT_THAT_ERROR(ROLayer->reigsterRuntimeFunctions(*JD), Succeeded());
+
+  ThreadSafeContext Ctx(std::make_unique<LLVMContext>());
+  auto M = std::make_unique<Module>("<main>", *Ctx.getContext());
+  M->setTargetTriple(sys::getProcessTriple());
+
+  (void)createRetFunction(M.get(), "main", 42);
+
+  EXPECT_THAT_ERROR(addIRModule(JD->getDefaultResourceTracker(),
+                                ThreadSafeModule(std::move(M), std::move(Ctx))),
+                    Succeeded());
+
+  auto Result = cantFail(ES->lookup({JD}, Mangle("main")));
+  auto FuncPtr = Result.getAddress().toPtr<int (*)()>();
+  for (size_t I = 0; I <= ReOptimizeLayer::CallCountThreshold; I++)
+    EXPECT_EQ(FuncPtr(), 42);
+  EXPECT_EQ(FuncPtr(), 53);
+}
diff --git a/llvm/unittests/IR/BasicBlockTest.cpp b/llvm/unittests/IR/BasicBlockTest.cpp
index eea2746a352aa62068058e949e5090d86d379c2e..88ac6611742ce903bc8f31f690282ff36746076b 100644
--- a/llvm/unittests/IR/BasicBlockTest.cpp
+++ b/llvm/unittests/IR/BasicBlockTest.cpp
@@ -109,8 +109,10 @@ TEST(BasicBlockTest, TestInstructionsWithoutDebug) {
   Argument *V = new Argument(Type::getInt32Ty(Ctx));
   Function *F = Function::Create(FT, Function::ExternalLinkage, "", M);
 
-  Function *DbgDeclare = Intrinsic::getDeclaration(M, Intrinsic::dbg_declare);
-  Function *DbgValue = Intrinsic::getDeclaration(M, Intrinsic::dbg_value);
+  Function *DbgDeclare =
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_declare);
+  Function *DbgValue =
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_value);
   Value *DIV = MetadataAsValue::get(Ctx, (Metadata *)nullptr);
   SmallVector<Value *, 3> Args = {DIV, DIV, DIV};
 
@@ -174,7 +176,7 @@ class InstrOrderInvalidationTest : public ::testing::Test {
 protected:
   void SetUp() override {
     M.reset(new Module("MyModule", Ctx));
-    Nop = Intrinsic::getDeclaration(M.get(), Intrinsic::donothing);
+    Nop = Intrinsic::getOrInsertDeclaration(M.get(), Intrinsic::donothing);
     FunctionType *FT = FunctionType::get(Type::getVoidTy(Ctx), {}, false);
     Function *F = Function::Create(FT, Function::ExternalLinkage, "foo", *M);
     BB = BasicBlock::Create(Ctx, "entry", F);
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 953df224e84dcb818b9d1f67fc07170c96653bb4..ea20c87d6b09b420aded671345a9faca4e94b083 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -693,7 +693,8 @@ TEST(IRBuilder, GetSetInsertionPointWithEmptyBasicBlock) {
   std::unique_ptr<BasicBlock> BB(BasicBlock::Create(C, "start"));
   Module *M = new Module("module", C);
   IRBuilder<> Builder(BB.get());
-  Function *DbgDeclare = Intrinsic::getDeclaration(M, Intrinsic::dbg_declare);
+  Function *DbgDeclare =
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::dbg_declare);
   Value *DIV = MetadataAsValue::get(C, (Metadata *)nullptr);
   SmallVector<Value *, 3> Args = {DIV, DIV, DIV};
   Builder.CreateCall(DbgDeclare, Args);
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index d5239f21147cdb34d079559c3666b3a7f1d47611..690af62d18020d7ee5fef6547baba812cac23cbe 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -413,8 +413,9 @@ TEST_F(IRBuilderTest, ConstrainedFPIntrinsics) {
 
   Builder.setDefaultConstrainedExcept(fp::ebStrict);
   Builder.setDefaultConstrainedRounding(RoundingMode::TowardZero);
-  Function *Fn = Intrinsic::getDeclaration(M.get(),
-      Intrinsic::experimental_constrained_roundeven, { Type::getDoubleTy(Ctx) });
+  Function *Fn = Intrinsic::getOrInsertDeclaration(
+      M.get(), Intrinsic::experimental_constrained_roundeven,
+      {Type::getDoubleTy(Ctx)});
   V = Builder.CreateConstrainedFPCall(Fn, { VDouble });
   CII = cast<ConstrainedFPIntrinsic>(V);
   EXPECT_EQ(Intrinsic::experimental_constrained_roundeven, CII->getIntrinsicID());
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index 0c4af28a2ab57ba276633e488429d6c99f07d2dd..7fe0bd79b80a6057c16b261b7093e3042bdcfb8c 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -50,7 +50,7 @@ public:
   Instruction *makeIntrinsic(Intrinsic::ID ID) const {
     IRBuilder<> Builder(BB);
     SmallVector<Value *, 4> ProcessedArgs;
-    auto *Decl = Intrinsic::getDeclaration(M.get(), ID);
+    auto *Decl = Intrinsic::getOrInsertDeclaration(M.get(), ID);
     for (auto *Ty : Decl->getFunctionType()->params()) {
       auto *Val = Constant::getNullValue(Ty);
       ProcessedArgs.push_back(Val);
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index 13f121a2b9c7dd02a346a0b655243c4e70064e98..7dc4b9f448d386f6915bdb860df179399bed28f7 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -1766,7 +1766,7 @@ TEST_F(PatternMatchTest, IntrinsicMatcher) {
   Value *Ops[] = {Name, Hash, Num, Index, Step};
   Module *M = BB->getParent()->getParent();
   Function *TheFn =
-      Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step);
+      Intrinsic::getOrInsertDeclaration(M, Intrinsic::instrprof_increment_step);
 
   Value *Intrinsic5 = CallInst::Create(TheFn, Ops, "", BB);
 
diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp
index 0e6c38a8ec46ad189cce2761a9fb7070bb711abb..acc706b9119a47e4c32959bb25716269f0b82830 100644
--- a/llvm/unittests/IR/VPIntrinsicTest.cpp
+++ b/llvm/unittests/IR/VPIntrinsicTest.cpp
@@ -436,7 +436,7 @@ TEST_F(VPIntrinsicTest, VPToNonPredIntrinsicRoundTrip) {
   ASSERT_TRUE(IsFullTrip);
 }
 
-/// Check that VPIntrinsic::getDeclarationForParams works.
+/// Check that VPIntrinsic::getOrInsertDeclarationForParams works.
 TEST_F(VPIntrinsicTest, VPIntrinsicDeclarationForParams) {
   std::unique_ptr<Module> M = createVPDeclarationModule();
   assert(M);
@@ -452,7 +452,7 @@ TEST_F(VPIntrinsicTest, VPIntrinsicDeclarationForParams) {
       Values.push_back(UndefValue::get(ParamTy));
 
     ASSERT_NE(F.getIntrinsicID(), Intrinsic::not_intrinsic);
-    auto *NewDecl = VPIntrinsic::getDeclarationForParams(
+    auto *NewDecl = VPIntrinsic::getOrInsertDeclarationForParams(
         OutM.get(), F.getIntrinsicID(), FuncTy->getReturnType(), Values);
     ASSERT_TRUE(NewDecl);
 
diff --git a/llvm/unittests/SandboxIR/RegionTest.cpp b/llvm/unittests/SandboxIR/RegionTest.cpp
index 602534530e248ce11d9db46eea2b5cac231088bb..a2efe551c8ff2cbf8b3dbcd4ab5c6b441d37db0e 100644
--- a/llvm/unittests/SandboxIR/RegionTest.cpp
+++ b/llvm/unittests/SandboxIR/RegionTest.cpp
@@ -108,6 +108,31 @@ define i8 @foo(i8 %v0, i8 %v1) {
   EXPECT_THAT(Regions[1]->insts(), testing::UnorderedElementsAre(T1, T2));
 }
 
+TEST_F(RegionTest, NonContiguousRegion) {
+  parseIR(C, R"IR(
+define i8 @foo(i8 %v0, i8 %v1) {
+  %t0 = add i8 %v0, 1, !sandboxvec !0
+  %t1 = add i8 %t0, %v1
+  %t2 = add i8 %t1, %v1, !sandboxvec !0
+  ret i8 %t2
+}
+
+!0 = distinct !{!"sandboxregion"}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *T0 = cast<sandboxir::Instruction>(&*It++);
+  [[maybe_unused]] auto *T1 = cast<sandboxir::Instruction>(&*It++);
+  auto *T2 = cast<sandboxir::Instruction>(&*It++);
+
+  SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
+      sandboxir::Region::createRegionsFromMD(*F);
+  EXPECT_THAT(Regions[0]->insts(), testing::UnorderedElementsAre(T0, T2));
+}
+
 TEST_F(RegionTest, DumpedMetadata) {
   parseIR(C, R"IR(
 define i8 @foo(i8 %v0, i8 %v1) {
diff --git a/llvm/unittests/SandboxIR/UtilsTest.cpp b/llvm/unittests/SandboxIR/UtilsTest.cpp
index 90396eaa53ab38084ac7656cc76d02ebec1f43f5..a30fc253a1a7423b23a3d379babc94b58cd4e60c 100644
--- a/llvm/unittests/SandboxIR/UtilsTest.cpp
+++ b/llvm/unittests/SandboxIR/UtilsTest.cpp
@@ -215,3 +215,35 @@ bb0:
   EXPECT_EQ(sandboxir::Utils::getNumBits(L2), 8u);
   EXPECT_EQ(sandboxir::Utils::getNumBits(L3), 64u);
 }
+
+TEST_F(UtilsTest, GetMemBase) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptrA, float %val, ptr %ptrB) {
+bb:
+  %gepA0 = getelementptr float, ptr %ptrA, i32 0
+  %gepA1 = getelementptr float, ptr %ptrA, i32 1
+  %gepB0 = getelementptr float, ptr %ptrB, i32 0
+  %gepB1 = getelementptr float, ptr %ptrB, i32 1
+  store float %val, ptr %gepA0
+  store float %val, ptr %gepA1
+  store float %val, ptr %gepB0
+  store float %val, ptr %gepB1
+  ret void
+}
+)IR");
+  llvm::Function &Foo = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(&Foo);
+
+  auto It = std::next(F->begin()->begin(), 4);
+  auto *St0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *St1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *St2 = cast<sandboxir::StoreInst>(&*It++);
+  auto *St3 = cast<sandboxir::StoreInst>(&*It++);
+  EXPECT_EQ(sandboxir::Utils::getMemInstructionBase(St0),
+            sandboxir::Utils::getMemInstructionBase(St1));
+  EXPECT_EQ(sandboxir::Utils::getMemInstructionBase(St2),
+            sandboxir::Utils::getMemInstructionBase(St3));
+  EXPECT_NE(sandboxir::Utils::getMemInstructionBase(St0),
+            sandboxir::Utils::getMemInstructionBase(St3));
+}
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 5b5d45f6c574bfad0b7c1d41c77d640d18c5fd27..87b78d502780d1fd420955f0f5376034d84b2d45 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -490,6 +490,9 @@ INSTANTIATE_TEST_SUITE_P(
         ARMCPUTestParams<uint64_t>("cortex-m33", "armv8-m.main", "fpv5-sp-d16",
                                    ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP,
                                    "8-M.Mainline"),
+        ARMCPUTestParams<uint64_t>("star-mc1", "armv8-m.main", "fpv5-sp-d16",
+                                   ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP,
+                                   "8-M.Mainline"),
         ARMCPUTestParams<uint64_t>("cortex-m35p", "armv8-m.main", "fpv5-sp-d16",
                                    ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP,
                                    "8-M.Mainline"),
@@ -518,7 +521,7 @@ INSTANTIATE_TEST_SUITE_P(
                                    "7-S")),
     ARMCPUTestParams<uint64_t>::PrintToStringParamName);
 
-static constexpr unsigned NumARMCPUArchs = 92;
+static constexpr unsigned NumARMCPUArchs = 93;
 
 TEST(TargetParserTest, testARMCPUArchList) {
   SmallVector<StringRef, NumARMCPUArchs> List;
diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp
index f4d53e41b6c3a7006f741ddd9af7f686dc8201f0..cb2fa95d431c66f76da817dcedc46efa342f900e 100644
--- a/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/llvm/unittests/TargetParser/TripleTest.cpp
@@ -93,6 +93,13 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::Hurd, T.getOS());
   EXPECT_EQ(Triple::GNU, T.getEnvironment());
 
+  T = Triple("i686-pc-linux-gnu");
+  EXPECT_EQ(Triple::x86, T.getArch());
+  EXPECT_EQ(Triple::PC, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::GNU, T.getEnvironment());
+  EXPECT_FALSE(T.isTime64ABI());
+
   T = Triple("x86_64-pc-linux-gnu");
   EXPECT_EQ(Triple::x86_64, T.getArch());
   EXPECT_EQ(Triple::PC, T.getVendor());
@@ -187,24 +194,52 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
   EXPECT_EQ(Triple::UnknownOS, T.getOS());
   EXPECT_EQ(Triple::EABI, T.getEnvironment());
+  EXPECT_FALSE(T.isHardFloatABI());
+
+  T = Triple("arm-none-none-eabihf");
+  EXPECT_EQ(Triple::arm, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::UnknownOS, T.getOS());
+  EXPECT_EQ(Triple::EABIHF, T.getEnvironment());
+  EXPECT_TRUE(T.isHardFloatABI());
 
   T = Triple("arm-none-linux-musleabi");
   EXPECT_EQ(Triple::arm, T.getArch());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::MuslEABI, T.getEnvironment());
+  EXPECT_FALSE(T.isHardFloatABI());
+
+  T = Triple("arm-none-linux-musleabihf");
+  EXPECT_EQ(Triple::arm, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::MuslEABIHF, T.getEnvironment());
+  EXPECT_TRUE(T.isHardFloatABI());
 
   T = Triple("armv6hl-none-linux-gnueabi");
   EXPECT_EQ(Triple::arm, T.getArch());
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
   EXPECT_EQ(Triple::GNUEABI, T.getEnvironment());
+  EXPECT_FALSE(T.isTime64ABI());
+  EXPECT_FALSE(T.isHardFloatABI());
 
   T = Triple("armv7hl-none-linux-gnueabi");
   EXPECT_EQ(Triple::arm, T.getArch());
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
   EXPECT_EQ(Triple::GNUEABI, T.getEnvironment());
+  EXPECT_FALSE(T.isTime64ABI());
+  EXPECT_FALSE(T.isHardFloatABI());
+
+  T = Triple("armv7hl-none-linux-gnueabihf");
+  EXPECT_EQ(Triple::arm, T.getArch());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::GNUEABIHF, T.getEnvironment());
+  EXPECT_FALSE(T.isTime64ABI());
+  EXPECT_TRUE(T.isHardFloatABI());
 
   T = Triple("amdil-unknown-unknown");
   EXPECT_EQ(Triple::amdil, T.getArch());
@@ -1283,6 +1318,29 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::PAuthTest, T.getEnvironment());
 
+  // Gentoo time64 triples
+  T = Triple("i686-pc-linux-gnut64");
+  EXPECT_EQ(Triple::x86, T.getArch());
+  EXPECT_EQ(Triple::PC, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::GNUT64, T.getEnvironment());
+  EXPECT_TRUE(T.isTime64ABI());
+
+  T = Triple("armv5tel-softfloat-linux-gnueabit64");
+  EXPECT_EQ(Triple::arm, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::GNUEABIT64, T.getEnvironment());
+  EXPECT_TRUE(T.isTime64ABI());
+
+  T = Triple("armv7a-unknown-linux-gnueabihft64");
+  EXPECT_EQ(Triple::arm, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::GNUEABIHFT64, T.getEnvironment());
+  EXPECT_TRUE(T.isTime64ABI());
+  EXPECT_TRUE(T.isHardFloatABI());
+
   T = Triple("huh");
   EXPECT_EQ(Triple::UnknownArch, T.getArch());
 }
@@ -2410,6 +2468,10 @@ TEST(TripleTest, NormalizeWindows) {
             Triple::normalize("i686-pc-windows-elf-elf"));
 
   EXPECT_TRUE(Triple("x86_64-pc-win32").isWindowsMSVCEnvironment());
+
+  EXPECT_TRUE(Triple(Triple::normalize("mipsel-windows-msvccoff")).isOSBinFormatCOFF());
+  EXPECT_TRUE(Triple(Triple::normalize("mipsel-windows-msvc")).isOSBinFormatCOFF());
+  EXPECT_TRUE(Triple(Triple::normalize("mipsel-windows-gnu")).isOSBinFormatCOFF());
 }
 
 TEST(TripleTest, NormalizeAndroid) {
diff --git a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
index 9ccb13934cbd3811ff72df46a391531442c5361a..a4c076a8752fc35c8f83c2da7d8a386d55c23bf8 100644
--- a/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/PGOInstrumentationTest.cpp
@@ -103,9 +103,13 @@ public:
                ModuleAnalysisManager::Invalidator &));
 };
 
-struct PGOInstrumentationGenTest
-    : public Test,
-      WithParamInterface<std::tuple<StringRef, StringRef>> {
+template <typename ParamType> struct PGOTestName {
+  std::string operator()(const TestParamInfo<ParamType> &Info) const {
+    return std::get<1>(Info.param).str();
+  }
+};
+
+struct PGOInstrumentationGenTest : public Test {
   ModulePassManager MPM;
   PassBuilder PB;
   MockModuleAnalysisHandle MMAHandle;
@@ -141,12 +145,47 @@ struct PGOInstrumentationGenTest
   }
 };
 
+struct PGOInstrumentationGenInstrumentTest
+    : PGOInstrumentationGenTest,
+      WithParamInterface<std::tuple<StringRef, StringRef>> {};
+
 static constexpr StringRef CodeWithFuncDefs = R"(
   define i32 @f(i32 %n) {
   entry:
     ret i32 0
   })";
 
+INSTANTIATE_TEST_SUITE_P(
+    PGOInstrumetationGenTestSuite, PGOInstrumentationGenInstrumentTest,
+    Values(std::make_tuple(CodeWithFuncDefs, "instrument_function_defs")),
+    PGOTestName<PGOInstrumentationGenInstrumentTest::ParamType>());
+
+TEST_P(PGOInstrumentationGenInstrumentTest, Instrumented) {
+  const StringRef Code = std::get<0>(GetParam());
+  parseAssembly(Code);
+
+  ASSERT_THAT(M, NotNull());
+
+  Sequence PassSequence;
+  EXPECT_CALL(MMAHandle, run(Ref(*M), _))
+      .InSequence(PassSequence)
+      .WillOnce(DoDefault());
+  EXPECT_CALL(MMAHandle, invalidate(Ref(*M), _, _))
+      .InSequence(PassSequence)
+      .WillOnce(DoDefault());
+
+  MPM.run(*M, MAM);
+
+  const auto *IRInstrVar =
+      M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
+  ASSERT_THAT(IRInstrVar, NotNull());
+  EXPECT_FALSE(IRInstrVar->isDeclaration());
+}
+
+struct PGOInstrumentationGenIgnoreTest
+    : PGOInstrumentationGenTest,
+      WithParamInterface<std::tuple<StringRef, StringRef>> {};
+
 static constexpr StringRef CodeWithFuncDecls = R"(
   declare i32 @f(i32);
 )";
@@ -157,33 +196,26 @@ static constexpr StringRef CodeWithGlobals = R"(
 )";
 
 INSTANTIATE_TEST_SUITE_P(
-    PGOInstrumetationGenTestSuite, PGOInstrumentationGenTest,
-    Values(std::make_tuple(CodeWithFuncDefs, "instrument_function_defs"),
-           std::make_tuple(CodeWithFuncDecls, "instrument_function_decls"),
+    PGOInstrumetationGenIgnoreTestSuite, PGOInstrumentationGenIgnoreTest,
+    Values(std::make_tuple(CodeWithFuncDecls, "instrument_function_decls"),
            std::make_tuple(CodeWithGlobals, "instrument_globals")),
-    [](const TestParamInfo<PGOInstrumentationGenTest::ParamType> &Info) {
-      return std::get<1>(Info.param).str();
-    });
+    PGOTestName<PGOInstrumentationGenIgnoreTest::ParamType>());
 
-TEST_P(PGOInstrumentationGenTest, Instrumented) {
+TEST_P(PGOInstrumentationGenIgnoreTest, NotInstrumented) {
   const StringRef Code = std::get<0>(GetParam());
+
   parseAssembly(Code);
 
   ASSERT_THAT(M, NotNull());
 
-  Sequence PassSequence;
-  EXPECT_CALL(MMAHandle, run(Ref(*M), _))
-      .InSequence(PassSequence)
-      .WillOnce(DoDefault());
-  EXPECT_CALL(MMAHandle, invalidate(Ref(*M), _, _))
-      .InSequence(PassSequence)
-      .WillOnce(DoDefault());
+  EXPECT_CALL(MMAHandle, run(Ref(*M), _)).WillOnce(DoDefault());
+  EXPECT_CALL(MMAHandle, invalidate(Ref(*M), _, _)).Times(0);
 
   MPM.run(*M, MAM);
 
   const auto *IRInstrVar =
       M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
-  EXPECT_THAT(IRInstrVar, NotNull());
+  ASSERT_THAT(IRInstrVar, NotNull());
   EXPECT_FALSE(IRInstrVar->isDeclaration());
 }
 
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
index 7e2be25fa25ae654a78c04514fae9cdcd342abde..3f84ad1f731de8f245adfdc46e2056cc7aa9487b 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -249,6 +249,11 @@ define void @foo(ptr %ptr, i8 %v0, i8 %v1) {
   EXPECT_TRUE(N0->memPreds().empty());
   EXPECT_THAT(N1->memPreds(), testing::ElementsAre(N0));
   EXPECT_TRUE(N2->preds(DAG).empty());
+
+  // Check UnscheduledSuccs.
+  EXPECT_EQ(N0->getNumUnscheduledSuccs(), 1u); // N1
+  EXPECT_EQ(N1->getNumUnscheduledSuccs(), 0u);
+  EXPECT_EQ(N2->getNumUnscheduledSuccs(), 0u);
 }
 
 TEST_F(DependencyGraphTest, Preds) {
@@ -286,6 +291,14 @@ define i8 @foo(i8 %v0, i8 %v1) {
   EXPECT_THAT(StN->preds(DAG),
               testing::UnorderedElementsAre(CallN, CallN, AddN2));
   EXPECT_THAT(RetN->preds(DAG), testing::ElementsAre(AddN2));
+
+  // Check UnscheduledSuccs.
+  EXPECT_EQ(AddN0->getNumUnscheduledSuccs(), 1u); // AddN2
+  EXPECT_EQ(AddN1->getNumUnscheduledSuccs(), 2u); // AddN2, CallN
+  EXPECT_EQ(AddN2->getNumUnscheduledSuccs(), 2u); // StN, RetN
+  EXPECT_EQ(CallN->getNumUnscheduledSuccs(), 2u); // StN, StN
+  EXPECT_EQ(StN->getNumUnscheduledSuccs(), 0u);
+  EXPECT_EQ(RetN->getNumUnscheduledSuccs(), 0u);
 }
 
 TEST_F(DependencyGraphTest, MemDGNode_getPrevNode_getNextNode) {
@@ -681,3 +694,83 @@ define void @foo() {
   EXPECT_FALSE(memDependency(StackSaveN, AllocaN));
   EXPECT_FALSE(memDependency(AllocaN, StackRestoreN));
 }
+
+TEST_F(DependencyGraphTest, Extend) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %v1, i8 %v2, i8 %v3, i8 %v4, i8 %v5) {
+  store i8 %v1, ptr %ptr
+  store i8 %v2, ptr %ptr
+  store i8 %v3, ptr %ptr
+  store i8 %v4, ptr %ptr
+  store i8 %v5, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *S1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S2 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S3 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S4 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S5 = cast<sandboxir::StoreInst>(&*It++);
+  sandboxir::DependencyGraph DAG(getAA(*LLVMF));
+  {
+    // Scenario 1: Build new DAG
+    auto NewIntvl = DAG.extend({S3, S3});
+    EXPECT_EQ(NewIntvl, sandboxir::Interval<sandboxir::Instruction>(S3, S3));
+    EXPECT_EQ(DAG.getInterval().top(), S3);
+    EXPECT_EQ(DAG.getInterval().bottom(), S3);
+    [[maybe_unused]] auto *S3N = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
+    // Check UnscheduledSuccs.
+    EXPECT_EQ(S3N->getNumUnscheduledSuccs(), 0u);
+  }
+  {
+    // Scenario 2: Extend below
+    auto NewIntvl = DAG.extend({S5, S5});
+    EXPECT_EQ(NewIntvl, sandboxir::Interval<sandboxir::Instruction>(S4, S5));
+    auto *S3N = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
+    auto *S4N = cast<sandboxir::MemDGNode>(DAG.getNode(S4));
+    auto *S5N = cast<sandboxir::MemDGNode>(DAG.getNode(S5));
+    EXPECT_TRUE(S4N->hasMemPred(S3N));
+    EXPECT_TRUE(S5N->hasMemPred(S4N));
+    EXPECT_TRUE(S5N->hasMemPred(S3N));
+    // Check UnscheduledSuccs.
+    EXPECT_EQ(S3N->getNumUnscheduledSuccs(), 2u); // S4N, S5N
+    EXPECT_EQ(S4N->getNumUnscheduledSuccs(), 1u); // S5N
+    EXPECT_EQ(S5N->getNumUnscheduledSuccs(), 0u);
+  }
+  {
+    // Scenario 3: Extend above
+    auto NewIntvl = DAG.extend({S1, S2});
+    EXPECT_EQ(NewIntvl, sandboxir::Interval<sandboxir::Instruction>(S1, S2));
+    auto *S1N = cast<sandboxir::MemDGNode>(DAG.getNode(S1));
+    auto *S2N = cast<sandboxir::MemDGNode>(DAG.getNode(S2));
+    auto *S3N = cast<sandboxir::MemDGNode>(DAG.getNode(S3));
+    auto *S4N = cast<sandboxir::MemDGNode>(DAG.getNode(S4));
+    auto *S5N = cast<sandboxir::MemDGNode>(DAG.getNode(S5));
+
+    EXPECT_TRUE(S2N->hasMemPred(S1N));
+
+    EXPECT_TRUE(S3N->hasMemPred(S2N));
+    EXPECT_TRUE(S3N->hasMemPred(S1N));
+
+    EXPECT_TRUE(S4N->hasMemPred(S3N));
+    EXPECT_TRUE(S4N->hasMemPred(S2N));
+    EXPECT_TRUE(S4N->hasMemPred(S1N));
+
+    EXPECT_TRUE(S5N->hasMemPred(S4N));
+    EXPECT_TRUE(S5N->hasMemPred(S3N));
+    EXPECT_TRUE(S5N->hasMemPred(S2N));
+    EXPECT_TRUE(S5N->hasMemPred(S1N));
+
+    // Check UnscheduledSuccs.
+    EXPECT_EQ(S1N->getNumUnscheduledSuccs(), 4u); // S2N, S3N, S4N, S5N
+    EXPECT_EQ(S2N->getNumUnscheduledSuccs(), 3u); // S3N, S4N, S5N
+    EXPECT_EQ(S3N->getNumUnscheduledSuccs(), 2u); // S4N, S5N
+    EXPECT_EQ(S4N->getNumUnscheduledSuccs(), 1u); // S5N
+    EXPECT_EQ(S5N->getNumUnscheduledSuccs(), 0u);
+  }
+}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
index ee461e48f0dc03515f8b6b6dd5cc1e0d810bf258..b04e4fc7cffcae4568a1aadac5ca71218c78308e 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
@@ -123,6 +123,25 @@ define void @foo(i8 %v0) {
     EXPECT_FALSE(Intvl1.disjoint(Intvl3));
     EXPECT_TRUE(Intvl1.disjoint(Empty));
   }
+  {
+    // Check comesBefore().
+    sandboxir::Interval<sandboxir::Instruction> Intvl1(I0, I0);
+    sandboxir::Interval<sandboxir::Instruction> Intvl2(I2, I2);
+    EXPECT_TRUE(Intvl1.comesBefore(Intvl2));
+    EXPECT_FALSE(Intvl2.comesBefore(Intvl1));
+
+    sandboxir::Interval<sandboxir::Instruction> Intvl12(I1, I2);
+    EXPECT_TRUE(Intvl1.comesBefore(Intvl12));
+    EXPECT_FALSE(Intvl12.comesBefore(Intvl1));
+    {
+#ifndef NDEBUG
+      // Check comesBefore() with non-disjoint intervals.
+      sandboxir::Interval<sandboxir::Instruction> Intvl1(I0, I2);
+      sandboxir::Interval<sandboxir::Instruction> Intvl2(I2, I2);
+      EXPECT_DEATH(Intvl1.comesBefore(Intvl2), ".*disjoint.*");
+#endif // NDEBUG
+    }
+  }
 }
 
 // Helper function for returning a vector of instruction pointers from a range
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
index dd41b0a6605095e8ae512d75e5e7708f5345b81e..82b230d50c4ec9538b52ac7963aaa83a2c78ea96 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
@@ -40,6 +40,15 @@ struct SeedBundleTest : public testing::Test {
   }
 };
 
+// Stub class to make the abstract base class testable.
+class SeedBundleForTest : public sandboxir::SeedBundle {
+public:
+  using sandboxir::SeedBundle::SeedBundle;
+  void insert(sandboxir::Instruction *I, ScalarEvolution &SE) override {
+    insertAt(Seeds.end(), I);
+  }
+};
+
 TEST_F(SeedBundleTest, SeedBundle) {
   parseIR(C, R"IR(
 define void @foo(float %v0, i32 %i0, i16 %i1, i8 %i2) {
@@ -66,7 +75,7 @@ bb:
   // Assume first two instructions are identical in the number of bits.
   const unsigned IOBits = sandboxir::Utils::getNumBits(I0, DL);
   // Constructor
-  sandboxir::SeedBundle SBO(I0);
+  SeedBundleForTest SBO(I0);
   EXPECT_EQ(*SBO.begin(), I0);
   // getNumUnusedBits after constructor
   EXPECT_EQ(SBO.getNumUnusedBits(), IOBits);
@@ -103,7 +112,7 @@ bb:
   EXPECT_EQ(BundleBits, 88u);
   auto Seeds = Insts;
   // Constructor
-  sandboxir::SeedBundle SB1(std::move(Seeds));
+  SeedBundleForTest SB1(std::move(Seeds));
   // getNumUnusedBits after constructor
   EXPECT_EQ(SB1.getNumUnusedBits(), BundleBits);
   // setUsed with index
@@ -196,3 +205,66 @@ bb:
   sandboxir::LoadSeedBundle LB(std::move(Loads), SE);
   EXPECT_THAT(LB, testing::ElementsAre(L0, L1, L2, L3));
 }
+
+TEST_F(SeedBundleTest, Container) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptrA, float %val, ptr %ptrB) {
+bb:
+  %gepA0 = getelementptr float, ptr %ptrA, i32 0
+  %gepA1 = getelementptr float, ptr %ptrA, i32 1
+  %gepB0 = getelementptr float, ptr %ptrB, i32 0
+  %gepB1 = getelementptr float, ptr %ptrB, i32 1
+  store float %val, ptr %gepA0
+  store float %val, ptr %gepA1
+  store float %val, ptr %gepB0
+  store float %val, ptr %gepB1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+
+  DominatorTree DT(LLVMF);
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+  DataLayout DL(M->getDataLayout());
+  LoopInfo LI(DT);
+  AssumptionCache AC(LLVMF);
+  ScalarEvolution SE(LLVMF, TLI, AC, DT, LI);
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto &BB = *F.begin();
+  auto It = std::next(BB.begin(), 4);
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S2 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S3 = cast<sandboxir::StoreInst>(&*It++);
+  sandboxir::SeedContainer SC(SE);
+  // Check begin() end() when empty.
+  EXPECT_EQ(SC.begin(), SC.end());
+
+  SC.insert(S0);
+  SC.insert(S1);
+  SC.insert(S2);
+  SC.insert(S3);
+  unsigned Cnt = 0;
+  SmallVector<sandboxir::SeedBundle *> Bndls;
+  for (auto &SeedBndl : SC) {
+    EXPECT_EQ(SeedBndl.size(), 2u);
+    ++Cnt;
+    Bndls.push_back(&SeedBndl);
+  }
+  EXPECT_EQ(Cnt, 2u);
+
+  // Mark them "Used" to check if operator++ skips them in the next loop.
+  for (auto *SeedBndl : Bndls)
+    for (auto Lane : seq<unsigned>(SeedBndl->size()))
+      SeedBndl->setUsed(Lane);
+  // Check if iterator::operator++ skips used lanes.
+  Cnt = 0;
+  for (auto &SeedBndl : SC) {
+    (void)SeedBndl;
+    ++Cnt;
+  }
+  EXPECT_EQ(Cnt, 0u);
+}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 372c5aaea59382e82bc632b15699b45d2b19a1df..376b00224eb5740047a7136e797361af901f2994 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1195,7 +1195,8 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
     // Test for a call to a function without side-effects.
     LLVMContext C;
     Module M("", C);
-    Function *TheFn = Intrinsic::getDeclaration(&M, Intrinsic::thread_pointer);
+    Function *TheFn =
+        Intrinsic::getOrInsertDeclaration(&M, Intrinsic::thread_pointer);
 
     auto *Call = CallInst::Create(TheFn->getFunctionType(), TheFn);
     VPValue Op1;
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn
index e398119fb38a957d94855b1af27967f9ecece3af..4bc2ccc79c85a286f5391e2cdcfb5ec98551821c 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn
@@ -155,6 +155,7 @@ source_set("sources") {
     "sanitizer_symbolizer_win.cpp",
     "sanitizer_termination.cpp",
     "sanitizer_thread_arg_retval.cpp",
+    "sanitizer_thread_history.cpp",
     "sanitizer_thread_registry.cpp",
     "sanitizer_thread_registry.h",
     "sanitizer_thread_safety.h",
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 5f2e0dd90da072acc9b1802cae6e3a328314f8af..e5628a1d860419ca065152d6d5593cf72442c701 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -819,6 +819,7 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/common_type.h",
       "__type_traits/conditional.h",
       "__type_traits/conjunction.h",
+      "__type_traits/container_traits.h",
       "__type_traits/copy_cv.h",
       "__type_traits/copy_cvref.h",
       "__type_traits/datasizeof.h",
diff --git a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
index 5faa365bb7bdbcd965a977147235bb53d053354a..4221fab1e237b8047ca796a093bac39cb0ca784d 100644
--- a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
@@ -21,6 +21,7 @@ static_library("Utility") {
     "DataEncoder.cpp",
     "DataExtractor.cpp",
     "Diagnostics.cpp",
+    "DiagnosticsRendering.cpp",
     "Environment.cpp",
     "ErrorMessages.cpp",
     "Event.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index 3e1370fa6541da220ff793b9fda2b34c5c86a63b..25436c3cdb47d5e8edd2267b95321a393036cdde 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -33,8 +33,10 @@ static_library("Orc") {
     "ExecutionUtils.cpp",
     "ExecutorProcessControl.cpp",
     "IRCompileLayer.cpp",
+    "IRPartitionLayer.cpp",
     "IRTransformLayer.cpp",
     "IndirectionUtils.cpp",
+    "JITLinkRedirectableSymbolManager.cpp",
     "JITTargetMachineBuilder.cpp",
     "LLJIT.cpp",
     "Layer.cpp",
@@ -52,6 +54,8 @@ static_library("Orc") {
     "OrcABISupport.cpp",
     "OrcV2CBindings.cpp",
     "RTDyldObjectLinkingLayer.cpp",
+    "ReOptimizeLayer.cpp",
+    "RedirectionManager.cpp",
     "SectCreate.cpp",
     "SimpleRemoteEPC.cpp",
     "SpeculateAnalyses.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
index 7861aebc7bfdb0b055e687d180e18129f54e496c..e2e719425b08d6fd75e609545a7259f8285c228a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -141,6 +141,7 @@ static_library("LLVMRISCVCodeGen") {
     "RISCVTargetMachine.cpp",
     "RISCVTargetObjectFile.cpp",
     "RISCVTargetTransformInfo.cpp",
+    "RISCVVLOptimizer.cpp",
     "RISCVVectorPeephole.cpp",
     "RISCVZacasABIFix.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
index 357e5d91ee19ee72172fc0236f711fc266b8687d..376f68977ed6b9b908562132ac25b49ffcb7d9ee 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
@@ -21,6 +21,7 @@ unittest("OrcJITTests") {
     "ExecutionSessionWrapperFunctionCallsTest.cpp",
     "ExecutorAddressTest.cpp",
     "IndirectionUtilsTest.cpp",
+    "JITLinkRedirectionManagerTest.cpp",
     "JITTargetMachineBuilderTest.cpp",
     "LazyCallThroughAndReexportsTest.cpp",
     "LookupAndRecordAddrsTest.cpp",
@@ -33,6 +34,7 @@ unittest("OrcJITTests") {
     "OrcCAPITest.cpp",
     "OrcTestCommon.cpp",
     "RTDyldObjectLinkingLayerTest.cpp",
+    "ReOptimizeLayerTest.cpp",
     "ResourceTrackerTest.cpp",
     "SharedMemoryMapperTest.cpp",
     "SimpleExecutorMemoryManagerTest.cpp",
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index ed78256ee414b4b807032d851baf1dcee718a6b0..5ccae4be09679651bb39d9a26e43136a04299c95 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -168,7 +168,7 @@ def parse_args():
     execution_group.add_argument(
         "--resultdb-output",
         type=lit.reports.ResultDBReport,
-        help="Write LuCI ResuldDB compatible JSON to the specified file",
+        help="Write LuCI ResultDB compatible JSON to the specified file",
     )
     execution_group.add_argument(
         "--time-trace-output",
diff --git a/mlir/docs/DefiningDialects/Operations.md b/mlir/docs/DefiningDialects/Operations.md
index c252d9caf98eba3be0be1009aa5a50d101ac9673..d93c0e7146011ce25c0001cbc960f4eb77c075fb 100644
--- a/mlir/docs/DefiningDialects/Operations.md
+++ b/mlir/docs/DefiningDialects/Operations.md
@@ -756,7 +756,7 @@ The available directives are as follows:
 
 *   `ref ( input )`
 
-    -   Represents a reference to the a variable or directive, that must have
+    -   Represents a reference to a variable or directive, that must have
         already been resolved, that may be used as a parameter to a `custom`
         directive.
     -   Used to pass previously parsed entities to custom directives.
diff --git a/mlir/docs/DefiningDialects/_index.md b/mlir/docs/DefiningDialects/_index.md
index 83070b616d0ef7d22c84bf8402390a6ddfdb5718..5196092dfbda114be8a4a67e355a593f14abf883 100644
--- a/mlir/docs/DefiningDialects/_index.md
+++ b/mlir/docs/DefiningDialects/_index.md
@@ -133,7 +133,7 @@ void MyOp::setOtherValue(StringAttr newValue);
 
 ### Dependent Dialects
 
-MLIR has a very large ecosystem, and contains dialects that server many different purposes. It
+MLIR has a very large ecosystem, and contains dialects that serve many different purposes. It
 is quite common, given the above, that dialects may want to reuse certain components from other
 dialects. This may mean generating operations from those dialects during canonicalization, reusing
 attributes or types, etc. When a dialect has a dependency on another, i.e. when it constructs and/or
diff --git a/mlir/docs/Dialects/OpenMPDialect/_index.md b/mlir/docs/Dialects/OpenMPDialect/_index.md
index b4e359284edae69ecb6837681bf27f744c7517dd..3d28fe7819129f615c2d772babadca3ab21aae78 100644
--- a/mlir/docs/Dialects/OpenMPDialect/_index.md
+++ b/mlir/docs/Dialects/OpenMPDialect/_index.md
@@ -362,9 +362,10 @@ of accessing clause-defined block arguments.
 Loop-associated OpenMP constructs are represented in the dialect as loop wrapper
 operations. These implement the `LoopWrapperInterface`, which enforces a series
 of restrictions upon the operation:
-  - It contains a single region with a single block; and
-  - Its block contains exactly two operations: another loop wrapper or
-`omp.loop_nest` operation and a terminator.
+  - It has the `NoTerminator` and `SingleBlock` traits;
+  - It contains a single region; and
+  - Its only block contains exactly one operation, which must be another loop
+wrapper or `omp.loop_nest` operation.
 
 This approach splits the representation for a loop nest and the loop-associated
 constructs that specify how its iterations are executed, possibly across various
@@ -393,7 +394,6 @@ omp.parallel ... {
       store %sum, %c[%i] : memref<?xf32>
       omp.yield
     }
-    omp.terminator
   }
   ...
   omp.terminator
@@ -490,9 +490,7 @@ omp.distribute ... {
       ...
       omp.yield
     }
-    omp.terminator
   } {omp.composite}
-  omp.terminator
 } {omp.composite}
 ```
 
@@ -518,9 +516,7 @@ omp.parallel ... {
         ...
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   } {omp.composite}
   ...
   omp.terminator
diff --git a/mlir/docs/PDLL.md b/mlir/docs/PDLL.md
index f5651156a860ca0c0edbf786f6fff8e525ed2827..ee1b615a2dcdee4506731b74130ca177f8c133ec 100644
--- a/mlir/docs/PDLL.md
+++ b/mlir/docs/PDLL.md
@@ -5,8 +5,8 @@ writing pattern rewrites targeting MLIR.
 
 Note: This document assumes a familiarity with MLIR concepts; more specifically
 the concepts detailed within the
-[MLIR Pattern Rewriting](https://mlir.llvm.org/docs/PatternRewriter/) and
-[Operation Definition Specification (ODS)](https://mlir.llvm.org/docs/OpDefinitions/)
+[MLIR Pattern Rewriting](PatternRewriter.md) and
+[Operation Definition Specification (ODS)](DefiningDialects/Operations.md)
 documentation.
 
 [TOC]
@@ -34,11 +34,11 @@ that no longer exist.
 ### Why build a new language instead of improving TableGen DRR?
 
 Note: This section assumes familiarity with
-[TDRR](https://mlir.llvm.org/docs/DeclarativeRewrites/), please refer the
+[TDRR](DeclarativeRewrites.md), please refer the
 relevant documentation before continuing.
 
 Tablegen DRR (TDRR), i.e.
-[Table-driven Declarative Rewrite Rules](https://mlir.llvm.org/docs/DeclarativeRewrites/),
+[Table-driven Declarative Rewrite Rules](DeclarativeRewrites.md),
 is a declarative DSL for defining MLIR pattern rewrites within the
 [TableGen](https://llvm.org/docs/TableGen/index.html) language. This
 infrastructure is currently the main way in which patterns may be defined
@@ -257,7 +257,7 @@ the current file.
 #### `.td` includes
 
 When including a `.td` file, PDLL will automatically import any pertinent
-[ODS](https://mlir.llvm.org/docs/OpDefinitions/) information within that file.
+[ODS](DefiningDialects/Operations.md) information within that file.
 This includes any defined operations, constraints, interfaces, and more, making
 them implicitly accessible within PDLL. This is important, as ODS information
 allows for certain PDLL constructs, such as the
diff --git a/mlir/docs/Tools/mlir-rewrite.md b/mlir/docs/Tools/mlir-rewrite.md
new file mode 100644
index 0000000000000000000000000000000000000000..178f92f72cbb6e2b7d541d4c03323a2d5ccb9f94
--- /dev/null
+++ b/mlir/docs/Tools/mlir-rewrite.md
@@ -0,0 +1,29 @@
+# mlir-rewrite
+
+Tool to simplify rewriting .mlir files. There are a couple of build in rewrites
+discussed below along with usage.
+
+Note: This is still in very early stage. Its so early its less a tool than a
+growing collection of useful functions: to use its best to do what's needed on
+a brance by just hacking it (dialects registered, rewrites etc) to say help
+ease a rename, upstream useful utility functions, point to ease others
+migrating, and then bin eventually. Once there are actually useful parts it
+should be refactored same as mlir-opt.
+
+[TOC]
+
+## simple-rename
+
+Rename per op given a substring to a target. The match and replace uses LLVM's
+regex sub for the match and replace while the op-name is matched via regular
+string comparison. E.g.,
+
+```
+mlir-rewrite input.mlir -o output.mlir --simple-rename \
+   --simple-rename-op-name="test.concat" --simple-rename-match="axis" \
+                                         --simple-rename-replace="bxis"
+```
+
+to replace `axis` substring in the text of the range corresponding to
+`test.concat` ops with `bxis`.
+
diff --git a/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h
index 80c8b86c63678a48fac03a74343fd6aab967c1d2..2250db823b551982fba878e28aa5a5baf6ebc45b 100644
--- a/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h
+++ b/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h
@@ -182,7 +182,7 @@ public:
 
   /// Visit an operation with control-flow semantics and deduce which of its
   /// successors are live.
-  LogicalResult visit(ProgramPoint point) override;
+  LogicalResult visit(ProgramPoint *point) override;
 
 private:
   /// Find and mark symbol callables with potentially unknown callsites as
diff --git a/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h
index 7917f1e3ba64853110b4c6f841582d6335762138..2e32bd1bc146177acb7a321ab022c93fe4b3c04f 100644
--- a/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h
+++ b/mlir/include/mlir/Analysis/DataFlow/DenseAnalysis.h
@@ -36,8 +36,7 @@ enum class CallControlFlowAction { EnterCallee, ExitCallee, ExternalCallee };
 //===----------------------------------------------------------------------===//
 
 /// This class represents a dense lattice. A dense lattice is attached to
-/// operations to represent the program state after their execution or to blocks
-/// to represent the program state at the beginning of the block. A dense
+/// program point to represent the program state at the program point.
 /// lattice is propagated through the IR by dense data-flow analysis.
 class AbstractDenseLattice : public AnalysisState {
 public:
@@ -59,15 +58,13 @@ public:
 //===----------------------------------------------------------------------===//
 
 /// Base class for dense forward data-flow analyses. Dense data-flow analysis
-/// attaches a lattice between the execution of operations and implements a
-/// transfer function from the lattice before each operation to the lattice
-/// after. The lattice contains information about the state of the program at
-/// that point.
+/// attaches a lattice to program points and implements a transfer function from
+/// the lattice before each operation to the lattice after. The lattice contains
+/// information about the state of the program at that program point.
 ///
-/// In this implementation, a lattice attached to an operation represents the
-/// state of the program after its execution, and a lattice attached to block
-/// represents the state of the program right before it starts executing its
-/// body.
+/// Visit a program point in forward dense data-flow analysis will invoke the
+/// transfer function of the operation preceding the program point iterator.
+/// Visit a program point at the begining of block will visit the block itself.
 class AbstractDenseForwardDataFlowAnalysis : public DataFlowAnalysis {
 public:
   using DataFlowAnalysis::DataFlowAnalysis;
@@ -76,13 +73,14 @@ public:
   /// may modify the program state; that is, every operation and block.
   LogicalResult initialize(Operation *top) override;
 
-  /// Visit a program point that modifies the state of the program. If this is a
-  /// block, then the state is propagated from control-flow predecessors or
-  /// callsites. If this is a call operation or region control-flow operation,
-  /// then the state after the execution of the operation is set by control-flow
-  /// or the callgraph. Otherwise, this function invokes the operation transfer
-  /// function.
-  LogicalResult visit(ProgramPoint point) override;
+  /// Visit a program point that modifies the state of the program. If the
+  /// program point is at the beginning of a block, then the state is propagated
+  /// from control-flow predecessors or callsites.  If the operation before
+  /// program point iterator is a call operation or region control-flow
+  /// operation, then the state after the execution of the operation is set by
+  /// control-flow or the callgraph. Otherwise, this function invokes the
+  /// operation transfer function before the program point iterator.
+  LogicalResult visit(ProgramPoint *point) override;
 
 protected:
   /// Propagate the dense lattice before the execution of an operation to the
@@ -91,15 +89,14 @@ protected:
                                            const AbstractDenseLattice &before,
                                            AbstractDenseLattice *after) = 0;
 
-  /// Get the dense lattice after the execution of the given lattice anchor.
+  /// Get the dense lattice on the given lattice anchor.
   virtual AbstractDenseLattice *getLattice(LatticeAnchor anchor) = 0;
 
-  /// Get the dense lattice after the execution of the given program point and
-  /// add it as a dependency to a lattice anchor. That is, every time the
-  /// lattice after anchor is updated, the dependent program point must be
-  /// visited, and the newly triggered visit might update the lattice after
-  /// dependent.
-  const AbstractDenseLattice *getLatticeFor(ProgramPoint dependent,
+  /// Get the dense lattice on the given lattice anchor and add dependent as its
+  /// dependency. That is, every time the lattice after anchor is updated, the
+  /// dependent program point must be visited, and the newly triggered visit
+  /// might update the lattice on dependent.
+  const AbstractDenseLattice *getLatticeFor(ProgramPoint *dependent,
                                             LatticeAnchor anchor);
 
   /// Set the dense lattice at control flow entry point and propagate an update
@@ -153,7 +150,7 @@ protected:
   /// Visit a program point within a region branch operation with predecessors
   /// in it. This can either be an entry block of one of the regions of the
   /// parent operation itself.
-  void visitRegionBranchOperation(ProgramPoint point,
+  void visitRegionBranchOperation(ProgramPoint *point,
                                   RegionBranchOpInterface branch,
                                   AbstractDenseLattice *after);
 
@@ -294,14 +291,12 @@ protected:
 //===----------------------------------------------------------------------===//
 
 /// Base class for dense backward dataflow analyses. Such analyses attach a
-/// lattice between the execution of operations and implement a transfer
-/// function from the lattice after the operation ot the lattice before it, thus
-/// propagating backward.
+/// lattice to program point and implement a transfer function from the lattice
+/// after the operation to the lattice before it, thus propagating backward.
 ///
-/// In this implementation, a lattice attached to an operation represents the
-/// state of the program before its execution, and a lattice attached to a block
-/// represents the state of the program before the end of the block, i.e., after
-/// its terminator.
+/// Visit a program point in dense backward data-flow analysis will invoke the
+/// transfer function of the operation following the program point iterator.
+/// Visit a program point at the end of block will visit the block itself.
 class AbstractDenseBackwardDataFlowAnalysis : public DataFlowAnalysis {
 public:
   /// Construct the analysis in the given solver. Takes a symbol table
@@ -321,9 +316,9 @@ public:
   /// operations, the state is propagated using the transfer function
   /// (visitOperation).
   ///
-  /// Note: the transfer function is currently *not* invoked for operations with
-  /// region or call interface, but *is* invoked for block terminators.
-  LogicalResult visit(ProgramPoint point) override;
+  /// Note: the transfer function is currently *not* invoked before operations
+  /// with region or call interface, but *is* invoked before block terminators.
+  LogicalResult visit(ProgramPoint *point) override;
 
 protected:
   /// Propagate the dense lattice after the execution of an operation to the
@@ -337,10 +332,11 @@ protected:
   /// block.
   virtual AbstractDenseLattice *getLattice(LatticeAnchor anchor) = 0;
 
-  /// Get the dense lattice before the execution of the program point in
-  /// `anchor` and declare that the `dependent` program point must be updated
-  /// every time `point` is.
-  const AbstractDenseLattice *getLatticeFor(ProgramPoint dependent,
+  /// Get the dense lattice on the given lattice anchor and add dependent as its
+  /// dependency. That is, every time the lattice after anchor is updated, the
+  /// dependent program point must be visited, and the newly triggered visit
+  /// might update the lattice before dependent.
+  const AbstractDenseLattice *getLatticeFor(ProgramPoint *dependent,
                                             LatticeAnchor anchor);
 
   /// Set the dense lattice before at the control flow exit point and propagate
@@ -400,7 +396,7 @@ private:
   /// (from which the state is propagated) in or after it. `regionNo` indicates
   /// the region that contains the successor, `nullopt` indicating the successor
   /// of the branch operation itself.
-  void visitRegionBranchOperation(ProgramPoint point,
+  void visitRegionBranchOperation(ProgramPoint *point,
                                   RegionBranchOpInterface branch,
                                   RegionBranchPoint branchPoint,
                                   AbstractDenseLattice *before);
diff --git a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
index 933790b4f2a6eb6f22d4054a945daf806c91d288..dce7ab3bb5ee793bcdb4f58900259477b2ec84cb 100644
--- a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
+++ b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
@@ -179,18 +179,22 @@ private:
 /// operands to the lattices of the results. This analysis will propagate
 /// lattices across control-flow edges and the callgraph using liveness
 /// information.
+///
+/// Visit a program point in sparse forward data-flow analysis will invoke the
+/// transfer function of the operation preceding the program point iterator.
+/// Visit a program point at the begining of block will visit the block itself.
 class AbstractSparseForwardDataFlowAnalysis : public DataFlowAnalysis {
 public:
   /// Initialize the analysis by visiting every owner of an SSA value: all
   /// operations and blocks.
   LogicalResult initialize(Operation *top) override;
 
-  /// Visit a program point. If this is a block and all control-flow
-  /// predecessors or callsites are known, then the arguments lattices are
-  /// propagated from them. If this is a call operation or an operation with
-  /// region control-flow, then its result lattices are set accordingly.
-  /// Otherwise, the operation transfer function is invoked.
-  LogicalResult visit(ProgramPoint point) override;
+  /// Visit a program point. If this is at beginning of block and all
+  /// control-flow predecessors or callsites are known, then the arguments
+  /// lattices are propagated from them. If this is after call operation or an
+  /// operation with region control-flow, then its result lattices are set
+  /// accordingly.  Otherwise, the operation transfer function is invoked.
+  LogicalResult visit(ProgramPoint *point) override;
 
 protected:
   explicit AbstractSparseForwardDataFlowAnalysis(DataFlowSolver &solver);
@@ -221,7 +225,7 @@ protected:
 
   /// Get a read-only lattice element for a value and add it as a dependency to
   /// a program point.
-  const AbstractSparseLattice *getLatticeElementFor(ProgramPoint point,
+  const AbstractSparseLattice *getLatticeElementFor(ProgramPoint *point,
                                                     Value value);
 
   /// Set the given lattice element(s) at control flow entry point(s).
@@ -251,7 +255,8 @@ private:
   /// operation `branch`, which can either be the entry block of one of the
   /// regions or the parent operation itself, and set either the argument or
   /// parent result lattices.
-  void visitRegionSuccessors(ProgramPoint point, RegionBranchOpInterface branch,
+  void visitRegionSuccessors(ProgramPoint *point,
+                             RegionBranchOpInterface branch,
                              RegionBranchPoint successor,
                              ArrayRef<AbstractSparseLattice *> lattices);
 };
@@ -312,7 +317,7 @@ protected:
 
   /// Get the lattice element for a value and create a dependency on the
   /// provided program point.
-  const StateT *getLatticeElementFor(ProgramPoint point, Value value) {
+  const StateT *getLatticeElementFor(ProgramPoint *point, Value value) {
     return static_cast<const StateT *>(
         AbstractSparseForwardDataFlowAnalysis::getLatticeElementFor(point,
                                                                     value));
@@ -377,10 +382,10 @@ public:
   /// under it.
   LogicalResult initialize(Operation *top) override;
 
-  /// Visit a program point. If this is a call operation or an operation with
+  /// Visit a program point. If it is after call operation or an operation with
   /// block or region control-flow, then operand lattices are set accordingly.
   /// Otherwise, invokes the operation transfer function (`visitOperationImpl`).
-  LogicalResult visit(ProgramPoint point) override;
+  LogicalResult visit(ProgramPoint *point) override;
 
 protected:
   explicit AbstractSparseBackwardDataFlowAnalysis(
@@ -445,14 +450,14 @@ private:
   /// Get the lattice element for a value, and also set up
   /// dependencies so that the analysis on the given ProgramPoint is re-invoked
   /// if the value changes.
-  const AbstractSparseLattice *getLatticeElementFor(ProgramPoint point,
+  const AbstractSparseLattice *getLatticeElementFor(ProgramPoint *point,
                                                     Value value);
 
   /// Get the lattice elements for a range of values, and also set up
   /// dependencies so that the analysis on the given ProgramPoint is re-invoked
   /// if any of the values change.
   SmallVector<const AbstractSparseLattice *>
-  getLatticeElementsFor(ProgramPoint point, ValueRange values);
+  getLatticeElementsFor(ProgramPoint *point, ValueRange values);
 
   SymbolTableCollection &symbolTable;
 };
@@ -465,6 +470,10 @@ private:
 /// backwards across the IR by implementing transfer functions for operations.
 ///
 /// `StateT` is expected to be a subclass of `AbstractSparseLattice`.
+///
+/// Visit a program point in sparse backward data-flow analysis will invoke the
+/// transfer function of the operation preceding the program point iterator.
+/// Visit a program point at the begining of block will visit the block itself.
 template <typename StateT>
 class SparseBackwardDataFlowAnalysis
     : public AbstractSparseBackwardDataFlowAnalysis {
diff --git a/mlir/include/mlir/Analysis/DataFlowFramework.h b/mlir/include/mlir/Analysis/DataFlowFramework.h
index b0450ecdbd99b8db25e8b7addce883fdbe5269c8..969664dc7a4fe3a9b10fdde6127dea023b378711 100644
--- a/mlir/include/mlir/Analysis/DataFlowFramework.h
+++ b/mlir/include/mlir/Analysis/DataFlowFramework.h
@@ -18,10 +18,12 @@
 
 #include "mlir/IR/Operation.h"
 #include "mlir/Support/StorageUniquer.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/TypeName.h"
 #include <queue>
+#include <tuple>
 
 namespace mlir {
 
@@ -51,23 +53,104 @@ class AnalysisState;
 
 /// Program point represents a specific location in the execution of a program.
 /// A sequence of program points can be combined into a control flow graph.
-struct ProgramPoint : public PointerUnion<Operation *, Block *> {
-  using ParentTy = PointerUnion<Operation *, Block *>;
-  /// Inherit constructors.
-  using ParentTy::PointerUnion;
-  /// Allow implicit conversion from the parent type.
-  ProgramPoint(ParentTy point = nullptr) : ParentTy(point) {}
-  /// Allow implicit conversions from operation wrappers.
-  /// TODO: For Windows only. Find a better solution.
-  template <typename OpT, typename = std::enable_if_t<
-                              std::is_convertible<OpT, Operation *>::value &&
-                              !std::is_same<OpT, Operation *>::value>>
-  ProgramPoint(OpT op) : ParentTy(op) {}
+struct ProgramPoint : public StorageUniquer::BaseStorage {
+  /// Creates a new program point at the given location.
+  ProgramPoint(Block *parentBlock, Block::iterator pp)
+      : block(parentBlock), point(pp) {}
+
+  /// Creates a new program point at the given operation.
+  ProgramPoint(Operation *op) : op(op) {}
+
+  /// The concrete key type used by the storage uniquer. This class is uniqued
+  /// by its contents.
+  using KeyTy = std::tuple<Block *, Block::iterator, Operation *>;
+
+  /// Create a empty program point.
+  ProgramPoint() {}
+
+  /// Create a new program point from the given program point.
+  ProgramPoint(const ProgramPoint &point) {
+    this->block = point.getBlock();
+    this->point = point.getPoint();
+    this->op = point.getOperation();
+  }
+
+  static ProgramPoint *construct(StorageUniquer::StorageAllocator &alloc,
+                                 KeyTy &&key) {
+    if (std::get<0>(key)) {
+      return new (alloc.allocate<ProgramPoint>())
+          ProgramPoint(std::get<0>(key), std::get<1>(key));
+    }
+    return new (alloc.allocate<ProgramPoint>()) ProgramPoint(std::get<2>(key));
+  }
+
+  /// Returns true if this program point is set.
+  bool isNull() const { return block == nullptr && op == nullptr; }
+
+  /// Two program points are equal if their block and iterator are equal.
+  bool operator==(const KeyTy &key) const {
+    return block == std::get<0>(key) && point == std::get<1>(key) &&
+           op == std::get<2>(key);
+  }
+
+  bool operator==(const ProgramPoint &pp) const {
+    return block == pp.block && point == pp.point && op == pp.op;
+  }
+
+  /// Get the block contains this program point.
+  Block *getBlock() const { return block; }
+
+  /// Get the the iterator this program point refers to.
+  Block::iterator getPoint() const { return point; }
+
+  /// Get the the iterator this program point refers to.
+  Operation *getOperation() const { return op; }
+
+  /// Get the next operation of this program point.
+  Operation *getNextOp() const {
+    assert(!isBlockEnd());
+    // If the current program point has no parent block, both the next op and
+    // the previous op point to the op corresponding to the current program
+    // point.
+    if (block == nullptr) {
+      return op;
+    }
+    return &*point;
+  }
+
+  /// Get the previous operation of this program point.
+  Operation *getPrevOp() const {
+    assert(!isBlockStart());
+    // If the current program point has no parent block, both the next op and
+    // the previous op point to the op corresponding to the current program
+    // point.
+    if (block == nullptr) {
+      return op;
+    }
+    return &*(--Block::iterator(point));
+  }
+
+  bool isBlockStart() const { return block && block->begin() == point; }
+
+  bool isBlockEnd() const { return block && block->end() == point; }
 
   /// Print the program point.
   void print(raw_ostream &os) const;
+
+private:
+  Block *block = nullptr;
+  Block::iterator point;
+
+  /// For operations without a parent block, we record the operation itself as
+  /// its program point.
+  Operation *op = nullptr;
 };
 
+inline raw_ostream &operator<<(raw_ostream &os, ProgramPoint point) {
+  point.print(os);
+  return os;
+}
+
 //===----------------------------------------------------------------------===//
 // GenericLatticeAnchor
 //===----------------------------------------------------------------------===//
@@ -165,21 +248,12 @@ private:
 
 /// Fundamental IR components are supported as first-class lattice anchor.
 struct LatticeAnchor
-    : public PointerUnion<GenericLatticeAnchor *, ProgramPoint, Value> {
-  using ParentTy = PointerUnion<GenericLatticeAnchor *, ProgramPoint, Value>;
+    : public PointerUnion<GenericLatticeAnchor *, ProgramPoint *, Value> {
+  using ParentTy = PointerUnion<GenericLatticeAnchor *, ProgramPoint *, Value>;
   /// Inherit constructors.
   using ParentTy::PointerUnion;
   /// Allow implicit conversion from the parent type.
   LatticeAnchor(ParentTy point = nullptr) : ParentTy(point) {}
-  /// Allow implicit conversions from operation wrappers.
-  /// TODO: For Windows only. Find a better solution.
-  template <typename OpT, typename = std::enable_if_t<
-                              std::is_convertible<OpT, Operation *>::value &&
-                              !std::is_same<OpT, Operation *>::value>>
-  LatticeAnchor(OpT op) : ParentTy(ProgramPoint(op)) {}
-
-  LatticeAnchor(Operation *op) : ParentTy(ProgramPoint(op)) {}
-  LatticeAnchor(Block *block) : ParentTy(ProgramPoint(block)) {}
 
   /// Print the lattice anchor.
   void print(raw_ostream &os) const;
@@ -238,7 +312,9 @@ private:
 class DataFlowSolver {
 public:
   explicit DataFlowSolver(const DataFlowConfig &config = DataFlowConfig())
-      : config(config) {}
+      : config(config) {
+    uniquer.registerParametricStorageType<ProgramPoint>();
+  }
 
   /// Load an analysis into the solver. Return the analysis instance.
   template <typename AnalysisT, typename... Args>
@@ -277,10 +353,39 @@ public:
     return AnchorT::get(uniquer, std::forward<Args>(args)...);
   }
 
+  /// Get a uniqued program point instance.
+  ProgramPoint *getProgramPointBefore(Operation *op) {
+    if (op->getBlock())
+      return uniquer.get<ProgramPoint>(/*initFn*/ {}, op->getBlock(),
+                                       Block::iterator(op), nullptr);
+    else
+      return uniquer.get<ProgramPoint>(/*initFn*/ {}, nullptr,
+                                       Block::iterator(), op);
+  }
+
+  ProgramPoint *getProgramPointBefore(Block *block) {
+    return uniquer.get<ProgramPoint>(/*initFn*/ {}, block, block->begin(),
+                                     nullptr);
+  }
+
+  ProgramPoint *getProgramPointAfter(Operation *op) {
+    if (op->getBlock())
+      return uniquer.get<ProgramPoint>(/*initFn*/ {}, op->getBlock(),
+                                       ++Block::iterator(op), nullptr);
+    else
+      return uniquer.get<ProgramPoint>(/*initFn*/ {}, nullptr,
+                                       Block::iterator(), op);
+  }
+
+  ProgramPoint *getProgramPointAfter(Block *block) {
+    return uniquer.get<ProgramPoint>(/*initFn*/ {}, block, block->end(),
+                                     nullptr);
+  }
+
   /// A work item on the solver queue is a program point, child analysis pair.
   /// Each item is processed by invoking the child analysis at the program
   /// point.
-  using WorkItem = std::pair<ProgramPoint, DataFlowAnalysis *>;
+  using WorkItem = std::pair<ProgramPoint *, DataFlowAnalysis *>;
   /// Push a work item onto the worklist.
   void enqueue(WorkItem item) { worklist.push(std::move(item)); }
 
@@ -343,7 +448,7 @@ class AnalysisState {
 public:
   virtual ~AnalysisState();
 
-  /// Create the analysis state at the given lattice anchor.
+  /// Create the analysis state on the given lattice anchor.
   AnalysisState(LatticeAnchor anchor) : anchor(anchor) {}
 
   /// Returns the lattice anchor this state is located at.
@@ -356,7 +461,7 @@ public:
   /// Add a dependency to this analysis state on a lattice anchor and an
   /// analysis. If this state is updated, the analysis will be invoked on the
   /// given lattice anchor again (in onUpdate()).
-  void addDependency(ProgramPoint point, DataFlowAnalysis *analysis);
+  void addDependency(ProgramPoint *point, DataFlowAnalysis *analysis);
 
 protected:
   /// This function is called by the solver when the analysis state is updated
@@ -446,12 +551,12 @@ public:
   /// `visit` can add new dependencies, but these dependencies will not be
   /// dynamically added to the worklist because the solver doesn't know what
   /// will provide a value for then.
-  virtual LogicalResult visit(ProgramPoint point) = 0;
+  virtual LogicalResult visit(ProgramPoint *point) = 0;
 
 protected:
   /// Create a dependency between the given analysis state and lattice anchor
   /// on this analysis.
-  void addDependency(AnalysisState *state, ProgramPoint point);
+  void addDependency(AnalysisState *state, ProgramPoint *point);
 
   /// Propagate an update to a state if it changed.
   void propagateIfChanged(AnalysisState *state, ChangeResult changed);
@@ -480,12 +585,29 @@ protected:
   /// on `dependent`. If the return state is updated elsewhere, this analysis is
   /// re-invoked on the dependent.
   template <typename StateT, typename AnchorT>
-  const StateT *getOrCreateFor(ProgramPoint dependent, AnchorT anchor) {
+  const StateT *getOrCreateFor(ProgramPoint *dependent, AnchorT anchor) {
     StateT *state = getOrCreate<StateT>(anchor);
     addDependency(state, dependent);
     return state;
   }
 
+  /// Get a uniqued program point instance.
+  ProgramPoint *getProgramPointBefore(Operation *op) {
+    return solver.getProgramPointBefore(op);
+  }
+
+  ProgramPoint *getProgramPointBefore(Block *block) {
+    return solver.getProgramPointBefore(block);
+  }
+
+  ProgramPoint *getProgramPointAfter(Operation *op) {
+    return solver.getProgramPointAfter(op);
+  }
+
+  ProgramPoint *getProgramPointAfter(Block *block) {
+    return solver.getProgramPointAfter(block);
+  }
+
   /// Return the configuration of the solver used for this analysis.
   const DataFlowConfig &getSolverConfig() const { return solver.getConfig(); }
 
@@ -539,12 +661,30 @@ inline raw_ostream &operator<<(raw_ostream &os, LatticeAnchor anchor) {
 namespace llvm {
 /// Allow hashing of lattice anchors and program points.
 template <>
-struct DenseMapInfo<mlir::LatticeAnchor>
-    : public DenseMapInfo<mlir::LatticeAnchor::ParentTy> {};
+struct DenseMapInfo<mlir::ProgramPoint> {
+  static mlir::ProgramPoint getEmptyKey() {
+    void *pointer = llvm::DenseMapInfo<void *>::getEmptyKey();
+    return mlir::ProgramPoint(
+        (mlir::Block *)pointer,
+        mlir::Block::iterator((mlir::Operation *)pointer));
+  }
+  static mlir::ProgramPoint getTombstoneKey() {
+    void *pointer = llvm::DenseMapInfo<void *>::getTombstoneKey();
+    return mlir::ProgramPoint(
+        (mlir::Block *)pointer,
+        mlir::Block::iterator((mlir::Operation *)pointer));
+  }
+  static unsigned getHashValue(mlir::ProgramPoint pp) {
+    return hash_combine(pp.getBlock(), pp.getPoint().getNodePtr());
+  }
+  static bool isEqual(mlir::ProgramPoint lhs, mlir::ProgramPoint rhs) {
+    return lhs == rhs;
+  }
+};
 
 template <>
-struct DenseMapInfo<mlir::ProgramPoint>
-    : public DenseMapInfo<mlir::ProgramPoint::ParentTy> {};
+struct DenseMapInfo<mlir::LatticeAnchor>
+    : public DenseMapInfo<mlir::LatticeAnchor::ParentTy> {};
 
 // Allow llvm::cast style functions.
 template <typename To>
@@ -555,19 +695,6 @@ template <typename To>
 struct CastInfo<To, const mlir::LatticeAnchor>
     : public CastInfo<To, const mlir::LatticeAnchor::PointerUnion> {};
 
-template <typename To>
-struct CastInfo<To, mlir::ProgramPoint>
-    : public CastInfo<To, mlir::ProgramPoint::PointerUnion> {};
-
-template <typename To>
-struct CastInfo<To, const mlir::ProgramPoint>
-    : public CastInfo<To, const mlir::ProgramPoint::PointerUnion> {};
-
-/// Allow stealing the low bits of a ProgramPoint.
-template <>
-struct PointerLikeTypeTraits<mlir::ProgramPoint>
-    : public PointerLikeTypeTraits<mlir::ProgramPoint::ParentTy> {};
-
 } // end namespace llvm
 
 #endif // MLIR_ANALYSIS_DATAFLOWFRAMEWORK_H
diff --git a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
index 5ae484eadf49ff9a24dffcf204b28e5eb78aed05..4822d6836ded11b10ff670ed6c285c0a99874fee 100644
--- a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
+++ b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
@@ -11,6 +11,7 @@
 #include <memory>
 
 namespace mlir {
+class DialectRegistry;
 class LLVMTypeConverter;
 class ConversionTarget;
 class MLIRContext;
@@ -28,6 +29,10 @@ void configureOpenMPToLLVMConversionLegality(
 /// Populate the given list with patterns that convert from OpenMP to LLVM.
 void populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter,
                                             RewritePatternSet &patterns);
+
+/// Registers the `ConvertToLLVMPatternInterface` interface in the `OpenMP`
+/// dialect.
+void registerConvertOpenMPToLLVMInterface(DialectRegistry &registry);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_OPENMPTOLLVM_CONVERTOPENMPTOLLVM_H
diff --git a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
index cca0008cbc2633e6be43cbf79d654d4a3828805e..22efe15aa83a50a7719795d75ac766a8ceee6ebe 100644
--- a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
+++ b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
@@ -251,7 +251,7 @@ def FuncOp : Func_Op<"func", [
 
     // A function that returns its argument twice:
     func.func @count(%x: i64) -> (i64, i64)
-      attributes {fruit: "banana"} {
+      attributes {fruit = "banana"} {
       return %x, %x: i64, i64
     }
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 0d904f13037c619c7078527d145bb2e4d57f11a4..855fa5702f5b9cf314cc8a2e0b041af22dde85df 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -471,6 +471,10 @@ def LLVM_DIGlobalVariableExpressionAttr
   let constBuilderCall = "$0";
 }
 
+def DIGlobalVariableExpressionArrayAttr :
+  TypedArrayAttrBase<LLVM_DIGlobalVariableExpressionAttr,
+  "an array of variable expressions">;
+
 //===----------------------------------------------------------------------===//
 // DIGlobalVariableAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
index edcc34461f2f26466fe98eae1035b6a4dd731f2a..27a2b418aadb2a4b4cf394e79bd9d29b4ffad727 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
@@ -33,6 +33,7 @@ def LLVM_Dialect : Dialect {
     static StringRef getAliasScopesAttrName() { return "alias_scopes"; }
     static StringRef getAccessGroupsAttrName() { return "access_groups"; }
     static StringRef getIdentAttrName() { return "llvm.ident"; }
+    static StringRef getCommandlineAttrName() { return "llvm.commandline"; }
 
     /// Names of llvm parameter attributes.
     static StringRef getAlignAttrName() { return "llvm.align"; }
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 5031426033aea157cfddd02212e72417c76b2569..ab40c8ec4b65885b667fd5847dd73b02d3d846f6 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -166,6 +166,9 @@ def LLVM_SMaxOp : LLVM_BinarySameArgsIntrOpI<"smax">;
 def LLVM_SMinOp : LLVM_BinarySameArgsIntrOpI<"smin">;
 def LLVM_UMaxOp : LLVM_BinarySameArgsIntrOpI<"umax">;
 def LLVM_UMinOp : LLVM_BinarySameArgsIntrOpI<"umin">;
+def LLVM_SinhOp : LLVM_UnaryIntrOpF<"sinh">;
+def LLVM_CoshOp : LLVM_UnaryIntrOpF<"cosh">;
+def LLVM_TanhOp : LLVM_UnaryIntrOpF<"tanh">;
 
 class LLVM_MemcpyIntrOpBase<string name> :
     LLVM_ZeroResultIntrOp<name, [0, 1, 2],
@@ -347,7 +350,7 @@ class LLVM_ConstrainedIntr<string mnem, int numArgs,
                     [&args](unsigned index) { return args[index]->getType(); });
     llvm::Module *module = builder.GetInsertBlock()->getModule();
     llvm::Function *callee =
-      llvm::Intrinsic::getDeclaration(module,
+      llvm::Intrinsic::getOrInsertDeclaration(module,
         llvm::Intrinsic::experimental_constrained_}] #
     mnem # [{, overloadedTypes); }] #
     !cond(!gt(hasRoundingMode, 0) : [{
@@ -541,7 +544,7 @@ class LLVM_DbgIntrOp<string name, string argName, list<Trait> traits = []>
     llvm::Module *module = builder.GetInsertBlock()->getModule();
     llvm::LLVMContext &ctx = module->getContext();
     llvm::Function *fn =
-      llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::}]
+      llvm::Intrinsic::getOrInsertDeclaration(module, llvm::Intrinsic::}]
        # !subst(".", "_", name) # [{);
     builder.CreateCall(fn, {
         llvm::MetadataAsValue::get(ctx,
@@ -594,7 +597,7 @@ def LLVM_DbgLabelOp : LLVM_IntrOp<"dbg.label", [], [], [], 0> {
     llvm::Module *module = builder.GetInsertBlock()->getModule();
     llvm::LLVMContext &ctx = module->getContext();
     llvm::Function *fn =
-      llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::dbg_label);
+      llvm::Intrinsic::getOrInsertDeclaration(module, llvm::Intrinsic::dbg_label);
     builder.CreateCall(fn, {
         llvm::MetadataAsValue::get(ctx, moduleTranslation.translateDebugInfo($label))
       });
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 000d92f9ea3bcba904a2d7bfab1ac389662ecc0a..bbca7bc7286acb4c5266dc40faf9ffd2652ddbf6 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1168,7 +1168,7 @@ def LLVM_GlobalOp : LLVM_Op<"mlir.global",
     OptionalAttr<UnnamedAddr>:$unnamed_addr,
     OptionalAttr<StrAttr>:$section,
     OptionalAttr<SymbolRefAttr>:$comdat,
-    DefaultValuedAttr<LLVM_DIGlobalVariableExpressionAttr, "{}">:$dbg_expr,
+    OptionalAttr<DIGlobalVariableExpressionArrayAttr>:$dbg_exprs,
     DefaultValuedAttr<Visibility, "mlir::LLVM::Visibility::Default">:$visibility_
   );
   let summary = "LLVM dialect global.";
@@ -1279,7 +1279,7 @@ def LLVM_GlobalOp : LLVM_Op<"mlir.global",
       CArg<"bool", "false">:$thread_local_,
       CArg<"SymbolRefAttr", "{}">:$comdat,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs,
-      CArg<"DIGlobalVariableExpressionAttr", "{}">:$dbgExpr)>
+      CArg<"ArrayRef<Attribute>", "{}">:$dbgExprs)>
   ];
 
   let extraClassDeclaration = [{
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h
index 0fcaa96ade4031a428f7ba292d143d0f080e4344..6f1c243cc4396d2e40b6530521a7753f2ae80125 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h
@@ -120,6 +120,16 @@ bool isaConvolutionOpInterface(LinalgOp linalgOp,
 /// Checks whether `linalgOp` is semantically equivalent to a `linalg.copyOp`.
 bool isaCopyOpInterface(LinalgOp linalgOp);
 
+/// Checks whether `genericOp` is semantically equivalent to a
+///  `linalg.broadcast`. Returns broadcast dimensions if true.
+std::optional<SmallVector<int64_t>>
+isaBroadcastOpInterface(GenericOp genericOp);
+
+/// Checks whether `genericOp` is semantically equivalent to a
+///  `linalg.transpose`. Returns permuted dimensions if true.
+std::optional<SmallVector<int64_t>>
+isaTransposeOpInterface(GenericOp genericOp);
+
 /// Checks whether a given `genericOp` is semantically equivalent to a single
 /// linalgelementwise unary op. e.g. linalg.exp.
 /// A linalg.generic body could be a series of unary elementwise ops e.g.
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
index e80dbb2afb9ef74d4290176a88b6a8d31fdaefd8..0a404194569c2201930ad43abac5cd1015a10486 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
@@ -243,6 +243,18 @@ def LinalgStructuredInterface
                            utils::IteratorType::parallel);
       }]
     >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return true if all loops are parallel.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isAllParallelLoops",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return getNumParallelLoops() ==  getNumParallelLoops();
+      }]
+    >,
     InterfaceMethod<
       /*desc=*/[{
         Return the dims that are parallel loops.
@@ -327,6 +339,18 @@ def LinalgStructuredInterface
         return !getBlock()->getArgument(bbArgNumber).use_empty();
       }]
     >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Returns true only if linalgOp takes one input and produces one result.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isSingleInputOutput",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_op.getNumDpsInputs() == 1 && $_op.getNumDpsInits() == 1;
+      }]
+    >,
     InterfaceMethod<
       /*desc=*/[{
         Return true if `opOperand` is an init tensor. This is true when it is
@@ -684,16 +708,6 @@ def LinalgStructuredInterface
         return;
       }]
     >,
-    InterfaceMethod<
-      /*desc=*/[{
-        Return true if the user has supplied an explicit indexing maps for this op.
-      }],
-      /*retTy=*/"bool",
-      /*methodName=*/"hasUserDefinedMaps",
-      /*args=*/(ins),
-      /*methodBody=*/"",
-      /*defaultImplementation=*/[{ return false; }]
-    >,
     //===------------------------------------------------------------------===//
     // Linalg generalization hooks.
     //===------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
index 97b90333e2b20008f4ee4d06bb52a7eee898ecd5..8cb698096ef5b71482831ec56cbfe2f1a927e4a7 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
@@ -1065,6 +1065,78 @@ structured_op: !LinalgStructuredOpConfig
         - !ScalarExpression
           scalar_arg: rhs
 --- !LinalgOpConfig
+metadata: !LinalgOpMetadata
+  name: matmul
+  cpp_class_name: MatmulOp
+  doc: |-
+    Performs a matrix multiplication of two 2D inputs.
+
+    Numeric casting is performed on the operands to the inner multiply, promoting
+    them to the same data type as the accumulator/output.
+  implements:
+  - LinalgContractionOpInterface
+structured_op: !LinalgStructuredOpConfig
+  args:
+  - !LinalgOperandDefConfig
+    name: A
+    kind: input_tensor
+    type_var: T1
+    shape_map: affine_map<()[s0, s1, s2] -> (s0, s1)>
+  - !LinalgOperandDefConfig
+    name: B
+    kind: input_tensor
+    type_var: T2
+    shape_map: affine_map<()[s0, s1, s2] -> (s1, s2)>
+  - !LinalgOperandDefConfig
+    name: C
+    kind: output_tensor
+    type_var: U
+    shape_map: affine_map<()[s0, s1, s2] -> (s0, s2)>
+  - !LinalgOperandDefConfig
+    name: cast
+    kind: type_fn_attr
+    default_fn: cast_signed
+  indexing_maps: !LinalgIndexingMapsConfig
+    static_indexing_maps:
+    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d2)>
+    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d2, d1)>
+    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1)>
+  iterator_types:
+  - parallel
+  - parallel
+  - reduction
+  assignments:
+  - !ScalarAssign
+    arg: C
+    value: !ScalarExpression
+      scalar_fn:
+        kind: binary
+        fn_name: add
+        operands:
+        - !ScalarExpression
+          scalar_arg: C
+        - !ScalarExpression
+          scalar_fn:
+            kind: binary
+            fn_name: mul
+            operands:
+            - !ScalarExpression
+              scalar_fn:
+                kind: type
+                attr_name: cast
+                type_var: U
+                operands:
+                - !ScalarExpression
+                  scalar_arg: A
+            - !ScalarExpression
+              scalar_fn:
+                kind: type
+                attr_name: cast
+                type_var: U
+                operands:
+                - !ScalarExpression
+                  scalar_arg: B
+--- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: quantized_matmul
   cpp_class_name: QuantizedMatmulOp
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index 5b6a90f806bedd996cca007852e9170b43d10949..e42fd5d2ce13c1bbd9b66296f7f465b18766a9b2 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -313,7 +313,7 @@ def Linalg_WinogradInputTransformOp : Linalg_Op<"winograd_input_transform",
 }
 
 def Linalg_WinogradOutputTransformOp : Linalg_Op<"winograd_output_transform",
-    [AllElementTypesMatch<["value", "output"]>,
+    [AllElementTypesMatch<["value", "output"]>, DestinationStyleOpInterface,
      DeclareOpInterfaceMethods<TilingInterface,
       ["getIterationDomain",
        "getLoopIteratorTypes",
@@ -396,6 +396,7 @@ def Linalg_WinogradOutputTransformOp : Linalg_Op<"winograd_output_transform",
     int64_t getOutputFDim() {
       return 3;
     }
+    MutableOperandRange getDpsInitsMutable() { return getOutputMutable(); }
   }];
   let hasVerifier = 1;
 }
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 61d4fc9734c6de81efbe93d27a734eee66e5c709..bfc609bd708164a97a5e211077d397f491d58c81 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -210,6 +210,24 @@ def GenericOp : LinalgStructuredBase_Op<"generic", [
     }
 
     MutableOperandRange getDpsInitsMutable() { return getOutputsMutable(); }
+
+    // Return true only if GenericOp has a single input and single
+    // output, and the body is a single yieldOp that yields the input.
+    // This check is useful when trying to determine if the op is
+    // essentially a transpose, broadcast, copy or something like that.
+    bool isSingleYieldOp() {
+      if (!isSingleInputOutput())
+        return false;
+     Block *body = getBody();
+     if (body->getOperations().size() != 1)
+       return false;
+
+     auto yieldOp = dyn_cast<linalg::YieldOp>(body->back());
+       if (!yieldOp || yieldOp.getNumOperands() != 1 ||
+           yieldOp->getOperand(0) != body->getArgument(0))
+         return false;
+     return true;
+   }
   }];
 
   let hasCanonicalizer = 1;
@@ -480,7 +498,7 @@ def BroadcastOp : LinalgStructuredBase_Op<"broadcast", [
     ```
       %bcast = linalg.broadcast
           ins(%input:tensor<16xf32>)
-          inits(%init:tensor<16x64xf32>)
+          outs(%init:tensor<16x64xf32>)
           dimensions = [1]
     ```
   }];
@@ -535,140 +553,6 @@ def BroadcastOp : LinalgStructuredBase_Op<"broadcast", [
   let hasCanonicalizer = 1;
 }
 
-//===----------------------------------------------------------------------===//
-// Op definition for MatmulOp
-//===----------------------------------------------------------------------===//
-
-def MatmulOp : LinalgStructuredBase_Op<"matmul", [
-               AttrSizedOperandSegments,
-               LinalgContractionOpInterface]> {
-    
-  let summary = [{
-    Performs a matrix multiplication of two 2D inputs without broadcast or transpose.
-    }];
-  let description = [{
-    Numeric casting is performed on the operands to the inner multiply,
-    promoting them to the same data type as the accumulator/output.
-
-    Broadcast and Transpose semantics can be appiled by specifying the explicit attribute
-    'indexing_maps' as shown below.This is a list attribute, so the list must include all
-    the maps if specified.
-
-    Example Transpose:
-    ```
-    linalg.matmul indexing_maps = [
-                   affine_map<(d0, d1, d2) -> (d2, d0)>, // transpose
-                   affine_map<(d0, d1, d2) -> (d2, d1)>,
-                   affine_map<(d0, d1, d2) -> (d0, d1)>
-                   ]
-                   ins(%arg0, %arg1 : memref<5x3xf32>,memref<5x7xf32>)
-                   outs(%arg2: memref<3x7xf32>)
-     ```
-
-    Example Broadcast:
-     ```
-    linalg.matmul indexing_maps = [
-                   affine_map<(d0, d1, d2) -> (d2)>,     // broadcast
-                   affine_map<(d0, d1, d2) -> (d2, d1)>,
-                   affine_map<(d0, d1, d2) -> (d0, d1)>
-                  ]
-                  ins(%arg0, %arg1 : memref<3xf32>, memref<5x7xf32>)
-                  outs(%arg2: memref<3x7xf32>)
-     ```
-
-     Example Broadcast and transpose:
-     ```
-     linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>, // transpose
-                       affine_map<(d0, d1, d2) -> (d2)>,     // broadcast
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5x3xf32>, memref<7xf32>) outs(%arg2: memref<3x7xf32>)
-    }];
-
-    let arguments = (ins
-      Variadic<AnyType>:$inputs,
-      Variadic<AnyShaped>:$outputs,
-      DefaultValuedOptionalAttr<AffineMapArrayAttr, "{}">:$indexing_maps,
-      DefaultValuedOptionalAttr<TypeFnAttr, "TypeFn::cast_signed">:$cast
-    );
-    let results = (outs Variadic<AnyRankedTensor>:$result_tensors);
-    let regions = (region AnyRegion:$region);
-
-    let skipDefaultBuilders = 1;
-    let builders = [
-      OpBuilder<
-      (ins "ValueRange":$inputs, "ValueRange":$outputs,
-            CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-      [{
-        buildStructuredOp($_builder, $_state, std::nullopt, inputs, outputs,
-          attributes, MatmulOp::getRegionBuilder());
-      }]>,
-      OpBuilder<
-      (ins "TypeRange":$resultTensorTypes, "ValueRange":$inputs,
-            "ValueRange":$outputs,
-            CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-      [{
-        buildStructuredOp($_builder, $_state, resultTensorTypes,
-          inputs, outputs, attributes, MatmulOp::getRegionBuilder());
-      }]>,
-      OpBuilder<
-      (ins "TypeRange":$resultTensorTypes, "ValueRange":$operands,
-            CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-      [{
-        $_state.addOperands(operands);
-        $_state.addAttributes(attributes);
-        $_state.addTypes(resultTensorTypes);
-        (void)$_state.addRegion();
-      }]>,
-      OpBuilder<
-      (ins "TypeRange":$resultTensorTypes, "ValueRange":$inputs,
-       "ValueRange":$outputs,
-       "Attribute":$cast, CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes),
-      [{
-        $_state.addAttribute("cast", cast);
-        buildStructuredOp($_builder, $_state, resultTensorTypes, inputs, outputs,
-          attributes, MatmulOp::getRegionBuilder());
-      }]>
-
-    ];
-    let hasCustomAssemblyFormat = 1;
-    let hasFolder = 1;
-    let hasVerifier = 1;
-
-    let extraClassDeclaration = structuredOpsBaseDecls # [{
-      SmallVector<utils::IteratorType> getIteratorTypesArray();
-
-      /// Implements the block region builder.
-      static void regionBuilder(ImplicitLocOpBuilder &b,
-                                Block &block, ArrayRef<NamedAttribute> attrs);
-
-      /// Returns a list of AffineMap with the typical matmul indexing charactristic.
-      SmallVector<AffineMap> getDefaultIndexingMaps();
-
-      /// Returns true if the given broadcast map \p bcastMap is valid for this op.
-      bool isValidLhsRhsBroadcastMap(AffineMap bcastMap);
-
-      static std::function<void(ImplicitLocOpBuilder &,
-                                Block &, ArrayRef<NamedAttribute>)>
-      getRegionBuilder() {
-        return regionBuilder;
-      }
-
-      ::mlir::MutableOperandRange getDpsInitsMutable() {
-        return getOutputsMutable();
-      }
-
-      // Generic methods.
-      static unsigned getNumRegionArgs();
-      std::string getLibraryCallName();
-      bool hasDynamicIndexingMaps();
-      /// Check if the op has broadcast and/or transpose semantic. Returns true if the
-      /// user defined indexing maps are not equal to default map.
-      bool hasUserDefinedMaps();
-    }];
-}
-
 //===----------------------------------------------------------------------===//
 // Named Linalg ops, implemented as a declarative configurations of generic ops.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 66f63fc02fe2f3189fa5b4cd4d6a76a06190b65b..11649ef2d03370a19524f4c50d6a46832262e1fd 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -119,6 +119,24 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
                    CArg<"TypeAttr">:$type)>
   ];
 
+  let extraClassDeclaration = [{
+    BlockArgument getAllocMoldArg() {
+      return getAllocRegion().getArgument(0);
+    }
+    BlockArgument getCopyMoldArg() {
+      auto &region = getCopyRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    BlockArgument getCopyPrivateArg() {
+      auto &region = getCopyRegion();
+      return region.empty() ? nullptr : region.getArgument(1);
+    }
+    BlockArgument getDeallocMoldArg() {
+      auto &region = getDeallocRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+  }];
+
   let hasVerifier = 1;
 }
 
@@ -364,9 +382,8 @@ def LoopNestOp : OpenMP_Op<"loop_nest", traits = [
 //===----------------------------------------------------------------------===//
 
 def WsloopOp : OpenMP_Op<"wsloop", traits = [
-    AttrSizedOperandSegments,
-    DeclareOpInterfaceMethods<ComposableOpInterface>,
-    DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    AttrSizedOperandSegments, DeclareOpInterfaceMethods<ComposableOpInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>, NoTerminator,
     RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AllocateClause, OpenMP_LinearClause, OpenMP_NowaitClause,
@@ -381,8 +398,8 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
     parallel region.
 
     The body region can only contain a single block which must contain a single
-    operation and a terminator. The operation must be another compatible loop
-    wrapper or an `omp.loop_nest`.
+    operation. This operation must be another compatible loop wrapper or an
+    `omp.loop_nest`.
 
     ```
     omp.wsloop <clauses> {
@@ -393,7 +410,6 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
         store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
         omp.yield
       }
-      omp.terminator
     }
     ```
   }] # clausesDescription;
@@ -417,9 +433,8 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
 //===----------------------------------------------------------------------===//
 
 def SimdOp : OpenMP_Op<"simd", traits = [
-    AttrSizedOperandSegments,
-    DeclareOpInterfaceMethods<ComposableOpInterface>,
-    DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    AttrSizedOperandSegments, DeclareOpInterfaceMethods<ComposableOpInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>, NoTerminator,
     RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AlignedClause, OpenMP_IfClause, OpenMP_LinearClause,
@@ -433,8 +448,8 @@ def SimdOp : OpenMP_Op<"simd", traits = [
     be executed concurrently using SIMD instructions).
 
     The body region can only contain a single block which must contain a single
-    operation and a terminator. The operation must be another compatible loop
-    wrapper or an `omp.loop_nest`.
+    operation. This operation must be another compatible loop wrapper or an
+    `omp.loop_nest`.
 
     ```
     omp.simd <clauses> {
@@ -445,7 +460,6 @@ def SimdOp : OpenMP_Op<"simd", traits = [
         store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
         omp.yield
       }
-      omp.terminator
     }
     ```
 
@@ -492,9 +506,8 @@ def YieldOp : OpenMP_Op<"yield",
 // Distribute construct [2.9.4.1]
 //===----------------------------------------------------------------------===//
 def DistributeOp : OpenMP_Op<"distribute", traits = [
-    AttrSizedOperandSegments,
-    DeclareOpInterfaceMethods<ComposableOpInterface>,
-    DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    AttrSizedOperandSegments, DeclareOpInterfaceMethods<ComposableOpInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>, NoTerminator,
     RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AllocateClause, OpenMP_DistScheduleClause, OpenMP_OrderClause,
@@ -517,8 +530,8 @@ def DistributeOp : OpenMP_Op<"distribute", traits = [
     region.
     
     The body region can only contain a single block which must contain a single
-    operation and a terminator. The operation must be another compatible loop
-    wrapper or an `omp.loop_nest`.
+    operation. This operation must be another compatible loop wrapper or an
+    `omp.loop_nest`.
 
     ```mlir
     omp.distribute <clauses> {
@@ -529,7 +542,6 @@ def DistributeOp : OpenMP_Op<"distribute", traits = [
         store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
         omp.yield
       }
-      omp.terminator
     }
     ```
   }] # clausesDescription;
@@ -595,7 +607,7 @@ def TaskOp : OpenMP_Op<"task", traits = [
 def TaskloopOp : OpenMP_Op<"taskloop", traits = [
     AttrSizedOperandSegments, AutomaticAllocationScope,
     DeclareOpInterfaceMethods<ComposableOpInterface>,
-    DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>, NoTerminator,
     RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AllocateClause, OpenMP_FinalClause, OpenMP_GrainsizeClause,
@@ -613,8 +625,8 @@ def TaskloopOp : OpenMP_Op<"taskloop", traits = [
     scheduled to be executed.
 
     The body region can only contain a single block which must contain a single
-    operation and a terminator. The operation must be another compatible loop
-    wrapper or an `omp.loop_nest`.
+    operation. This operation must be another compatible loop wrapper or an
+    `omp.loop_nest`.
 
     ```
     omp.taskloop <clauses> {
@@ -625,7 +637,6 @@ def TaskloopOp : OpenMP_Op<"taskloop", traits = [
         store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
         omp.yield
       }
-      omp.terminator
     }
     ```
 
@@ -1608,22 +1619,41 @@ def DeclareReductionOp : OpenMP_Op<"declare_reduction", [IsolatedFromAbove,
                        "( `cleanup` $cleanupRegion^ )? ";
 
   let extraClassDeclaration = [{
+    BlockArgument getAllocMoldArg() {
+      auto &region = getAllocRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    BlockArgument getInitializerMoldArg() {
+      return getInitializerRegion().getArgument(0);
+    }
+    BlockArgument getInitializerAllocArg() {
+      return getAllocRegion().empty() ?
+          nullptr : getInitializerRegion().getArgument(1);
+    }
+    BlockArgument getReductionLhsArg() {
+      return getReductionRegion().getArgument(0);
+    }
+    BlockArgument getReductionRhsArg() {
+      return getReductionRegion().getArgument(1);
+    }
+    BlockArgument getAtomicReductionLhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    BlockArgument getAtomicReductionRhsArg() {
+      auto &region = getAtomicReductionRegion();
+      return region.empty() ? nullptr : region.getArgument(1);
+    }
+    BlockArgument getCleanupAllocArg() {
+      auto &region = getCleanupRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+
     PointerLikeType getAccumulatorType() {
       if (getAtomicReductionRegion().empty())
         return {};
 
-      return cast<PointerLikeType>(getAtomicReductionRegion().front().getArgument(0).getType());
-    }
-
-    Value getInitializerMoldArg() {
-      return getInitializerRegion().front().getArgument(0);
-    }
-
-    Value getInitializerAllocArg() {
-      if (getAllocRegion().empty() ||
-          getInitializerRegion().front().getNumArguments() != 2)
-        return {nullptr};
-      return getInitializerRegion().front().getArgument(1);
+      return cast<PointerLikeType>(getAtomicReductionLhsArg().getType());
     }
   }];
   let hasRegionVerifier = 1;
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
index cb6c1b63e4e4b0bcd4aaa5ae0363f1bedcc6643b..adcf6fac752fe6ed10ae389f88ffedd141915a45 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
@@ -586,10 +586,10 @@ def IsSparseTensorSlicePred
           "  ::mlir::sparse_tensor::getSparseTensorEncoding($_self).isSlice()">;
 
 class SparseTensorOf<list<Type> allowedTypes>
-  : TensorOf<allowedTypes, [IsSparseTensorPred], "sparse tensor">;
+  : RankedTensorOf<allowedTypes, [IsSparseTensorPred], "sparse tensor">;
 
 class SparseTensorSliceOf<list<Type> allowedTypes>
-  : TensorOf<allowedTypes, [IsSparseTensorSlicePred], "sparse tensor slice">;
+  : RankedTensorOf<allowedTypes, [IsSparseTensorSlicePred], "sparse tensor slice">;
 
 class ScalarLikeOf<list<Type> allowedTypes>
   : AnyTypeOf<[0DTensorOf<allowedTypes>, AnyTypeOf<allowedTypes>], "scalar like">;
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
index 96a61419a541f7da4dd56fcb7803411fa356506c..2c281c9f6aa85d376a7822059a715e60652ffea2 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
@@ -92,8 +92,8 @@ def SparseTensor_AssembleOp : SparseTensor_Op<"assemble", [Pure]> {
     ```
   }];
 
-  let arguments = (ins Variadic<TensorOf<[AnySignlessIntegerOrIndex]>>:$levels,
-                   TensorOf<[AnyType]>:$values);
+  let arguments = (ins Variadic<RankedTensorOf<[AnySignlessIntegerOrIndex]>>:$levels,
+                       RankedTensorOf<[AnyType]>:$values);
   let results = (outs AnySparseTensor: $result);
   let assemblyFormat =
     "` ` `(` $levels       `)` `,` $values attr-dict `:`"
@@ -138,12 +138,12 @@ def SparseTensor_DisassembleOp : SparseTensor_Op<"disassemble", [Pure, SameVaria
   }];
 
   let arguments = (ins AnySparseTensor:$tensor,
-                   Variadic<TensorOf<[AnySignlessIntegerOrIndex]>>:$out_levels,
-                   TensorOf<[AnyType]>:$out_values);
-  let results = (outs Variadic<TensorOf<[AnySignlessIntegerOrIndex]>>:$ret_levels,
-                  TensorOf<[AnyType]>:$ret_values,
-                  Variadic<AnyIndexingScalarLike>:$lvl_lens,
-                  AnyIndexingScalarLike:$val_len);
+                       Variadic<RankedTensorOf<[AnySignlessIntegerOrIndex]>>:$out_levels,
+                       RankedTensorOf<[AnyType]>:$out_values);
+  let results = (outs Variadic<RankedTensorOf<[AnySignlessIntegerOrIndex]>>:$ret_levels,
+                      RankedTensorOf<[AnyType]>:$ret_values,
+                      Variadic<AnyIndexingScalarLike>:$lvl_lens,
+                      AnyIndexingScalarLike:$val_len);
   let assemblyFormat =
     "$tensor attr-dict `:` type($tensor)"
     "`out_lvls` `(` $out_levels `:` type($out_levels) `)` "
@@ -196,8 +196,8 @@ def SparseTensor_ConvertOp : SparseTensor_Op<"convert",
 
   }];
 
-  let arguments = (ins AnyTensor:$source);
-  let results = (outs AnyTensor:$dest);
+  let arguments = (ins AnyRankedTensor:$source);
+  let results = (outs AnyRankedTensor:$dest);
   let assemblyFormat = "$source attr-dict `:` type($source) `to` type($dest)";
 
   let extraClassDeclaration = [{
@@ -1447,7 +1447,7 @@ def SparseTensor_ForeachOp : SparseTensor_Op<"foreach",
   ];
 
   let regions = (region SizedRegion<1>:$region);
-  let arguments = (ins AnyTensor:$tensor,
+  let arguments = (ins AnyRankedTensor:$tensor,
                        Variadic<AnyType>:$initArgs,
                        OptionalAttr<AffineMapAttr>:$order);
   let results = (outs Variadic<AnyType>:$results);
diff --git a/mlir/include/mlir/IR/AsmState.h b/mlir/include/mlir/IR/AsmState.h
index 42cbedcf9f88374a2f6e5708ff2566a4bed90d89..edbd3bb6fc15db72f2e1a871a9603d648f02d44a 100644
--- a/mlir/include/mlir/IR/AsmState.h
+++ b/mlir/include/mlir/IR/AsmState.h
@@ -82,6 +82,8 @@ class AsmStateImpl;
 //===----------------------------------------------------------------------===//
 // Resource Entry
 
+class HeapAsmResourceBlob;
+
 /// This class represents a processed binary blob of data. A resource blob is
 /// essentially a collection of data, potentially mutable, with an associated
 /// deleter function (used if the data needs to be destroyed).
@@ -177,6 +179,8 @@ private:
 
   /// Whether the data is mutable.
   bool dataIsMutable;
+
+  friend class HeapAsmResourceBlob;
 };
 
 /// This class provides a simple utility wrapper for creating heap allocated
@@ -196,8 +200,11 @@ public:
   static AsmResourceBlob allocateAndCopyWithAlign(ArrayRef<char> data,
                                                   size_t align,
                                                   bool dataIsMutable = true) {
-    AsmResourceBlob blob = allocate(data.size(), align, dataIsMutable);
+    // This sets the blob to be mutable initially to allow writing
+    // (getMutableData) below.
+    AsmResourceBlob blob = allocate(data.size(), align, /*dataIsMutable=*/true);
     std::memcpy(blob.getMutableData().data(), data.data(), data.size());
+    blob.dataIsMutable = dataIsMutable;
     return blob;
   }
   template <typename T>
diff --git a/mlir/include/mlir/IR/Block.h b/mlir/include/mlir/IR/Block.h
index 67825eb1704bbe984a87070db199b497d29dd6cc..536cbf9018e89852b9fb849f6d4463743f0d2772 100644
--- a/mlir/include/mlir/IR/Block.h
+++ b/mlir/include/mlir/IR/Block.h
@@ -406,4 +406,25 @@ private:
 raw_ostream &operator<<(raw_ostream &, Block &);
 } // namespace mlir
 
+namespace llvm {
+template <>
+struct DenseMapInfo<mlir::Block::iterator> {
+  static mlir::Block::iterator getEmptyKey() {
+    void *pointer = llvm::DenseMapInfo<void *>::getEmptyKey();
+    return mlir::Block::iterator((mlir::Operation *)pointer);
+  }
+  static mlir::Block::iterator getTombstoneKey() {
+    void *pointer = llvm::DenseMapInfo<void *>::getTombstoneKey();
+    return mlir::Block::iterator((mlir::Operation *)pointer);
+  }
+  static unsigned getHashValue(mlir::Block::iterator iter) {
+    return hash_value(iter.getNodePtr());
+  }
+  static bool isEqual(mlir::Block::iterator lhs, mlir::Block::iterator rhs) {
+    return lhs == rhs;
+  }
+};
+
+} // end namespace llvm
+
 #endif // MLIR_IR_BLOCK_H
diff --git a/mlir/include/mlir/IR/BuiltinAttributes.td b/mlir/include/mlir/IR/BuiltinAttributes.td
index f0d417540014004c7f598fe87baacbfb4b424935..530ba7d2f11e5cc8b0b80e9f962d666f959e81ca 100644
--- a/mlir/include/mlir/IR/BuiltinAttributes.td
+++ b/mlir/include/mlir/IR/BuiltinAttributes.td
@@ -701,8 +701,10 @@ def Builtin_IntegerAttr : Builtin_Attr<"Integer", "integer",
         return $_get(type.getContext(), type, apValue);
       }
 
+      // TODO: Avoid implicit trunc?
       IntegerType intTy = ::llvm::cast<IntegerType>(type);
-      APInt apValue(intTy.getWidth(), value, intTy.isSignedInteger());
+      APInt apValue(intTy.getWidth(), value, intTy.isSignedInteger(),
+                    /*implicitTrunc=*/true);
       return $_get(type.getContext(), type, apValue);
     }]>
   ];
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index e2472eea8a3714ecc06b9189e73b0ccee7a5f680..a7222794f320b2fb4b62ea48a4175de9369e56d2 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -749,7 +749,9 @@ public:
     // zero for non-negated integers.
     result =
         (IntT)uintResult.sextOrTrunc(sizeof(IntT) * CHAR_BIT).getLimitedValue();
-    if (APInt(uintResult.getBitWidth(), result) != uintResult)
+    if (APInt(uintResult.getBitWidth(), result,
+              /*isSigned=*/std::is_signed_v<IntT>,
+              /*implicitTrunc=*/true) != uintResult)
       return emitError(loc, "integer value too large");
     return success();
   }
diff --git a/mlir/include/mlir/InitAllExtensions.h b/mlir/include/mlir/InitAllExtensions.h
index dc5d4fbea04f49bce05d289acd8f04d4219c0d7d..2a241fa4b192fee71ae28768206f37ec11e51b03 100644
--- a/mlir/include/mlir/InitAllExtensions.h
+++ b/mlir/include/mlir/InitAllExtensions.h
@@ -22,6 +22,7 @@
 #include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
 #include "mlir/Conversion/NVVMToLLVM/NVVMToLLVM.h"
+#include "mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h"
 #include "mlir/Conversion/UBToLLVM/UBToLLVM.h"
 #include "mlir/Dialect/Affine/TransformOps/AffineTransformOps.h"
 #include "mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.h"
@@ -67,6 +68,7 @@ inline void registerAllExtensions(DialectRegistry &registry) {
   registerConvertMathToLLVMInterface(registry);
   registerConvertMemRefToLLVMInterface(registry);
   registerConvertNVVMToLLVMInterface(registry);
+  registerConvertOpenMPToLLVMInterface(registry);
   ub::registerConvertUBToLLVMInterface(registry);
 
   // Register all transform dialect extensions.
diff --git a/mlir/include/mlir/TableGen/AttrOrTypeDef.h b/mlir/include/mlir/TableGen/AttrOrTypeDef.h
index 36744c85bc70865d9e3252d7f89ca2de702ebc1d..c3d730e42ef70e89fc8beb04c6cd46f2772bb369 100644
--- a/mlir/include/mlir/TableGen/AttrOrTypeDef.h
+++ b/mlir/include/mlir/TableGen/AttrOrTypeDef.h
@@ -105,7 +105,7 @@ public:
   std::optional<StringRef> getDefaultValue() const;
 
   /// Return the underlying def of this parameter.
-  llvm::Init *getDef() const;
+  const llvm::Init *getDef() const;
 
   /// The parameter is pointer-comparable.
   bool operator==(const AttrOrTypeParameter &other) const {
diff --git a/mlir/include/mlir/TableGen/Dialect.h b/mlir/include/mlir/TableGen/Dialect.h
index 3530d240c976c642cd2895989221804216c257f0..ea8f40555e4451beec60c72b7ef7f92235a58c26 100644
--- a/mlir/include/mlir/TableGen/Dialect.h
+++ b/mlir/include/mlir/TableGen/Dialect.h
@@ -92,7 +92,7 @@ public:
   /// dialect.
   bool usePropertiesForAttributes() const;
 
-  llvm::DagInit *getDiscardableAttributes() const;
+  const llvm::DagInit *getDiscardableAttributes() const;
 
   const llvm::Record *getDef() const { return def; }
 
diff --git a/mlir/include/mlir/TableGen/Operator.h b/mlir/include/mlir/TableGen/Operator.h
index 768291a3a7267b221c229b72215667498f481b8e..9e570373d9cd3273d392a9bbd3a6c007fc36bac1 100644
--- a/mlir/include/mlir/TableGen/Operator.h
+++ b/mlir/include/mlir/TableGen/Operator.h
@@ -119,14 +119,15 @@ public:
 
   /// A utility iterator over a list of variable decorators.
   struct VariableDecoratorIterator
-      : public llvm::mapped_iterator<llvm::Init *const *,
-                                     VariableDecorator (*)(llvm::Init *)> {
+      : public llvm::mapped_iterator<const llvm::Init *const *,
+                                     VariableDecorator (*)(
+                                         const llvm::Init *)> {
     /// Initializes the iterator to the specified iterator.
-    VariableDecoratorIterator(llvm::Init *const *it)
-        : llvm::mapped_iterator<llvm::Init *const *,
-                                VariableDecorator (*)(llvm::Init *)>(it,
-                                                                     &unwrap) {}
-    static VariableDecorator unwrap(llvm::Init *init);
+    VariableDecoratorIterator(const llvm::Init *const *it)
+        : llvm::mapped_iterator<const llvm::Init *const *,
+                                VariableDecorator (*)(const llvm::Init *)>(
+              it, &unwrap) {}
+    static VariableDecorator unwrap(const llvm::Init *init);
   };
   using var_decorator_iterator = VariableDecoratorIterator;
   using var_decorator_range = llvm::iterator_range<VariableDecoratorIterator>;
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
index 436675793062eb6fdc81d5cb35bd0c7d76c2eefb..9f300bcafea53737d71ba0809555cfdcf5af8164 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
@@ -199,6 +199,10 @@ public:
   /// Converts !llvm.ident metadata to the llvm.ident LLVM ModuleOp attribute.
   LogicalResult convertIdentMetadata();
 
+  /// Converts !llvm.commandline metadata to the llvm.commandline LLVM ModuleOp
+  /// attribute.
+  LogicalResult convertCommandlineMetadata();
+
   /// Converts all LLVM metadata nodes that translate to attributes such as
   /// alias analysis or access group metadata, and builds a map from the
   /// metadata nodes to the converted attributes.
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 3c85338bc642f690f77ae22236c3a28a37238cd6..ffeeeae57ae9521c351e49862e243f87ca2454d5 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -332,6 +332,9 @@ private:
   /// Process the ident LLVM Metadata, if it exists.
   LogicalResult createIdentMetadata();
 
+  /// Process the llvm.commandline LLVM Metadata, if it exists.
+  LogicalResult createCommandlineMetadata();
+
   /// Translates dialect attributes attached to the given operation.
   LogicalResult
   convertDialectAttributes(Operation *op,
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 65e279e046e8865a3d1a467d50ed6cd940a666bd..45ad6f8586daaeee65d61d17f35d2105a21fca82 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -138,7 +138,8 @@ public:
   };
 
   /// Register a conversion function. A conversion function must be convertible
-  /// to any of the following forms(where `T` is a class derived from `Type`:
+  /// to any of the following forms (where `T` is a class derived from `Type`):
+  ///
   ///   * std::optional<Type>(T)
   ///     - This form represents a 1-1 type conversion. It should return nullptr
   ///       or `std::nullopt` to signify failure. If `std::nullopt` is returned,
@@ -151,15 +152,7 @@ public:
   ///       existing value are expected to be removed during conversion. If
   ///       `std::nullopt` is returned, the converter is allowed to try another
   ///       conversion function to perform the conversion.
-  ///   * std::optional<LogicalResult>(T, SmallVectorImpl<Type> &,
-  ///                                  ArrayRef<Type>)
-  ///     - This form represents a 1-N type conversion supporting recursive
-  ///       types. The first two arguments and the return value are the same as
-  ///       for the regular 1-N form. The third argument is contains is the
-  ///       "call stack" of the recursive conversion: it contains the list of
-  ///       types currently being converted, with the current type being the
-  ///       last one. If it is present more than once in the list, the
-  ///       conversion concerns a recursive type.
+  ///
   /// Note: When attempting to convert a type, e.g. via 'convertType', the
   ///       mostly recently added conversions will be invoked first.
   template <typename FnT, typename T = typename llvm::function_traits<
@@ -178,6 +171,9 @@ public:
   /// it failed but other materialization can be attempted, and `nullptr` on
   /// unrecoverable failure. Materialization functions must be provided when a
   /// type conversion may persist after the conversion has finished.
+  ///
+  /// Note: Target materializations may optionally accept an additional Type
+  /// parameter, which is the original type of the SSA value.
 
   /// This method registers a materialization that will be called when
   /// converting (potentially multiple) block arguments that were the result of
@@ -203,11 +199,22 @@ public:
 
   /// This method registers a materialization that will be called when
   /// converting an illegal (source) value to a legal (target) type.
+  ///
+  /// Note: For target materializations, users can optionally take the original
+  /// type. This type may be different from the type of the input. For example,
+  /// let's assume that a conversion pattern "P1" replaced an SSA value "v1"
+  /// (type "t1") with "v2" (type "t2"). Then a different conversion pattern
+  /// "P2" matches an op that has "v1" as an operand. Let's furthermore assume
+  /// that "P2" determines that the legalized type of "t1" is "t3", which may
+  /// be different from "t2". In this example, the target materialization
+  /// will be invoked with: outputType = "t3", inputs = "v2",
+  // originalType = "t1". Note  that the original type "t1" cannot be recovered
+  /// from just "t3" and "v2"; that's why the originalType parameter exists.
   template <typename FnT, typename T = typename llvm::function_traits<
                               std::decay_t<FnT>>::template arg_t<1>>
   void addTargetMaterialization(FnT &&callback) {
     targetMaterializations.emplace_back(
-        wrapMaterialization<T>(std::forward<FnT>(callback)));
+        wrapTargetMaterialization<T>(std::forward<FnT>(callback)));
   }
 
   /// Register a conversion function for attributes within types. Type
@@ -303,21 +310,12 @@ public:
   /// `add*Materialization` for more information on the context for these
   /// methods.
   Value materializeArgumentConversion(OpBuilder &builder, Location loc,
-                                      Type resultType,
-                                      ValueRange inputs) const {
-    return materializeConversion(argumentMaterializations, builder, loc,
-                                 resultType, inputs);
-  }
+                                      Type resultType, ValueRange inputs) const;
   Value materializeSourceConversion(OpBuilder &builder, Location loc,
-                                    Type resultType, ValueRange inputs) const {
-    return materializeConversion(sourceMaterializations, builder, loc,
-                                 resultType, inputs);
-  }
+                                    Type resultType, ValueRange inputs) const;
   Value materializeTargetConversion(OpBuilder &builder, Location loc,
-                                    Type resultType, ValueRange inputs) const {
-    return materializeConversion(targetMaterializations, builder, loc,
-                                 resultType, inputs);
-  }
+                                    Type resultType, ValueRange inputs,
+                                    Type originalType = {}) const;
 
   /// Convert an attribute present `attr` from within the type `type` using
   /// the registered conversion functions. If no applicable conversion has been
@@ -333,21 +331,23 @@ private:
   using ConversionCallbackFn = std::function<std::optional<LogicalResult>(
       Type, SmallVectorImpl<Type> &)>;
 
-  /// The signature of the callback used to materialize a conversion.
+  /// The signature of the callback used to materialize a source/argument
+  /// conversion.
+  ///
+  /// Arguments: builder, result type, inputs, location
   using MaterializationCallbackFn = std::function<std::optional<Value>(
       OpBuilder &, Type, ValueRange, Location)>;
 
+  /// The signature of the callback used to materialize a target conversion.
+  ///
+  /// Arguments: builder, result type, inputs, location, original type
+  using TargetMaterializationCallbackFn = std::function<std::optional<Value>(
+      OpBuilder &, Type, ValueRange, Location, Type)>;
+
   /// The signature of the callback used to convert a type attribute.
   using TypeAttributeConversionCallbackFn =
       std::function<AttributeConversionResult(Type, Attribute)>;
 
-  /// Attempt to materialize a conversion using one of the provided
-  /// materialization functions.
-  Value
-  materializeConversion(ArrayRef<MaterializationCallbackFn> materializations,
-                        OpBuilder &builder, Location loc, Type resultType,
-                        ValueRange inputs) const;
-
   /// Generate a wrapper for the given callback. This allows for accepting
   /// different callback forms, that all compose into a single version.
   /// With callback of form: `std::optional<Type>(T)`
@@ -388,9 +388,10 @@ private:
     cachedMultiConversions.clear();
   }
 
-  /// Generate a wrapper for the given materialization callback. The callback
-  /// may take any subclass of `Type` and the wrapper will check for the target
-  /// type to be of the expected class before calling the callback.
+  /// Generate a wrapper for the given argument/source materialization
+  /// callback. The callback may take any subclass of `Type` and the
+  /// wrapper will check for the target type to be of the expected class
+  /// before calling the callback.
   template <typename T, typename FnT>
   MaterializationCallbackFn wrapMaterialization(FnT &&callback) const {
     return [callback = std::forward<FnT>(callback)](
@@ -402,6 +403,41 @@ private:
     };
   }
 
+  /// Generate a wrapper for the given target materialization callback.
+  /// The callback may take any subclass of `Type` and the wrapper will check
+  /// for the target type to be of the expected class before calling the
+  /// callback.
+  ///
+  /// With callback of form:
+  /// `Value(OpBuilder &, T, ValueRange, Location, Type)`
+  template <typename T, typename FnT>
+  std::enable_if_t<
+      std::is_invocable_v<FnT, OpBuilder &, T, ValueRange, Location, Type>,
+      TargetMaterializationCallbackFn>
+  wrapTargetMaterialization(FnT &&callback) const {
+    return [callback = std::forward<FnT>(callback)](
+               OpBuilder &builder, Type resultType, ValueRange inputs,
+               Location loc, Type originalType) -> std::optional<Value> {
+      if (T derivedType = dyn_cast<T>(resultType))
+        return callback(builder, derivedType, inputs, loc, originalType);
+      return std::nullopt;
+    };
+  }
+  /// With callback of form:
+  /// `Value(OpBuilder &, T, ValueRange, Location)`
+  template <typename T, typename FnT>
+  std::enable_if_t<
+      std::is_invocable_v<FnT, OpBuilder &, T, ValueRange, Location>,
+      TargetMaterializationCallbackFn>
+  wrapTargetMaterialization(FnT &&callback) const {
+    return wrapTargetMaterialization<T>(
+        [callback = std::forward<FnT>(callback)](
+            OpBuilder &builder, T resultType, ValueRange inputs, Location loc,
+            Type originalType) -> std::optional<Value> {
+          return callback(builder, resultType, inputs, loc);
+        });
+  }
+
   /// Generate a wrapper for the given memory space conversion callback. The
   /// callback may take any subclass of `Attribute` and the wrapper will check
   /// for the target attribute to be of the expected class before calling the
@@ -434,7 +470,7 @@ private:
   /// The list of registered materialization functions.
   SmallVector<MaterializationCallbackFn, 2> argumentMaterializations;
   SmallVector<MaterializationCallbackFn, 2> sourceMaterializations;
-  SmallVector<MaterializationCallbackFn, 2> targetMaterializations;
+  SmallVector<TargetMaterializationCallbackFn, 2> targetMaterializations;
 
   /// The list of registered type attribute conversion functions.
   SmallVector<TypeAttributeConversionCallbackFn, 2> typeAttributeConversions;
diff --git a/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp b/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
index beb68018a3b16e80cea820b5fdb25ba02237817d..3c190d4e9919192e7fc99b0b14539698e378b221 100644
--- a/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
@@ -46,22 +46,23 @@ void Executable::print(raw_ostream &os) const {
 void Executable::onUpdate(DataFlowSolver *solver) const {
   AnalysisState::onUpdate(solver);
 
-  if (ProgramPoint pp = llvm::dyn_cast_if_present<ProgramPoint>(anchor)) {
-    if (Block *block = llvm::dyn_cast_if_present<Block *>(pp)) {
+  if (ProgramPoint *pp = llvm::dyn_cast_if_present<ProgramPoint *>(anchor)) {
+    if (pp->isBlockStart()) {
       // Re-invoke the analyses on the block itself.
       for (DataFlowAnalysis *analysis : subscribers)
-        solver->enqueue({block, analysis});
+        solver->enqueue({pp, analysis});
       // Re-invoke the analyses on all operations in the block.
       for (DataFlowAnalysis *analysis : subscribers)
-        for (Operation &op : *block)
-          solver->enqueue({&op, analysis});
+        for (Operation &op : *pp->getBlock())
+          solver->enqueue({solver->getProgramPointAfter(&op), analysis});
     }
   } else if (auto *latticeAnchor =
                  llvm::dyn_cast_if_present<GenericLatticeAnchor *>(anchor)) {
     // Re-invoke the analysis on the successor block.
     if (auto *edge = dyn_cast<CFGEdge>(latticeAnchor)) {
       for (DataFlowAnalysis *analysis : subscribers)
-        solver->enqueue({edge->getTo(), analysis});
+        solver->enqueue(
+            {solver->getProgramPointBefore(edge->getTo()), analysis});
     }
   }
 }
@@ -125,7 +126,8 @@ LogicalResult DeadCodeAnalysis::initialize(Operation *top) {
   for (Region &region : top->getRegions()) {
     if (region.empty())
       continue;
-    auto *state = getOrCreate<Executable>(&region.front());
+    auto *state =
+        getOrCreate<Executable>(getProgramPointBefore(&region.front()));
     propagateIfChanged(state, state->setToLive());
   }
 
@@ -154,7 +156,8 @@ void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
       // Public symbol callables or those for which we can't see all uses have
       // potentially unknown callsites.
       if (symbol.isPublic() || (!allUsesVisible && symbol.isNested())) {
-        auto *state = getOrCreate<PredecessorState>(callable);
+        auto *state =
+            getOrCreate<PredecessorState>(getProgramPointAfter(callable));
         propagateIfChanged(state, state->setHasUnknownPredecessors());
       }
       foundSymbolCallable = true;
@@ -171,7 +174,8 @@ void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
       // If we couldn't gather the symbol uses, conservatively assume that
       // we can't track information for any nested symbols.
       return top->walk([&](CallableOpInterface callable) {
-        auto *state = getOrCreate<PredecessorState>(callable);
+        auto *state =
+            getOrCreate<PredecessorState>(getProgramPointAfter(callable));
         propagateIfChanged(state, state->setHasUnknownPredecessors());
       });
     }
@@ -182,7 +186,7 @@ void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
       // If a callable symbol has a non-call use, then we can't be guaranteed to
       // know all callsites.
       Operation *symbol = symbolTable.lookupSymbolIn(top, use.getSymbolRef());
-      auto *state = getOrCreate<PredecessorState>(symbol);
+      auto *state = getOrCreate<PredecessorState>(getProgramPointAfter(symbol));
       propagateIfChanged(state, state->setHasUnknownPredecessors());
     }
   };
@@ -193,7 +197,7 @@ void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
 /// Returns true if the operation is a returning terminator in region
 /// control-flow or the terminator of a callable region.
 static bool isRegionOrCallableReturn(Operation *op) {
-  return !op->getNumSuccessors() &&
+  return op->getBlock() != nullptr && !op->getNumSuccessors() &&
          isa<RegionBranchOpInterface, CallableOpInterface>(op->getParentOp()) &&
          op->getBlock()->getTerminator() == op;
 }
@@ -205,9 +209,10 @@ LogicalResult DeadCodeAnalysis::initializeRecursively(Operation *op) {
     // When the liveness of the parent block changes, make sure to re-invoke the
     // analysis on the op.
     if (op->getBlock())
-      getOrCreate<Executable>(op->getBlock())->blockContentSubscribe(this);
+      getOrCreate<Executable>(getProgramPointBefore(op->getBlock()))
+          ->blockContentSubscribe(this);
     // Visit the op.
-    if (failed(visit(op)))
+    if (failed(visit(getProgramPointAfter(op))))
       return failure();
   }
   // Recurse on nested operations.
@@ -219,7 +224,7 @@ LogicalResult DeadCodeAnalysis::initializeRecursively(Operation *op) {
 }
 
 void DeadCodeAnalysis::markEdgeLive(Block *from, Block *to) {
-  auto *state = getOrCreate<Executable>(to);
+  auto *state = getOrCreate<Executable>(getProgramPointBefore(to));
   propagateIfChanged(state, state->setToLive());
   auto *edgeState =
       getOrCreate<Executable>(getLatticeAnchor<CFGEdge>(from, to));
@@ -230,18 +235,20 @@ void DeadCodeAnalysis::markEntryBlocksLive(Operation *op) {
   for (Region &region : op->getRegions()) {
     if (region.empty())
       continue;
-    auto *state = getOrCreate<Executable>(&region.front());
+    auto *state =
+        getOrCreate<Executable>(getProgramPointBefore(&region.front()));
     propagateIfChanged(state, state->setToLive());
   }
 }
 
-LogicalResult DeadCodeAnalysis::visit(ProgramPoint point) {
-  if (point.is<Block *>())
+LogicalResult DeadCodeAnalysis::visit(ProgramPoint *point) {
+  if (point->isBlockStart())
     return success();
-  auto *op = point.get<Operation *>();
+  Operation *op = point->getPrevOp();
 
   // If the parent block is not executable, there is nothing to do.
-  if (!getOrCreate<Executable>(op->getBlock())->isLive())
+  if (op->getBlock() != nullptr &&
+      !getOrCreate<Executable>(getProgramPointBefore(op->getBlock()))->isLive())
     return success();
 
   // We have a live call op. Add this as a live predecessor of the callee.
@@ -256,7 +263,8 @@ LogicalResult DeadCodeAnalysis::visit(ProgramPoint point) {
 
       // Check if this is a callable operation.
     } else if (auto callable = dyn_cast<CallableOpInterface>(op)) {
-      const auto *callsites = getOrCreateFor<PredecessorState>(op, callable);
+      const auto *callsites = getOrCreateFor<PredecessorState>(
+          getProgramPointAfter(op), getProgramPointAfter(callable));
 
       // If the callsites could not be resolved or are known to be non-empty,
       // mark the callable as executable.
@@ -316,11 +324,13 @@ void DeadCodeAnalysis::visitCallOperation(CallOpInterface call) {
   if (isa_and_nonnull<SymbolOpInterface>(callableOp) &&
       !isExternalCallable(callableOp)) {
     // Add the live callsite.
-    auto *callsites = getOrCreate<PredecessorState>(callableOp);
+    auto *callsites =
+        getOrCreate<PredecessorState>(getProgramPointAfter(callableOp));
     propagateIfChanged(callsites, callsites->join(call));
   } else {
     // Mark this call op's predecessors as overdefined.
-    auto *predecessors = getOrCreate<PredecessorState>(call);
+    auto *predecessors =
+        getOrCreate<PredecessorState>(getProgramPointAfter(call));
     propagateIfChanged(predecessors, predecessors->setHasUnknownPredecessors());
   }
 }
@@ -378,9 +388,10 @@ void DeadCodeAnalysis::visitRegionBranchOperation(
   branch.getEntrySuccessorRegions(*operands, successors);
   for (const RegionSuccessor &successor : successors) {
     // The successor can be either an entry block or the parent operation.
-    ProgramPoint point = successor.getSuccessor()
-                             ? &successor.getSuccessor()->front()
-                             : ProgramPoint(branch);
+    ProgramPoint *point =
+        successor.getSuccessor()
+            ? getProgramPointBefore(&successor.getSuccessor()->front())
+            : getProgramPointAfter(branch);
     // Mark the entry block as executable.
     auto *state = getOrCreate<Executable>(point);
     propagateIfChanged(state, state->setToLive());
@@ -409,12 +420,15 @@ void DeadCodeAnalysis::visitRegionTerminator(Operation *op,
   for (const RegionSuccessor &successor : successors) {
     PredecessorState *predecessors;
     if (Region *region = successor.getSuccessor()) {
-      auto *state = getOrCreate<Executable>(&region->front());
+      auto *state =
+          getOrCreate<Executable>(getProgramPointBefore(&region->front()));
       propagateIfChanged(state, state->setToLive());
-      predecessors = getOrCreate<PredecessorState>(&region->front());
+      predecessors = getOrCreate<PredecessorState>(
+          getProgramPointBefore(&region->front()));
     } else {
       // Add this terminator as a predecessor to the parent op.
-      predecessors = getOrCreate<PredecessorState>(branch);
+      predecessors =
+          getOrCreate<PredecessorState>(getProgramPointAfter(branch));
     }
     propagateIfChanged(predecessors,
                        predecessors->join(op, successor.getSuccessorInputs()));
@@ -424,11 +438,13 @@ void DeadCodeAnalysis::visitRegionTerminator(Operation *op,
 void DeadCodeAnalysis::visitCallableTerminator(Operation *op,
                                                CallableOpInterface callable) {
   // Add as predecessors to all callsites this return op.
-  auto *callsites = getOrCreateFor<PredecessorState>(op, callable);
+  auto *callsites = getOrCreateFor<PredecessorState>(
+      getProgramPointAfter(op), getProgramPointAfter(callable));
   bool canResolve = op->hasTrait<OpTrait::ReturnLike>();
   for (Operation *predecessor : callsites->getKnownPredecessors()) {
     assert(isa<CallOpInterface>(predecessor));
-    auto *predecessors = getOrCreate<PredecessorState>(predecessor);
+    auto *predecessors =
+        getOrCreate<PredecessorState>(getProgramPointAfter(predecessor));
     if (canResolve) {
       propagateIfChanged(predecessors, predecessors->join(op));
     } else {
diff --git a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
index 300c6e5f9b891918f147a10cabc249cae7775b85..340aa399ec12e5774787caf25bc614707204027b 100644
--- a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
@@ -44,10 +44,10 @@ LogicalResult AbstractDenseForwardDataFlowAnalysis::initialize(Operation *top) {
   return success();
 }
 
-LogicalResult AbstractDenseForwardDataFlowAnalysis::visit(ProgramPoint point) {
-  if (auto *op = llvm::dyn_cast_if_present<Operation *>(point))
-    return processOperation(op);
-  visitBlock(point.get<Block *>());
+LogicalResult AbstractDenseForwardDataFlowAnalysis::visit(ProgramPoint *point) {
+  if (!point->isBlockStart())
+    return processOperation(point->getPrevOp());
+  visitBlock(point->getBlock());
   return success();
 }
 
@@ -64,8 +64,8 @@ void AbstractDenseForwardDataFlowAnalysis::visitCallOperation(
         call, CallControlFlowAction::ExternalCallee, before, after);
   }
 
-  const auto *predecessors =
-      getOrCreateFor<PredecessorState>(call.getOperation(), call);
+  const auto *predecessors = getOrCreateFor<PredecessorState>(
+      getProgramPointAfter(call.getOperation()), getProgramPointAfter(call));
   // Otherwise, if not all return sites are known, then conservatively assume we
   // can't reason about the data-flow.
   if (!predecessors->allPredecessorsKnown())
@@ -87,7 +87,8 @@ void AbstractDenseForwardDataFlowAnalysis::visitCallOperation(
     //   }
     AbstractDenseLattice *latticeAfterCall = after;
     const AbstractDenseLattice *latticeAtCalleeReturn =
-        getLatticeFor(call.getOperation(), predecessor);
+        getLatticeFor(getProgramPointAfter(call.getOperation()),
+                      getProgramPointAfter(predecessor));
     visitCallControlFlowTransfer(call, CallControlFlowAction::ExitCallee,
                                  *latticeAtCalleeReturn, latticeAfterCall);
   }
@@ -95,24 +96,24 @@ void AbstractDenseForwardDataFlowAnalysis::visitCallOperation(
 
 LogicalResult
 AbstractDenseForwardDataFlowAnalysis::processOperation(Operation *op) {
+  ProgramPoint *point = getProgramPointAfter(op);
   // If the containing block is not executable, bail out.
-  if (!getOrCreateFor<Executable>(op, op->getBlock())->isLive())
+  if (op->getBlock() != nullptr &&
+      !getOrCreateFor<Executable>(point, getProgramPointBefore(op->getBlock()))
+           ->isLive())
     return success();
 
   // Get the dense lattice to update.
-  AbstractDenseLattice *after = getLattice(op);
+  AbstractDenseLattice *after = getLattice(point);
 
   // Get the dense state before the execution of the op.
-  const AbstractDenseLattice *before;
-  if (Operation *prev = op->getPrevNode())
-    before = getLatticeFor(op, prev);
-  else
-    before = getLatticeFor(op, op->getBlock());
+  const AbstractDenseLattice *before =
+      getLatticeFor(point, getProgramPointBefore(op));
 
   // If this op implements region control-flow, then control-flow dictates its
   // transfer function.
   if (auto branch = dyn_cast<RegionBranchOpInterface>(op)) {
-    visitRegionBranchOperation(op, branch, after);
+    visitRegionBranchOperation(point, branch, after);
     return success();
   }
 
@@ -129,11 +130,12 @@ AbstractDenseForwardDataFlowAnalysis::processOperation(Operation *op) {
 
 void AbstractDenseForwardDataFlowAnalysis::visitBlock(Block *block) {
   // If the block is not executable, bail out.
-  if (!getOrCreateFor<Executable>(block, block)->isLive())
+  ProgramPoint *point = getProgramPointBefore(block);
+  if (!getOrCreateFor<Executable>(point, point)->isLive())
     return;
 
   // Get the dense lattice to update.
-  AbstractDenseLattice *after = getLattice(block);
+  AbstractDenseLattice *after = getLattice(point);
 
   // The dense lattices of entry blocks are set by region control-flow or the
   // callgraph.
@@ -141,7 +143,8 @@ void AbstractDenseForwardDataFlowAnalysis::visitBlock(Block *block) {
     // Check if this block is the entry block of a callable region.
     auto callable = dyn_cast<CallableOpInterface>(block->getParentOp());
     if (callable && callable.getCallableRegion() == block->getParent()) {
-      const auto *callsites = getOrCreateFor<PredecessorState>(block, callable);
+      const auto *callsites = getOrCreateFor<PredecessorState>(
+          point, getProgramPointAfter(callable));
       // If not all callsites are known, conservatively mark all lattices as
       // having reached their pessimistic fixpoints. Do the same if
       // interprocedural analysis is not enabled.
@@ -151,10 +154,7 @@ void AbstractDenseForwardDataFlowAnalysis::visitBlock(Block *block) {
       for (Operation *callsite : callsites->getKnownPredecessors()) {
         // Get the dense lattice before the callsite.
         const AbstractDenseLattice *before;
-        if (Operation *prev = callsite->getPrevNode())
-          before = getLatticeFor(block, prev);
-        else
-          before = getLatticeFor(block, callsite->getBlock());
+        before = getLatticeFor(point, getProgramPointBefore(callsite));
 
         visitCallControlFlowTransfer(cast<CallOpInterface>(callsite),
                                      CallControlFlowAction::EnterCallee,
@@ -165,7 +165,7 @@ void AbstractDenseForwardDataFlowAnalysis::visitBlock(Block *block) {
 
     // Check if we can reason about the control-flow.
     if (auto branch = dyn_cast<RegionBranchOpInterface>(block->getParentOp()))
-      return visitRegionBranchOperation(block, branch, after);
+      return visitRegionBranchOperation(point, branch, after);
 
     // Otherwise, we can't reason about the data-flow.
     return setToEntryState(after);
@@ -177,17 +177,18 @@ void AbstractDenseForwardDataFlowAnalysis::visitBlock(Block *block) {
     // Skip control edges that aren't executable.
     Block *predecessor = *it;
     if (!getOrCreateFor<Executable>(
-             block, getLatticeAnchor<CFGEdge>(predecessor, block))
+             point, getLatticeAnchor<CFGEdge>(predecessor, block))
              ->isLive())
       continue;
 
     // Merge in the state from the predecessor's terminator.
-    join(after, *getLatticeFor(block, predecessor->getTerminator()));
+    join(after, *getLatticeFor(
+                    point, getProgramPointAfter(predecessor->getTerminator())));
   }
 }
 
 void AbstractDenseForwardDataFlowAnalysis::visitRegionBranchOperation(
-    ProgramPoint point, RegionBranchOpInterface branch,
+    ProgramPoint *point, RegionBranchOpInterface branch,
     AbstractDenseLattice *after) {
   // Get the terminator predecessors.
   const auto *predecessors = getOrCreateFor<PredecessorState>(point, point);
@@ -198,19 +199,15 @@ void AbstractDenseForwardDataFlowAnalysis::visitRegionBranchOperation(
     const AbstractDenseLattice *before;
     // If the predecessor is the parent, get the state before the parent.
     if (op == branch) {
-      if (Operation *prev = op->getPrevNode())
-        before = getLatticeFor(point, prev);
-      else
-        before = getLatticeFor(point, op->getBlock());
-
+      before = getLatticeFor(point, getProgramPointBefore(op));
       // Otherwise, get the state after the terminator.
     } else {
-      before = getLatticeFor(point, op);
+      before = getLatticeFor(point, getProgramPointAfter(op));
     }
 
     // This function is called in two cases:
-    //   1. when visiting the block (point = block);
-    //   2. when visiting the parent operation (point = parent op).
+    //   1. when visiting the block (point = block start);
+    //   2. when visiting the parent operation (point = iter after parent op).
     // In both cases, we are looking for predecessor operations of the point,
     //   1. predecessor may be the terminator of another block from another
     //   region (assuming that the block does belong to another region via an
@@ -224,12 +221,12 @@ void AbstractDenseForwardDataFlowAnalysis::visitRegionBranchOperation(
     std::optional<unsigned> regionFrom =
         op == branch ? std::optional<unsigned>()
                      : op->getBlock()->getParent()->getRegionNumber();
-    if (auto *toBlock = point.dyn_cast<Block *>()) {
-      unsigned regionTo = toBlock->getParent()->getRegionNumber();
+    if (point->isBlockStart()) {
+      unsigned regionTo = point->getBlock()->getParent()->getRegionNumber();
       visitRegionBranchControlFlowTransfer(branch, regionFrom, regionTo,
                                            *before, after);
     } else {
-      assert(point.get<Operation *>() == branch &&
+      assert(point->getPrevOp() == branch &&
              "expected to be visiting the branch itself");
       // Only need to call the arc transfer when the predecessor is the region
       // or the op itself, not the previous op.
@@ -244,7 +241,7 @@ void AbstractDenseForwardDataFlowAnalysis::visitRegionBranchOperation(
 }
 
 const AbstractDenseLattice *
-AbstractDenseForwardDataFlowAnalysis::getLatticeFor(ProgramPoint dependent,
+AbstractDenseForwardDataFlowAnalysis::getLatticeFor(ProgramPoint *dependent,
                                                     LatticeAnchor anchor) {
   AbstractDenseLattice *state = getLattice(anchor);
   addDependency(state, dependent);
@@ -273,10 +270,11 @@ AbstractDenseBackwardDataFlowAnalysis::initialize(Operation *top) {
   return success();
 }
 
-LogicalResult AbstractDenseBackwardDataFlowAnalysis::visit(ProgramPoint point) {
-  if (auto *op = llvm::dyn_cast_if_present<Operation *>(point))
-    return processOperation(op);
-  visitBlock(point.get<Block *>());
+LogicalResult
+AbstractDenseBackwardDataFlowAnalysis::visit(ProgramPoint *point) {
+  if (!point->isBlockEnd())
+    return processOperation(point->getNextOp());
+  visitBlock(point->getBlock());
   return success();
 }
 
@@ -316,11 +314,9 @@ void AbstractDenseBackwardDataFlowAnalysis::visitCallOperation(
   //     ...
   //   }
   Block *calleeEntryBlock = &region->front();
-  ProgramPoint calleeEntry = calleeEntryBlock->empty()
-                                 ? ProgramPoint(calleeEntryBlock)
-                                 : &calleeEntryBlock->front();
+  ProgramPoint *calleeEntry = getProgramPointBefore(calleeEntryBlock);
   const AbstractDenseLattice &latticeAtCalleeEntry =
-      *getLatticeFor(call.getOperation(), calleeEntry);
+      *getLatticeFor(getProgramPointBefore(call.getOperation()), calleeEntry);
   AbstractDenseLattice *latticeBeforeCall = before;
   visitCallControlFlowTransfer(call, CallControlFlowAction::EnterCallee,
                                latticeAtCalleeEntry, latticeBeforeCall);
@@ -328,23 +324,24 @@ void AbstractDenseBackwardDataFlowAnalysis::visitCallOperation(
 
 LogicalResult
 AbstractDenseBackwardDataFlowAnalysis::processOperation(Operation *op) {
+  ProgramPoint *point = getProgramPointBefore(op);
   // If the containing block is not executable, bail out.
-  if (!getOrCreateFor<Executable>(op, op->getBlock())->isLive())
+  if (op->getBlock() != nullptr &&
+      !getOrCreateFor<Executable>(point, getProgramPointBefore(op->getBlock()))
+           ->isLive())
     return success();
 
   // Get the dense lattice to update.
-  AbstractDenseLattice *before = getLattice(op);
+  AbstractDenseLattice *before = getLattice(point);
 
   // Get the dense state after execution of this op.
-  const AbstractDenseLattice *after;
-  if (Operation *next = op->getNextNode())
-    after = getLatticeFor(op, next);
-  else
-    after = getLatticeFor(op, op->getBlock());
+  const AbstractDenseLattice *after =
+      getLatticeFor(point, getProgramPointAfter(op));
 
   // Special cases where control flow may dictate data flow.
   if (auto branch = dyn_cast<RegionBranchOpInterface>(op)) {
-    visitRegionBranchOperation(op, branch, RegionBranchPoint::parent(), before);
+    visitRegionBranchOperation(point, branch, RegionBranchPoint::parent(),
+                               before);
     return success();
   }
   if (auto call = dyn_cast<CallOpInterface>(op)) {
@@ -357,11 +354,13 @@ AbstractDenseBackwardDataFlowAnalysis::processOperation(Operation *op) {
 }
 
 void AbstractDenseBackwardDataFlowAnalysis::visitBlock(Block *block) {
+  ProgramPoint *point = getProgramPointAfter(block);
   // If the block is not executable, bail out.
-  if (!getOrCreateFor<Executable>(block, block)->isLive())
+  if (!getOrCreateFor<Executable>(point, getProgramPointBefore(block))
+           ->isLive())
     return;
 
-  AbstractDenseLattice *before = getLattice(block);
+  AbstractDenseLattice *before = getLattice(point);
 
   // We need "exit" blocks, i.e. the blocks that may return control to the
   // parent operation.
@@ -382,7 +381,8 @@ void AbstractDenseBackwardDataFlowAnalysis::visitBlock(Block *block) {
     // themselves are predecessors of the callable.
     auto callable = dyn_cast<CallableOpInterface>(block->getParentOp());
     if (callable && callable.getCallableRegion() == block->getParent()) {
-      const auto *callsites = getOrCreateFor<PredecessorState>(block, callable);
+      const auto *callsites = getOrCreateFor<PredecessorState>(
+          point, getProgramPointAfter(callable));
       // If not all call sites are known, conservative mark all lattices as
       // having reached their pessimistic fix points.
       if (!callsites->allPredecessorsKnown() ||
@@ -391,11 +391,8 @@ void AbstractDenseBackwardDataFlowAnalysis::visitBlock(Block *block) {
       }
 
       for (Operation *callsite : callsites->getKnownPredecessors()) {
-        const AbstractDenseLattice *after;
-        if (Operation *next = callsite->getNextNode())
-          after = getLatticeFor(block, next);
-        else
-          after = getLatticeFor(block, callsite->getBlock());
+        const AbstractDenseLattice *after =
+            getLatticeFor(point, getProgramPointAfter(callsite));
         visitCallControlFlowTransfer(cast<CallOpInterface>(callsite),
                                      CallControlFlowAction::ExitCallee, *after,
                                      before);
@@ -406,7 +403,7 @@ void AbstractDenseBackwardDataFlowAnalysis::visitBlock(Block *block) {
     // If this block is exiting from an operation with region-based control
     // flow, propagate the lattice back along the control flow edge.
     if (auto branch = dyn_cast<RegionBranchOpInterface>(block->getParentOp())) {
-      visitRegionBranchOperation(block, branch, block->getParent(), before);
+      visitRegionBranchOperation(point, branch, block->getParent(), before);
       return;
     }
 
@@ -417,22 +414,19 @@ void AbstractDenseBackwardDataFlowAnalysis::visitBlock(Block *block) {
 
   // Meet the state with the state before block's successors.
   for (Block *successor : block->getSuccessors()) {
-    if (!getOrCreateFor<Executable>(block,
+    if (!getOrCreateFor<Executable>(point,
                                     getLatticeAnchor<CFGEdge>(block, successor))
              ->isLive())
       continue;
 
     // Merge in the state from the successor: either the first operation, or the
     // block itself when empty.
-    if (successor->empty())
-      meet(before, *getLatticeFor(block, successor));
-    else
-      meet(before, *getLatticeFor(block, &successor->front()));
+    meet(before, *getLatticeFor(point, getProgramPointBefore(successor)));
   }
 }
 
 void AbstractDenseBackwardDataFlowAnalysis::visitRegionBranchOperation(
-    ProgramPoint point, RegionBranchOpInterface branch,
+    ProgramPoint *point, RegionBranchOpInterface branch,
     RegionBranchPoint branchPoint, AbstractDenseLattice *before) {
 
   // The successors of the operation may be either the first operation of the
@@ -443,22 +437,18 @@ void AbstractDenseBackwardDataFlowAnalysis::visitRegionBranchOperation(
   for (const RegionSuccessor &successor : successors) {
     const AbstractDenseLattice *after;
     if (successor.isParent() || successor.getSuccessor()->empty()) {
-      if (Operation *next = branch->getNextNode())
-        after = getLatticeFor(point, next);
-      else
-        after = getLatticeFor(point, branch->getBlock());
+      after = getLatticeFor(point, getProgramPointAfter(branch));
     } else {
       Region *successorRegion = successor.getSuccessor();
       assert(!successorRegion->empty() && "unexpected empty successor region");
       Block *successorBlock = &successorRegion->front();
 
-      if (!getOrCreateFor<Executable>(point, successorBlock)->isLive())
+      if (!getOrCreateFor<Executable>(point,
+                                      getProgramPointBefore(successorBlock))
+               ->isLive())
         continue;
 
-      if (successorBlock->empty())
-        after = getLatticeFor(point, successorBlock);
-      else
-        after = getLatticeFor(point, &successorBlock->front());
+      after = getLatticeFor(point, getProgramPointBefore(successorBlock));
     }
 
     visitRegionBranchControlFlowTransfer(branch, branchPoint, successor, *after,
@@ -467,7 +457,7 @@ void AbstractDenseBackwardDataFlowAnalysis::visitRegionBranchOperation(
 }
 
 const AbstractDenseLattice *
-AbstractDenseBackwardDataFlowAnalysis::getLatticeFor(ProgramPoint dependent,
+AbstractDenseBackwardDataFlowAnalysis::getLatticeFor(ProgramPoint *dependent,
                                                      LatticeAnchor anchor) {
   AbstractDenseLattice *state = getLattice(anchor);
   addDependency(state, dependent);
diff --git a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp
index 9a95f172d5df48ed0c67f81e68241c03461d84d8..bf9eabbedc3a1fed67ac21571428a83f366c358d 100644
--- a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp
@@ -111,7 +111,7 @@ void IntegerRangeAnalysis::visitNonControlFlowArguments(
     LLVM_DEBUG(llvm::dbgs() << "Inferring ranges for " << *op << "\n");
 
     auto argRanges = llvm::map_to_vector(op->getOperands(), [&](Value value) {
-      return getLatticeElementFor(op, value)->getValue();
+      return getLatticeElementFor(getProgramPointAfter(op), value)->getValue();
     });
 
     auto joinCallback = [&](Value v, const IntegerValueRange &attrs) {
@@ -159,7 +159,7 @@ void IntegerRangeAnalysis::visitNonControlFlowArguments(
           return bound.getValue();
       } else if (auto value = llvm::dyn_cast_if_present<Value>(*loopBound)) {
         const IntegerValueRangeLattice *lattice =
-            getLatticeElementFor(op, value);
+            getLatticeElementFor(getProgramPointAfter(op), value);
         if (lattice != nullptr && !lattice->getValue().isUninitialized())
           return getUpper ? lattice->getValue().getValue().smax()
                           : lattice->getValue().getValue().smin();
diff --git a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
index 57a4d4a6800be067502fa88bdbe6842b68f19bb1..9fb4d9df2530df0a648970a506248d09e5995169 100644
--- a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
@@ -87,7 +87,7 @@ LivenessAnalysis::visitOperation(Operation *op, ArrayRef<Liveness *> operands,
         meet(operand, *r);
       foundLiveResult = true;
     }
-    addDependency(const_cast<Liveness *>(r), op);
+    addDependency(const_cast<Liveness *>(r), getProgramPointAfter(op));
   }
   return success();
 }
diff --git a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
index 1bd6defef90be0c91b3665264faa68b08b7d1d77..67cf8c9c5b81f21494846f7658ca5f9fbdcfdb5d 100644
--- a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
@@ -36,7 +36,7 @@ void AbstractSparseLattice::onUpdate(DataFlowSolver *solver) const {
   // Push all users of the value to the queue.
   for (Operation *user : anchor.get<Value>().getUsers())
     for (DataFlowAnalysis *analysis : useDefSubscribers)
-      solver->enqueue({user, analysis});
+      solver->enqueue({solver->getProgramPointAfter(user), analysis});
 }
 
 //===----------------------------------------------------------------------===//
@@ -72,7 +72,8 @@ AbstractSparseForwardDataFlowAnalysis::initializeRecursively(Operation *op) {
 
   for (Region &region : op->getRegions()) {
     for (Block &block : region) {
-      getOrCreate<Executable>(&block)->blockContentSubscribe(this);
+      getOrCreate<Executable>(getProgramPointBefore(&block))
+          ->blockContentSubscribe(this);
       visitBlock(&block);
       for (Operation &op : block)
         if (failed(initializeRecursively(&op)))
@@ -83,10 +84,11 @@ AbstractSparseForwardDataFlowAnalysis::initializeRecursively(Operation *op) {
   return success();
 }
 
-LogicalResult AbstractSparseForwardDataFlowAnalysis::visit(ProgramPoint point) {
-  if (Operation *op = llvm::dyn_cast_if_present<Operation *>(point))
-    return visitOperation(op);
-  visitBlock(point.get<Block *>());
+LogicalResult
+AbstractSparseForwardDataFlowAnalysis::visit(ProgramPoint *point) {
+  if (!point->isBlockStart())
+    return visitOperation(point->getPrevOp());
+  visitBlock(point->getBlock());
   return success();
 }
 
@@ -97,7 +99,8 @@ AbstractSparseForwardDataFlowAnalysis::visitOperation(Operation *op) {
     return success();
 
   // If the containing block is not executable, bail out.
-  if (!getOrCreate<Executable>(op->getBlock())->isLive())
+  if (op->getBlock() != nullptr &&
+      !getOrCreate<Executable>(getProgramPointBefore(op->getBlock()))->isLive())
     return success();
 
   // Get the result lattices.
@@ -110,7 +113,7 @@ AbstractSparseForwardDataFlowAnalysis::visitOperation(Operation *op) {
 
   // The results of a region branch operation are determined by control-flow.
   if (auto branch = dyn_cast<RegionBranchOpInterface>(op)) {
-    visitRegionSuccessors({branch}, branch,
+    visitRegionSuccessors(getProgramPointAfter(branch), branch,
                           /*successor=*/RegionBranchPoint::parent(),
                           resultLattices);
     return success();
@@ -138,7 +141,8 @@ AbstractSparseForwardDataFlowAnalysis::visitOperation(Operation *op) {
 
     // Otherwise, the results of a call operation are determined by the
     // callgraph.
-    const auto *predecessors = getOrCreateFor<PredecessorState>(op, call);
+    const auto *predecessors = getOrCreateFor<PredecessorState>(
+        getProgramPointAfter(op), getProgramPointAfter(call));
     // If not all return sites are known, then conservatively assume we can't
     // reason about the data-flow.
     if (!predecessors->allPredecessorsKnown()) {
@@ -148,7 +152,8 @@ AbstractSparseForwardDataFlowAnalysis::visitOperation(Operation *op) {
     for (Operation *predecessor : predecessors->getKnownPredecessors())
       for (auto &&[operand, resLattice] :
            llvm::zip(predecessor->getOperands(), resultLattices))
-        join(resLattice, *getLatticeElementFor(op, operand));
+        join(resLattice,
+             *getLatticeElementFor(getProgramPointAfter(op), operand));
     return success();
   }
 
@@ -162,7 +167,7 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) {
     return;
 
   // If the block is not executable, bail out.
-  if (!getOrCreate<Executable>(block)->isLive())
+  if (!getOrCreate<Executable>(getProgramPointBefore(block))->isLive())
     return;
 
   // Get the argument lattices.
@@ -179,7 +184,8 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) {
     // Check if this block is the entry block of a callable region.
     auto callable = dyn_cast<CallableOpInterface>(block->getParentOp());
     if (callable && callable.getCallableRegion() == block->getParent()) {
-      const auto *callsites = getOrCreateFor<PredecessorState>(block, callable);
+      const auto *callsites = getOrCreateFor<PredecessorState>(
+          getProgramPointBefore(block), getProgramPointAfter(callable));
       // If not all callsites are known, conservatively mark all lattices as
       // having reached their pessimistic fixpoints.
       if (!callsites->allPredecessorsKnown() ||
@@ -189,15 +195,17 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) {
       for (Operation *callsite : callsites->getKnownPredecessors()) {
         auto call = cast<CallOpInterface>(callsite);
         for (auto it : llvm::zip(call.getArgOperands(), argLattices))
-          join(std::get<1>(it), *getLatticeElementFor(block, std::get<0>(it)));
+          join(std::get<1>(it),
+               *getLatticeElementFor(getProgramPointBefore(block),
+                                     std::get<0>(it)));
       }
       return;
     }
 
     // Check if the lattices can be determined from region control flow.
     if (auto branch = dyn_cast<RegionBranchOpInterface>(block->getParentOp())) {
-      return visitRegionSuccessors(block, branch, block->getParent(),
-                                   argLattices);
+      return visitRegionSuccessors(getProgramPointBefore(block), branch,
+                                   block->getParent(), argLattices);
     }
 
     // Otherwise, we can't reason about the data-flow.
@@ -226,7 +234,8 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) {
           branch.getSuccessorOperands(it.getSuccessorIndex());
       for (auto [idx, lattice] : llvm::enumerate(argLattices)) {
         if (Value operand = operands[idx]) {
-          join(lattice, *getLatticeElementFor(block, operand));
+          join(lattice,
+               *getLatticeElementFor(getProgramPointBefore(block), operand));
         } else {
           // Conservatively consider internally produced arguments as entry
           // points.
@@ -240,7 +249,7 @@ void AbstractSparseForwardDataFlowAnalysis::visitBlock(Block *block) {
 }
 
 void AbstractSparseForwardDataFlowAnalysis::visitRegionSuccessors(
-    ProgramPoint point, RegionBranchOpInterface branch,
+    ProgramPoint *point, RegionBranchOpInterface branch,
     RegionBranchPoint successor, ArrayRef<AbstractSparseLattice *> lattices) {
   const auto *predecessors = getOrCreateFor<PredecessorState>(point, point);
   assert(predecessors->allPredecessorsKnown() &&
@@ -270,7 +279,7 @@ void AbstractSparseForwardDataFlowAnalysis::visitRegionSuccessors(
 
     unsigned firstIndex = 0;
     if (inputs.size() != lattices.size()) {
-      if (llvm::dyn_cast_if_present<Operation *>(point)) {
+      if (!point->isBlockStart()) {
         if (!inputs.empty())
           firstIndex = cast<OpResult>(inputs.front()).getResultNumber();
         visitNonControlFlowArgumentsImpl(
@@ -281,7 +290,7 @@ void AbstractSparseForwardDataFlowAnalysis::visitRegionSuccessors(
       } else {
         if (!inputs.empty())
           firstIndex = cast<BlockArgument>(inputs.front()).getArgNumber();
-        Region *region = point.get<Block *>()->getParent();
+        Region *region = point->getBlock()->getParent();
         visitNonControlFlowArgumentsImpl(
             branch,
             RegionSuccessor(region, region->getArguments().slice(
@@ -296,7 +305,7 @@ void AbstractSparseForwardDataFlowAnalysis::visitRegionSuccessors(
 }
 
 const AbstractSparseLattice *
-AbstractSparseForwardDataFlowAnalysis::getLatticeElementFor(ProgramPoint point,
+AbstractSparseForwardDataFlowAnalysis::getLatticeElementFor(ProgramPoint *point,
                                                             Value value) {
   AbstractSparseLattice *state = getLatticeElement(value);
   addDependency(state, point);
@@ -336,7 +345,8 @@ AbstractSparseBackwardDataFlowAnalysis::initializeRecursively(Operation *op) {
 
   for (Region &region : op->getRegions()) {
     for (Block &block : region) {
-      getOrCreate<Executable>(&block)->blockContentSubscribe(this);
+      getOrCreate<Executable>(getProgramPointBefore(&block))
+          ->blockContentSubscribe(this);
       // Initialize ops in reverse order, so we can do as much initial
       // propagation as possible without having to go through the
       // solver queue.
@@ -349,14 +359,14 @@ AbstractSparseBackwardDataFlowAnalysis::initializeRecursively(Operation *op) {
 }
 
 LogicalResult
-AbstractSparseBackwardDataFlowAnalysis::visit(ProgramPoint point) {
-  if (Operation *op = llvm::dyn_cast_if_present<Operation *>(point))
-    return visitOperation(op);
+AbstractSparseBackwardDataFlowAnalysis::visit(ProgramPoint *point) {
   // For backward dataflow, we don't have to do any work for the blocks
   // themselves. CFG edges between blocks are processed by the BranchOp
   // logic in `visitOperation`, and entry blocks for functions are tied
   // to the CallOp arguments by visitOperation.
-  return success();
+  if (point->isBlockStart())
+    return success();
+  return visitOperation(point->getPrevOp());
 }
 
 SmallVector<AbstractSparseLattice *>
@@ -372,7 +382,7 @@ AbstractSparseBackwardDataFlowAnalysis::getLatticeElements(ValueRange values) {
 
 SmallVector<const AbstractSparseLattice *>
 AbstractSparseBackwardDataFlowAnalysis::getLatticeElementsFor(
-    ProgramPoint point, ValueRange values) {
+    ProgramPoint *point, ValueRange values) {
   SmallVector<const AbstractSparseLattice *> resultLattices;
   resultLattices.reserve(values.size());
   for (Value result : values) {
@@ -390,13 +400,14 @@ static MutableArrayRef<OpOperand> operandsToOpOperands(OperandRange &operands) {
 LogicalResult
 AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
   // If we're in a dead block, bail out.
-  if (!getOrCreate<Executable>(op->getBlock())->isLive())
+  if (op->getBlock() != nullptr &&
+      !getOrCreate<Executable>(getProgramPointBefore(op->getBlock()))->isLive())
     return success();
 
   SmallVector<AbstractSparseLattice *> operandLattices =
       getLatticeElements(op->getOperands());
   SmallVector<const AbstractSparseLattice *> resultLattices =
-      getLatticeElementsFor(op, op->getResults());
+      getLatticeElementsFor(getProgramPointAfter(op), op->getResults());
 
   // Block arguments of region branch operations flow back into the operands
   // of the parent op
@@ -425,7 +436,7 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
                   detail::getBranchSuccessorArgument(
                       successorOperands, operand.getOperandNumber(), block)) {
             meet(getLatticeElement(operand.get()),
-                 *getLatticeElementFor(op, *blockArg));
+                 *getLatticeElementFor(getProgramPointAfter(op), *blockArg));
           }
         }
       }
@@ -467,7 +478,7 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
       for (auto [blockArg, argOpOperand] :
            llvm::zip(block.getArguments(), argOpOperands)) {
         meet(getLatticeElement(argOpOperand.get()),
-             *getLatticeElementFor(op, blockArg));
+             *getLatticeElementFor(getProgramPointAfter(op), blockArg));
         unaccounted.reset(argOpOperand.getOperandNumber());
       }
 
@@ -502,12 +513,13 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
     // Going backwards, the operands of the return are derived from the
     // results of all CallOps calling this CallableOp.
     if (auto callable = dyn_cast<CallableOpInterface>(op->getParentOp())) {
-      const PredecessorState *callsites =
-          getOrCreateFor<PredecessorState>(op, callable);
+      const PredecessorState *callsites = getOrCreateFor<PredecessorState>(
+          getProgramPointAfter(op), getProgramPointAfter(callable));
       if (callsites->allPredecessorsKnown()) {
         for (Operation *call : callsites->getKnownPredecessors()) {
           SmallVector<const AbstractSparseLattice *> callResultLattices =
-              getLatticeElementsFor(op, call->getResults());
+              getLatticeElementsFor(getProgramPointAfter(op),
+                                    call->getResults());
           for (auto [op, result] :
                llvm::zip(operandLattices, callResultLattices))
             meet(op, *result);
@@ -542,7 +554,8 @@ void AbstractSparseBackwardDataFlowAnalysis::visitRegionSuccessors(
     MutableArrayRef<OpOperand> opoperands = operandsToOpOperands(operands);
     ValueRange inputs = successor.getSuccessorInputs();
     for (auto [operand, input] : llvm::zip(opoperands, inputs)) {
-      meet(getLatticeElement(operand.get()), *getLatticeElementFor(op, input));
+      meet(getLatticeElement(operand.get()),
+           *getLatticeElementFor(getProgramPointAfter(op), input));
       unaccounted.reset(operand.getOperandNumber());
     }
   }
@@ -576,7 +589,7 @@ void AbstractSparseBackwardDataFlowAnalysis::
     MutableArrayRef<OpOperand> opOperands = operandsToOpOperands(operands);
     for (auto [opOperand, input] : llvm::zip(opOperands, inputs)) {
       meet(getLatticeElement(opOperand.get()),
-           *getLatticeElementFor(terminator, input));
+           *getLatticeElementFor(getProgramPointAfter(terminator), input));
       unaccounted.reset(const_cast<OpOperand &>(opOperand).getOperandNumber());
     }
   }
@@ -588,8 +601,8 @@ void AbstractSparseBackwardDataFlowAnalysis::
 }
 
 const AbstractSparseLattice *
-AbstractSparseBackwardDataFlowAnalysis::getLatticeElementFor(ProgramPoint point,
-                                                             Value value) {
+AbstractSparseBackwardDataFlowAnalysis::getLatticeElementFor(
+    ProgramPoint *point, Value value) {
   AbstractSparseLattice *state = getLatticeElement(value);
   addDependency(state, point);
   return state;
diff --git a/mlir/lib/Analysis/DataFlowFramework.cpp b/mlir/lib/Analysis/DataFlowFramework.cpp
index a65ddc13143bae3caeb33f41024c57dfb14dcdf2..7e83668c067652c784a1b22a789f39136cf23288 100644
--- a/mlir/lib/Analysis/DataFlowFramework.cpp
+++ b/mlir/lib/Analysis/DataFlowFramework.cpp
@@ -37,7 +37,7 @@ GenericLatticeAnchor::~GenericLatticeAnchor() = default;
 
 AnalysisState::~AnalysisState() = default;
 
-void AnalysisState::addDependency(ProgramPoint dependent,
+void AnalysisState::addDependency(ProgramPoint *dependent,
                                   DataFlowAnalysis *analysis) {
   auto inserted = dependents.insert({dependent, analysis});
   (void)inserted;
@@ -53,7 +53,7 @@ void AnalysisState::addDependency(ProgramPoint dependent,
 void AnalysisState::dump() const { print(llvm::errs()); }
 
 //===----------------------------------------------------------------------===//
-// LatticeAnchor
+// ProgramPoint
 //===----------------------------------------------------------------------===//
 
 void ProgramPoint::print(raw_ostream &os) const {
@@ -61,12 +61,18 @@ void ProgramPoint::print(raw_ostream &os) const {
     os << "<NULL POINT>";
     return;
   }
-  if (Operation *op = llvm::dyn_cast<Operation *>(*this)) {
-    return op->print(os, OpPrintingFlags().skipRegions());
+  if (!isBlockStart()) {
+    os << "<after operation>:";
+    return getPrevOp()->print(os, OpPrintingFlags().skipRegions());
   }
-  return get<Block *>()->print(os);
+  os << "<before operation>:";
+  return getNextOp()->print(os, OpPrintingFlags().skipRegions());
 }
 
+//===----------------------------------------------------------------------===//
+// LatticeAnchor
+//===----------------------------------------------------------------------===//
+
 void LatticeAnchor::print(raw_ostream &os) const {
   if (isNull()) {
     os << "<NULL POINT>";
@@ -78,7 +84,7 @@ void LatticeAnchor::print(raw_ostream &os) const {
     return value.print(os, OpPrintingFlags().skipRegions());
   }
 
-  return get<ProgramPoint>().print(os);
+  return get<ProgramPoint *>()->print(os);
 }
 
 Location LatticeAnchor::getLoc() const {
@@ -87,10 +93,10 @@ Location LatticeAnchor::getLoc() const {
   if (auto value = llvm::dyn_cast<Value>(*this))
     return value.getLoc();
 
-  ProgramPoint pp = get<ProgramPoint>();
-  if (auto *op = llvm::dyn_cast<Operation *>(pp))
-    return op->getLoc();
-  return pp.get<Block *>()->getParent()->getLoc();
+  ProgramPoint *pp = get<ProgramPoint *>();
+  if (!pp->isBlockStart())
+    return pp->getPrevOp()->getLoc();
+  return pp->getBlock()->getParent()->getLoc();
 }
 
 //===----------------------------------------------------------------------===//
@@ -144,7 +150,8 @@ DataFlowAnalysis::~DataFlowAnalysis() = default;
 
 DataFlowAnalysis::DataFlowAnalysis(DataFlowSolver &solver) : solver(solver) {}
 
-void DataFlowAnalysis::addDependency(AnalysisState *state, ProgramPoint point) {
+void DataFlowAnalysis::addDependency(AnalysisState *state,
+                                     ProgramPoint *point) {
   state->addDependency(point, this);
 }
 
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index e83574b734272569953f82c54e874f3c11508897..04e85c2b337dec8ca13c6a91b35c22f5a2126735 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -373,8 +373,9 @@ void mlir::populateGpuToNVVMConversionPatterns(
       NVVM::BlockInClusterIdYOp, NVVM::BlockInClusterIdZOp>>(
       converter, IndexKind::Other, IntrType::Id);
   patterns.add<gpu::index_lowering::OpLowering<
-      gpu::ClusterDimOp, NVVM::ClusterDimXOp, NVVM::ClusterDimYOp,
-      NVVM::ClusterDimZOp>>(converter, IndexKind::Other, IntrType::Dim);
+      gpu::ClusterDimBlocksOp, NVVM::ClusterDimBlocksXOp,
+      NVVM::ClusterDimBlocksYOp, NVVM::ClusterDimBlocksZOp>>(
+      converter, IndexKind::Other, IntrType::Dim);
   patterns.add<gpu::index_lowering::OpLowering<
       gpu::BlockIdOp, NVVM::BlockIdXOp, NVVM::BlockIdYOp, NVVM::BlockIdZOp>>(
       converter, IndexKind::Grid, IntrType::Id);
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index 2b7ac4b529cf0d13f4a487f7308481746352bbf2..39532d34f616eb4b787ab62b9e2cba713e4e6688 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -167,7 +167,9 @@ void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) {
   typeConverter.addConversion(
       [&](MemRefType memRefType) -> std::optional<Type> {
         if (!memRefType.hasStaticShape() ||
-            !memRefType.getLayout().isIdentity() || memRefType.getRank() == 0) {
+            !memRefType.getLayout().isIdentity() || memRefType.getRank() == 0 ||
+            llvm::any_of(memRefType.getShape(),
+                         [](int64_t dim) { return dim == 0; })) {
           return {};
         }
         Type convertedElementType =
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index bf1b59a9913f499daf09397179cacb3589bc333e..aa824a95b1574a06269feffe4313d81585fb7cf2 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -10,6 +10,7 @@
 
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
+#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
@@ -314,3 +315,31 @@ void ConvertOpenMPToLLVMPass::runOnOperation() {
   if (failed(applyPartialConversion(module, target, std::move(patterns))))
     signalPassFailure();
 }
+
+//===----------------------------------------------------------------------===//
+// ConvertToLLVMPatternInterface implementation
+//===----------------------------------------------------------------------===//
+namespace {
+/// Implement the interface to convert OpenMP to LLVM.
+struct OpenMPToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
+  using ConvertToLLVMPatternInterface::ConvertToLLVMPatternInterface;
+  void loadDependentDialects(MLIRContext *context) const final {
+    context->loadDialect<LLVM::LLVMDialect>();
+  }
+
+  /// Hook for derived dialect interface to provide conversion patterns
+  /// and mark dialect legal for the conversion target.
+  void populateConvertToLLVMConversionPatterns(
+      ConversionTarget &target, LLVMTypeConverter &typeConverter,
+      RewritePatternSet &patterns) const final {
+    configureOpenMPToLLVMConversionLegality(target, typeConverter);
+    populateOpenMPToLLVMConversionPatterns(typeConverter, patterns);
+  }
+};
+} // namespace
+
+void mlir::registerConvertOpenMPToLLVMInterface(DialectRegistry &registry) {
+  registry.addExtension(+[](MLIRContext *ctx, omp::OpenMPDialect *dialect) {
+    dialect->addInterfaces<OpenMPToLLVMDialectInterface>();
+  });
+}
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 6d250237e0a33409dbf56c57765a5a6ef3ecba48..aa241b91d758ca3b979f7fc28dd49a4dbfe7b8ac 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -488,9 +488,6 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
             llvm::SmallVector<mlir::Location>(reductionVariables.size(),
                                               parallelOp.getLoc()));
 
-        rewriter.setInsertionPoint(
-            rewriter.create<omp::TerminatorOp>(parallelOp.getLoc()));
-
         // Create loop nest and populate region with contents of scf.parallel.
         auto loopOp = rewriter.create<omp::LoopNestOp>(
             parallelOp.getLoc(), parallelOp.getLowerBound(),
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 50e57682a2dc8daae53c76329812572260823e48..593dbaa6c6545abfd83021bfaef1d768b1e4d1ba 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -43,7 +43,7 @@ Type matchContainerType(Type element, Type container) {
 TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
   if (auto shapedTy = dyn_cast<ShapedType>(type)) {
     Type eTy = shapedTy.getElementType();
-    APInt valueInt(eTy.getIntOrFloatBitWidth(), value);
+    APInt valueInt(eTy.getIntOrFloatBitWidth(), value, /*isSigned=*/true);
     return DenseIntElementsAttr::get(shapedTy, valueInt);
   }
 
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
index c48ded094eacff6ec0220cf4521c9e106241f8df..2d49854347d40676e283e062c6ed220c38fc9096 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateWideInt.cpp
@@ -1081,7 +1081,7 @@ arith::WideIntEmulationConverter::WideIntEmulationConverter(
     if (width == 2 * maxIntWidth)
       return VectorType::get(2, IntegerType::get(ty.getContext(), maxIntWidth));
 
-    return std::nullopt;
+    return nullptr;
   });
 
   // Vector case.
@@ -1102,7 +1102,7 @@ arith::WideIntEmulationConverter::WideIntEmulationConverter(
                              IntegerType::get(ty.getContext(), maxIntWidth));
     }
 
-    return std::nullopt;
+    return nullptr;
   });
 
   // Function case.
@@ -1111,11 +1111,11 @@ arith::WideIntEmulationConverter::WideIntEmulationConverter(
     //   (i2N, i2N) -> i2N --> (vector<2xiN>, vector<2xiN>) -> vector<2xiN>
     SmallVector<Type> inputs;
     if (failed(convertTypes(ty.getInputs(), inputs)))
-      return std::nullopt;
+      return nullptr;
 
     SmallVector<Type> results;
     if (failed(convertTypes(ty.getResults(), results)))
-      return std::nullopt;
+      return nullptr;
 
     return FunctionType::get(ty.getContext(), inputs, results);
   });
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index 8005f9103b235642d50599056c3bb28a2f8c6c23..521138c1f6f4cd8763b7aab76d58099e1409e4f1 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -80,7 +80,7 @@ public:
 
 protected:
   void notifyOperationErased(Operation *op) override {
-    s.eraseState(op);
+    s.eraseState(s.getProgramPointAfter(op));
     for (Value res : op->getResults())
       s.eraseState(res);
   }
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index 98b429de1fd85c401cbc98e3f25dfa8e5fe8c7a3..edd7f607f24f4dd38d587b1cb2dbcc8d0ce61568 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -528,7 +528,7 @@ static ParseResult parseSwitchOpCases(
     int64_t value = 0;
     if (failed(parser.parseInteger(value)))
       return failure();
-    values.push_back(APInt(bitWidth, value));
+    values.push_back(APInt(bitWidth, value, /*isSigned=*/true));
 
     Block *destination;
     SmallVector<OpAsmParser::UnresolvedOperand> operands;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 006d412936a33758e8ebd632e3ceea4b6fc9e343..12ed8cc88ae7b7c4d78b5c26eb911b0156520a4d 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -598,7 +598,7 @@ static ParseResult parseSwitchOpCases(
     int64_t value = 0;
     if (failed(parser.parseInteger(value)))
       return failure();
-    values.push_back(APInt(bitWidth, value));
+    values.push_back(APInt(bitWidth, value, /*isSigned=*/true));
 
     Block *destination;
     SmallVector<OpAsmParser::UnresolvedOperand> operands;
@@ -2089,7 +2089,7 @@ void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
                      Attribute value, uint64_t alignment, unsigned addrSpace,
                      bool dsoLocal, bool threadLocal, SymbolRefAttr comdat,
                      ArrayRef<NamedAttribute> attrs,
-                     DIGlobalVariableExpressionAttr dbgExpr) {
+                     ArrayRef<Attribute> dbgExprs) {
   result.addAttribute(getSymNameAttrName(result.name),
                       builder.getStringAttr(name));
   result.addAttribute(getGlobalTypeAttrName(result.name), TypeAttr::get(type));
@@ -2121,8 +2121,9 @@ void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
                         builder.getI32IntegerAttr(addrSpace));
   result.attributes.append(attrs.begin(), attrs.end());
 
-  if (dbgExpr)
-    result.addAttribute(getDbgExprAttrName(result.name), dbgExpr);
+  if (!dbgExprs.empty())
+    result.addAttribute(getDbgExprsAttrName(result.name),
+                        ArrayAttr::get(builder.getContext(), dbgExprs));
 
   result.addRegion();
 }
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
index 3b9194098fa78341d5df418a35bcf1a2a6842154..bd77965194b27fd538993ee4bf38a8bf2755ba62 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
@@ -15,20 +15,14 @@
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/BuiltinTypeInterfaces.h"
-#include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/raw_ostream.h"
 #include <algorithm>
-#include <optional>
+#include <numeric>
 
 using namespace mlir;
 using namespace mlir::linalg;
@@ -60,112 +54,180 @@ bool linalg::detail::canOpOperandsBeDroppedImpl(
 // CopyOpInterface implementation
 //===----------------------------------------------------------------------===//
 
-bool linalg::isaCopyOpInterface(LinalgOp linalgOp) {
-  // Structural.
-  if (linalgOp.getNumParallelLoops() != linalgOp.getNumLoops())
+bool linalg::isaCopyOpInterface(LinalgOp op) {
+  // Check all loops are parallel and linalgOp is single input and output.
+  if (!op.isAllParallelLoops() || !op.isSingleInputOutput())
     return false;
 
-  // Operands and maps.
-  if (linalgOp.getNumDpsInputs() != 1 || linalgOp.getNumDpsInits() != 1)
-    return false;
-  auto mapRange = linalgOp.getIndexingMapsArray();
+  auto mapRange = op.getIndexingMapsArray();
   if (mapRange.size() != 2 || !mapRange.front().isIdentity() ||
       !mapRange.back().isIdentity()) {
     return false;
   }
   // Region.
-  return llvm::hasSingleElement(linalgOp.getBlock()->getOperations());
+  return llvm::hasSingleElement(op.getBlock()->getOperations());
 }
 
 //===----------------------------------------------------------------------===//
 // FillOpInterface implementation
 //===----------------------------------------------------------------------===//
-std::optional<Value> linalg::isaFillOpInterface(GenericOp genericOp) {
+std::optional<Value> linalg::isaFillOpInterface(GenericOp op) {
   // Structural.
-  if (genericOp.getNumParallelLoops() != genericOp.getNumLoops() ||
-      genericOp.getNumDpsInputs() != 1 || genericOp.getNumDpsInits() != 1)
+  if (!op.isAllParallelLoops() || !op.isSingleInputOutput() ||
+      !op.isSingleYieldOp())
     return std::nullopt;
 
   // Input should be referenced and init should not.
-  if (!genericOp.payloadUsesValueFromOperand(genericOp.getDpsInputOperand(0)) ||
-      genericOp.payloadUsesValueFromOperand(genericOp.getDpsInitOperand(0)))
+  if (!op.payloadUsesValueFromOperand(op.getDpsInputOperand(0)) ||
+      op.payloadUsesValueFromOperand(op.getDpsInitOperand(0)))
     return std::nullopt;
 
-  OpOperand *value = genericOp.getDpsInputOperand(0);
-  if (!genericOp.isScalar(value))
+  OpOperand *value = op.getDpsInputOperand(0);
+  if (!op.isScalar(value))
     return std::nullopt;
+  return value->get();
+}
 
-  Block *body = genericOp.getBody();
-  if (body->getOperations().size() != 1)
+//===----------------------------------------------------------------------===//
+// BroadcastOpInterface implementation
+//===----------------------------------------------------------------------===//
+std::optional<SmallVector<int64_t>>
+linalg::isaBroadcastOpInterface(GenericOp op) {
+  // Structural.
+  if (!op.isAllParallelLoops() || !op.isSingleInputOutput() ||
+      !op.isSingleYieldOp())
     return std::nullopt;
 
-  auto yieldOp = dyn_cast<linalg::YieldOp>(body->back());
-  if (!yieldOp || yieldOp.getNumOperands() != 1 ||
-      yieldOp->getOperand(0) != body->getArgument(0))
+  auto srcTy = op.getDpsInputOperand(0)->get().getType();
+  auto dstTy = op.getDpsInitOperand(0)->get().getType();
+  if (!isa<MemRefType, RankedTensorType>(srcTy) ||
+      !isa<MemRefType, RankedTensorType>(dstTy))
     return std::nullopt;
-  return value->get();
+
+  // Check output is identity map. Broadcast could additionally be
+  // employing permutation of indices and that would be expressible
+  // in linalg.generic but is not expressible for named broadcast op.
+  auto dstMap = op.getIndexingMapsArray()[1];
+  if (!dstMap.isIdentity())
+    return std::nullopt;
+
+  SmallVector<int64_t> position;
+  auto srcMap = op.getIndexingMapsArray()[0];
+
+  if (srcMap.getResults().size() >= dstMap.getResults().size())
+    return std::nullopt;
+
+  // Check input map is monotonically increasing DimIds.
+  for (unsigned i = 0; i < srcMap.getNumResults(); ++i) {
+    auto expr = llvm::dyn_cast<AffineDimExpr>(srcMap.getResults()[i]);
+    if (!expr)
+      return std::nullopt;
+    int64_t pos = expr.getPosition();
+    if (i > 0 && pos <= position[i - 1])
+      return std::nullopt;
+    position.push_back(expr.getPosition());
+  }
+
+  SmallVector<int64_t> broadcastedDims;
+  auto numDims = srcMap.getNumDims();
+  // This is quadratic but number of items is generally small.
+  for (auto dim : llvm::seq<int64_t>(0, numDims)) {
+    if (!llvm::is_contained(position, dim))
+      broadcastedDims.push_back(dim);
+  }
+  return broadcastedDims;
+}
+
+//===----------------------------------------------------------------------===//
+// TranposeOpInterface implementation
+//===----------------------------------------------------------------------===//
+std::optional<SmallVector<int64_t>>
+linalg::isaTransposeOpInterface(GenericOp op) {
+  // To specialize as a transpose op, the genericOp must be
+  // all parallel loops, single input, single output, and its body
+  // should be just a yield op, yielding input as output as is (no compute).
+  if (!op.isAllParallelLoops() || !op.isSingleInputOutput() ||
+      !op.isSingleYieldOp())
+    return std::nullopt;
+
+  auto mapRange = op.getIndexingMapsArray();
+  if (mapRange.size() != 2)
+    return std::nullopt;
+
+  auto mapOfInput = mapRange.front();
+  auto mapOfResult = mapRange.back();
+
+  // linalg.transpose permutes the dimensions of input using this
+  // rule: dim(result, i) = dim(input, permutation[i])
+  if (!mapOfResult.isIdentity() || !mapOfInput.isPermutation())
+    return std::nullopt;
+
+  SmallVector<int64_t> permutation(mapOfInput.getNumDims());
+  for (unsigned i = 0; i < mapOfInput.getNumDims(); ++i) {
+    auto expr = llvm::cast<AffineDimExpr>(mapOfInput.getResults()[i]);
+    permutation[expr.getPosition()] = i;
+  }
+  return permutation;
 }
 
 //===----------------------------------------------------------------------===//
 // Elementwise Single Unary/Binary-OpInterface implementation
 //===----------------------------------------------------------------------===//
-static bool
-isaElemwiseSingleUnaryOrBinaryOpInterface(linalg::GenericOp genericOp,
-                                          unsigned arity) {
+static bool isaElemwiseSingleUnaryOrBinaryOpInterface(linalg::GenericOp op,
+                                                      unsigned arity) {
   // Check all loops are parallel.
-  if (genericOp.getNumParallelLoops() != genericOp.getNumLoops() ||
-      genericOp.getNumLoops() < 1)
+  if (!op.isAllParallelLoops() || op.getNumLoops() < 1)
     return false;
 
   // Check there are arity-inputs, 1-output and all are identity-maps.
-  if (genericOp.getNumDpsInputs() != arity || genericOp.getNumDpsInits() != 1 ||
-      !llvm::all_of(genericOp.getIndexingMapsArray(),
+  if (op.getNumDpsInputs() != arity || op.getNumDpsInits() != 1 ||
+      !llvm::all_of(op.getIndexingMapsArray(),
                     [](AffineMap map) { return map.isIdentity(); }))
     return false;
 
   // Init should not be referenced for elementwise operations.
-  if (genericOp.payloadUsesValueFromOperand(genericOp.getDpsInitOperand(0)))
+  if (op.payloadUsesValueFromOperand(op.getDpsInitOperand(0)))
     return false;
 
   // A linalg.generic could be series of elementwise ops e.g. exp(neg(x)) such
   // as resulting from producer-consumer fusion. Here, we restrict to two ops in
   // the body, where the first is the elementwise single op and the second a
   // yield.
-  Block *body = genericOp.getBody();
+  Block *body = op.getBody();
   if (body->getOperations().size() != 2)
     return false;
 
-  Operation *op = &body->front();
-  if (op->getNumOperands() != arity || op->getNumResults() != 1)
+  Operation *oper = &body->front();
+  if (oper->getNumOperands() != arity || oper->getNumResults() != 1)
     return false;
 
   auto yieldOp = dyn_cast<linalg::YieldOp>(body->back());
   if (!yieldOp || yieldOp.getNumOperands() != 1 ||
-      yieldOp->getOperand(0).getDefiningOp() != op)
+      yieldOp->getOperand(0).getDefiningOp() != oper)
     return false;
   return true;
 }
 
-bool linalg::isaElemwiseSingleUnaryOpInterface(linalg::GenericOp genericOp) {
+bool linalg::isaElemwiseSingleUnaryOpInterface(linalg::GenericOp op) {
   // All basic elemwise checks.
-  if (!isaElemwiseSingleUnaryOrBinaryOpInterface(genericOp, 1))
+  if (!isaElemwiseSingleUnaryOrBinaryOpInterface(op, 1))
     return false;
 
   // Check input is actully used.
-  if (!genericOp.payloadUsesValueFromOperand(genericOp.getDpsInputOperand(0)))
+  if (!op.payloadUsesValueFromOperand(op.getDpsInputOperand(0)))
     return false;
   return true;
 }
 
-bool linalg::isaElemwiseSingleBinaryOpInterface(linalg::GenericOp genericOp) {
-  if (!isaElemwiseSingleUnaryOrBinaryOpInterface(genericOp, 2))
+bool linalg::isaElemwiseSingleBinaryOpInterface(linalg::GenericOp op) {
+  if (!isaElemwiseSingleUnaryOrBinaryOpInterface(op, 2))
     return false;
 
   // Check both inputs are used (elementwise).
-  OpOperand *inputOpOperand0 = genericOp.getDpsInputOperand(0);
-  OpOperand *inputOpOperand1 = genericOp.getDpsInputOperand(1);
-  if (!genericOp.payloadUsesValueFromOperand(inputOpOperand0) ||
-      !genericOp.payloadUsesValueFromOperand(inputOpOperand1))
+  OpOperand *inputOpOperand0 = op.getDpsInputOperand(0);
+  OpOperand *inputOpOperand1 = op.getDpsInputOperand(1);
+  if (!op.payloadUsesValueFromOperand(inputOpOperand0) ||
+      !op.payloadUsesValueFromOperand(inputOpOperand1))
     return false;
   return true;
 }
@@ -1149,6 +1211,7 @@ int64_t LinalgOp::getIndexingMapIndex(OpOperand *opOperand) {
 
 LogicalResult mlir::linalg::detail::verifyStructuredOpInterface(Operation *op) {
   LinalgOp linalgOp = cast<LinalgOp>(op);
+
   // Mixed tensor/buffer operands are not allowed.
   if (!linalgOp.hasPureTensorSemantics() &&
       !linalgOp.hasPureBufferSemantics() && op->getNumOperands() > 0)
@@ -1168,8 +1231,6 @@ LogicalResult mlir::linalg::detail::verifyStructuredOpInterface(Operation *op) {
            << ") to be equal to the number of input/output operands ("
            << linalgOp->getNumOperands() << ")";
 
-  // Set this flag if this op has user defined maps. This is required to guard
-  // the below error condition which assume default indexing maps.
   for (OpOperand &opOperand : linalgOp->getOpOperands()) {
     AffineMap indexingMap = linalgOp.getMatchingIndexingMap(&opOperand);
 
@@ -1186,13 +1247,13 @@ LogicalResult mlir::linalg::detail::verifyStructuredOpInterface(Operation *op) {
              << " dim(s) to match the number of loops";
 
     int64_t rank = linalgOp.getRank(&opOperand);
-
     if (indexingMap.getNumResults() != rank)
       return op->emitOpError("expected operand rank (")
              << rank << ") to match the result rank of indexing_map #"
              << opOperand.getOperandNumber() << " ("
              << indexingMap.getNumResults() << ")";
   }
+
   SmallVector<unsigned> redDims;
   linalgOp.getReductionDims(redDims);
 
@@ -1202,8 +1263,9 @@ LogicalResult mlir::linalg::detail::verifyStructuredOpInterface(Operation *op) {
   // Check if given shapes match to inferred shapes.
   SmallVector<int64_t, 4> endLoopRangeValues = linalgOp.getStaticLoopRanges();
   SmallVector<int64_t, 4> startLoopRangeValues(endLoopRangeValues.size(), 0);
-  // Verify only static cases since we can't get exact dimension sizes and
-  // loop ranges for dynamic cases in this stage.
+
+  // Verify only static cases since we can't get exact dimension sizes and loop
+  // ranges for dynamic cases in this stage.
   if (llvm::none_of(endLoopRangeValues, ShapedType::isDynamic)) {
     for (int64_t &range : endLoopRangeValues)
       range -= 1;
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index c909d13e4314b485f4f529f22e2f400f35ecd114..730c478c2883ef86f824bb7d59f6928640e5c0fe 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -27,7 +27,6 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypeInterfaces.h"
 #include "mlir/IR/Matchers.h"
@@ -38,17 +37,12 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/FormatVariadic.h"
-#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
-#include <cassert>
 #include <optional>
 
 using namespace mlir;
@@ -155,36 +149,15 @@ static void fillStructuredOpRegion(OpBuilder &opBuilder, Region &region,
   // iterator_types is an auto-generated method.
 }
 
-/// Helper to create a typical indexing map for MatmulOp. Returns a list of
-/// AffineMap.
-static SmallVector<AffineMap, 3>
-getDefaultIndexingMapsForMatmul(MLIRContext *context) {
-  AffineExpr d0, d1, d2;
-  SmallVector<AffineMap, 3> indexingMaps;
-  bindDims(context, d0, d1, d2);
-  indexingMaps.push_back(AffineMap::get(3, 0, {d0, d2}, context));
-  indexingMaps.push_back(AffineMap::get(3, 0, {d2, d1}, context));
-  indexingMaps.push_back(AffineMap::get(3, 0, {d0, d1}, context));
-  return indexingMaps;
-}
-
-/// Wrapper to return the typical indexing map array attribute for MatmulOp.
-static SmallVector<Attribute> getDefaultIndexingMapAttr(MLIRContext *context) {
-  return llvm::map_to_vector(
-      getDefaultIndexingMapsForMatmul(context),
-      [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
-}
-
 /// Creates a structured operation given `inputs`, `outputs`, and `attributes`.
 /// The result types are derived automatically if `resultTensorTypes` is none.
 /// The body of the operation is filled using `regionBuilder`. All ods-gen
 /// created structured operations use the method to implement their builders.
-static void buildStructuredOp(
-    OpBuilder &b, OperationState &state,
-    std::optional<TypeRange> resultTensorTypes, ValueRange inputs,
-    ValueRange outputs, ArrayRef<NamedAttribute> attributes,
-    RegionBuilderFn regionBuilder,
-    std::optional<ArrayRef<AffineMap>> indexingMaps = std::nullopt) {
+static void buildStructuredOp(OpBuilder &b, OperationState &state,
+                              std::optional<TypeRange> resultTensorTypes,
+                              ValueRange inputs, ValueRange outputs,
+                              ArrayRef<NamedAttribute> attributes,
+                              RegionBuilderFn regionBuilder) {
   // Derive the result types if needed.
   SmallVector<Type> derivedResultTypes =
       resultTensorTypes.value_or(TypeRange());
@@ -195,20 +168,6 @@ static void buildStructuredOp(
   state.addOperands(inputs);
   state.addOperands(outputs);
   state.addTypes(derivedResultTypes);
-
-  // Initialize indexingMaps, for MatmulOp.
-  SmallVector<Attribute, 3> indexingMapsAttrVal;
-  if (indexingMaps.has_value()) {
-    for (mlir::AffineMap map : *indexingMaps) {
-      // Convert each AffineMap to an AffineMapAttr
-      indexingMapsAttrVal.push_back(AffineMapAttr::get(map));
-    }
-    state.addAttribute("indexing_maps", b.getArrayAttr(indexingMapsAttrVal));
-  } else {
-    indexingMapsAttrVal = getDefaultIndexingMapAttr(b.getContext());
-    state.addAttribute("indexing_maps", b.getArrayAttr(indexingMapsAttrVal));
-  }
-
   state.addAttributes(attributes);
   state.addAttribute(
       "operandSegmentSizes",
@@ -340,48 +299,11 @@ static ParseResult parseNamedStructuredOp(OpAsmParser &parser,
                                           OperationState &result,
                                           unsigned numRegionArgs,
                                           RegionBuilderFn regionBuilder) {
-
-  SmallVector<Attribute, 3> indexingMapsAttr;
-  Attribute mapAttr;
-  if (succeeded(parser.parseOptionalKeyword("indexing_maps"))) {
-    if (parser.parseEqual())
-      return failure();
-
-    if (parser.parseLSquare())
-      return failure();
-
-    do {
-      if (parser.parseAttribute(mapAttr))
-        return failure();
-      if (!isa<AffineMapAttr>(mapAttr)) {
-        return parser.emitError(parser.getCurrentLocation(),
-                                "expected affine map attribute");
-      }
-      indexingMapsAttr.push_back(mapAttr);
-
-      if (parser.parseOptionalComma())
-        break;
-    } while (true);
-
-    if (parser.parseRSquare())
-      return failure();
-  }
-  // Initialize indexingMaps, if not supplied explicitly.
-  if (indexingMapsAttr.empty()) {
-    indexingMapsAttr = getDefaultIndexingMapAttr(result.getContext());
-  }
-  result.addAttribute("indexing_maps",
-                      parser.getBuilder().getArrayAttr(indexingMapsAttr));
-
   // TODO: Enable when ods-gen supports captures.
   SmallVector<Type, 1> inputTypes, outputTypes;
   if (parseCommonStructuredOpParts(parser, result, inputTypes, outputTypes))
     return failure();
 
-  // Parse optional attributes.
-  if (parser.parseOptionalAttrDict(result.attributes))
-    return failure();
-
   // TODO: consider merging results parsing into region parsing.
   // Need to wait for declarative assembly resolution to decide.
   SmallVector<Type, 1> outputTensorsTypes;
@@ -407,9 +329,13 @@ static void printNamedStructuredOpResults(OpAsmPrinter &p,
 }
 
 static void printNamedStructuredOp(OpAsmPrinter &p, Operation *op,
-                                   ValueRange inputs, ValueRange outputs,
-                                   ArrayRef<StringRef> elidedAttrs = {}) {
-  p.printOptionalAttrDict(op->getAttrs(), elidedAttrs);
+                                   ValueRange inputs, ValueRange outputs) {
+  p.printOptionalAttrDict(
+      op->getAttrs(),
+      /*elidedAttrs=*/{"operandSegmentSizes",
+                       // See generated code in
+                       // LinalgNamedStructuredOps.yamlgen.cpp.inc
+                       "linalg.memoized_indexing_maps"});
 
   // Printing is shared with generic ops, except for the region and
   // attributes.
@@ -3456,168 +3382,3 @@ Operation *LinalgDialect::materializeConstant(OpBuilder &builder,
                                               Location loc) {
   return arith::ConstantOp::materialize(builder, value, type, loc);
 }
-
-/// Returns true if the result AffineExpr of the \p explicitMap is same as \p
-/// defaultMap.
-static bool isValidResultDimExprs(AffineMap explictMap, AffineMap defaultMap) {
-  auto explicitRange = explictMap.getResults();
-  auto defaultRange = defaultMap.getResults();
-  DenseSet<AffineExpr> explicitSet(explicitRange.begin(), explicitRange.end());
-  DenseSet<AffineExpr> defaultSet(defaultRange.begin(), defaultRange.end());
-  llvm::set_union(explicitSet, defaultSet);
-  return explicitSet == defaultSet;
-}
-
-/// Returns true if the \p explictMap is broadcasted with respect to the
-/// \p defaultMap.
-static bool isBroadcasted(AffineMap explictMap, AffineMap defaultMap) {
-  return explictMap.getNumResults() < defaultMap.getNumResults();
-}
-
-/// Verifies the broadcast and transpose semantic sepecified by the explicit
-/// indexing map for the MatmulOp \p op for each operand specified by \p
-/// opIndex.
-static LogicalResult verifyExtendedMatmulSemantic(MatmulOp matmulOp,
-                                                  unsigned opIndex) {
-  SmallVector<AffineMap, 3> opIndexingMaps = matmulOp.getIndexingMapsArray();
-  SmallVector<AffineMap, 3> defaultIndexingMaps =
-      matmulOp.getDefaultIndexingMaps();
-
-  auto opIndexingMap = opIndexingMaps[opIndex];
-  auto defaultIndexingMap = defaultIndexingMaps[opIndex];
-  // Check general validity of indexing map results.
-  if (!isValidResultDimExprs(opIndexingMap, defaultIndexingMap))
-    return matmulOp->emitOpError()
-           << "Unexpected dim expression in map result.";
-
-  // Check if the requested broadcast is valid.
-  if (isBroadcasted(opIndexingMap, defaultIndexingMap)) {
-    if (!matmulOp.isValidLhsRhsBroadcastMap(opIndexingMap)) {
-      return matmulOp->emitOpError()
-             << "Invalid broadcast requested, should be (d2).";
-    }
-    return success();
-  }
-  return success();
-}
-
-namespace mlir {
-namespace linalg {
-//===----------------------------------------------------------------------===//
-// MatMulOp
-//===----------------------------------------------------------------------===//
-SmallVector<utils::IteratorType> MatmulOp::getIteratorTypesArray() {
-  return SmallVector<utils::IteratorType>{utils::IteratorType::parallel,
-                                          utils::IteratorType::parallel,
-                                          utils::IteratorType::reduction};
-}
-
-unsigned MatmulOp::getNumRegionArgs() { return 3; }
-
-std::string MatmulOp::getLibraryCallName() {
-  return generateLibraryCallName(getOperation());
-}
-
-bool MatmulOp::hasDynamicIndexingMaps() { return true; }
-
-/// Check if the op has broadcast and/or transpose semantic. Returns true if the
-/// user defined indexing maps are not equal to default map.
-bool MatmulOp::hasUserDefinedMaps() {
-  SmallVector<AffineMap, 3> defaultMaps = getDefaultIndexingMaps();
-  SmallVector<AffineMap, 3> explicitMaps = getIndexingMapsArray();
-  return defaultMaps != explicitMaps;
-}
-
-/// Implements the block region builder for the MatmulOp. This is called by
-/// 'fillStructuredOpRegion'.
-void MatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
-                             ArrayRef<NamedAttribute> attrs) {
-  assert(3 > 0 && block.getNumArguments() == 3 &&
-         "MatmulOp regionBuilder expects 3 (>=0) args");
-  RegionBuilderHelper helper(b, block);
-  SmallVector<Value> yields;
-
-  TypeFn castVal = TypeFn::cast_signed;
-  auto castIter = llvm::find_if(attrs, [&](const NamedAttribute &attr) {
-    return attr.getName() == "cast";
-  });
-  if (castIter != attrs.end()) {
-    if (auto attr = llvm::dyn_cast<TypeFnAttr>(castIter->getValue()))
-      castVal = attr.getValue();
-  }
-
-  Value value1 = helper.buildTypeFn(castVal, block.getArgument(2).getType(),
-                                    block.getArgument(0));
-  Value value2 = helper.buildTypeFn(castVal, block.getArgument(2).getType(),
-                                    block.getArgument(1));
-  Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2);
-  Value value4 =
-      helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3);
-  yields.push_back(value4);
-  helper.yieldOutputs(yields);
-}
-
-/// Returns a list of AffineMap with the typical matmul indexing charactristic.
-SmallVector<AffineMap> MatmulOp::getDefaultIndexingMaps() {
-  MLIRContext *context = this->getContext();
-  return getDefaultIndexingMapsForMatmul(context);
-}
-
-/// Returns true if the given broadcast map \p bcastMap is valid for this op.
-bool MatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap) {
-  assert(bcastMap.getNumResults() == 1 && "Expected single result dim expr.");
-  AffineExpr exp = bcastMap.getResult(0);
-  // Invalid map if the common dimension of matmul not found.
-  return exp.isFunctionOfDim(bcastMap.getNumDims() - 1);
-}
-
-ParseResult MatmulOp::parse(OpAsmParser &parser, OperationState &result) {
-  return parseNamedStructuredOp(parser, result, MatmulOp::getNumRegionArgs(),
-                                MatmulOp::getRegionBuilder());
-}
-void MatmulOp::print(OpAsmPrinter &p) {
-  SmallVector<StringRef, 3> elidedAttrs = {
-      "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
-  printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
-                         elidedAttrs);
-
-  SmallVector<Attribute, 3> indexingMaps =
-      getDefaultIndexingMapAttr(getContext());
-  if (!llvm::equal(getIndexingMaps(), indexingMaps)) {
-    p << " indexing_maps = [";
-    llvm::interleaveComma(getIndexingMaps(), p,
-                          [&](Attribute attr) { p.printAttribute(attr); });
-    p << "]";
-  }
-}
-
-/// Verify the user defined indexing maps.
-LogicalResult MatmulOp::verify() {
-  // Verification of pure matmul is handled by verifyStructuredOpInterface().
-  if (!hasUserDefinedMaps())
-    return success();
-
-  for (unsigned opIndex = 0; opIndex < 2; opIndex++) {
-    if (failed(verifyExtendedMatmulSemantic(*this, opIndex)))
-      return failure();
-  }
-  return success();
-}
-
-LogicalResult MatmulOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
-  return memref::foldMemRefCast(*this);
-}
-void MatmulOp::getEffects(
-    SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
-        &effects) {
-  if (hasPureTensorSemantics())
-    return;
-  getGenericEffectsImpl(effects, cast<LinalgOp>(getOperation()));
-}
-
-Speculation::Speculatability MatmulOp::getSpeculatability() {
-  return getGenericSpeculatabilityImpl(cast<LinalgOp>(getOperation()));
-}
-
-} // namespace linalg
-} // namespace mlir
diff --git a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
index 6db51f4b84d11268274129ea54c7979a59442b9b..2a1445fb92fdc6fa45e6af92425b5e1de2fb7f16 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/InlineScalarOperands.cpp
@@ -78,9 +78,12 @@ struct InlineScalarOperands : public OpRewritePattern<GenericOp> {
       for (auto idx : indices)
         indicesValues.emplace_back(
             rewriter.create<arith::ConstantIndexOp>(loc, idx));
-      Value extractedValue = rewriter.create<tensor::ExtractOp>(
-          loc, opOperand->get(), indicesValues);
-      body->getArgument(idx).replaceAllUsesWith(extractedValue);
+      Value scalarValue = opOperand->get();
+      if (isa<RankedTensorType>(scalarValue.getType())) {
+        scalarValue =
+            rewriter.create<tensor::ExtractOp>(loc, scalarValue, indicesValues);
+      }
+      body->getArgument(idx).replaceAllUsesWith(scalarValue);
       body->eraseArgument(idx);
     }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
index 4d7b748d7200e2b1e781388d576efd5fbd8c6502..dfafffce9d9b607757e05fdc2e3e1ba7e9ce1ba7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
@@ -259,18 +259,43 @@ static FailureOr<LinalgOp> specializeLinalgContractions(RewriterBase &rewriter,
 //===----------------------------------------------------------------------===//
 FailureOr<LinalgOp> mlir::linalg::specializeGenericOp(RewriterBase &rewriter,
                                                       GenericOp genericOp) {
+  // Copy
   if (isaCopyOpInterface(genericOp)) {
     LinalgOp namedOp = rewriter.replaceOpWithNewOp<CopyOp>(
         genericOp, genericOp.getDpsInputs()[0], genericOp.getDpsInits()[0]);
     return namedOp;
   }
 
+  // Fill
   if (isaFillOpInterface(genericOp)) {
     LinalgOp namedOp = rewriter.replaceOpWithNewOp<FillOp>(
         genericOp, genericOp.getDpsInputs()[0], genericOp.getDpsInits()[0]);
     return namedOp;
   }
 
+  // Broadcast
+  std::optional<SmallVector<int64_t>> equivalentToBroadcast =
+      isaBroadcastOpInterface(genericOp);
+  if (equivalentToBroadcast) {
+    auto dims = *equivalentToBroadcast;
+    LinalgOp namedOp = rewriter.replaceOpWithNewOp<BroadcastOp>(
+        genericOp, genericOp.getDpsInputs()[0], genericOp.getDpsInits()[0],
+        dims);
+    return namedOp;
+  }
+
+  // Transpose
+  std::optional<SmallVector<int64_t>> equivalentToTranspose =
+      isaTransposeOpInterface(genericOp);
+  if (equivalentToTranspose) {
+    auto permutation = *equivalentToTranspose;
+    LinalgOp namedOp = rewriter.replaceOpWithNewOp<TransposeOp>(
+        genericOp, genericOp.getDpsInputs()[0], genericOp.getDpsInits()[0],
+        permutation);
+    return namedOp;
+  }
+
+  // Elementwise Unary
   if (isaElemwiseSingleUnaryOpInterface(genericOp)) {
     Operation *op = &genericOp.getBody()->front();
     if (isa<math::ExpOp>(op)) {
@@ -279,6 +304,7 @@ FailureOr<LinalgOp> mlir::linalg::specializeGenericOp(RewriterBase &rewriter,
     }
   }
 
+  // Elementwise Binary
   if (isaElemwiseSingleBinaryOpInterface(genericOp)) {
     bool swap = areBinOpsSwapped(genericOp);
     Operation *op = &genericOp.getBody()->front();
@@ -300,6 +326,7 @@ FailureOr<LinalgOp> mlir::linalg::specializeGenericOp(RewriterBase &rewriter,
     }
   }
 
+  // Contraction - e.g. matmul
   if (isaContractionOpInterface(genericOp)) {
     return specializeLinalgContractions(rewriter, genericOp);
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index 6b934f7e8157d472ac5731dc4546227aacc0dd12..aa0052ce47fa7b29385b55ce886215b70ee9cd2a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -31,13 +31,6 @@ using namespace mlir::linalg;
 FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
                                                      linalg::MatmulOp matmulOp,
                                                      bool transposeLHS) {
-  // Check to not let go the matmul with extended semantic, through this
-  // transform.
-  if (matmulOp.hasUserDefinedMaps()) {
-    return rewriter.notifyMatchFailure(
-        matmulOp, "only matmul ops with non-extended semantics are supported");
-  }
-
   if (!bufferization::hasTensorSemantics(matmulOp))
     return rewriter.notifyMatchFailure(
         matmulOp, "only matmul ops with tensors are supported");
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index e3f010d9cfb20bef24159c2bc700d66d07247d25..09c6b2683b438805d0196597fdca1d7028875e8e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -2071,11 +2071,6 @@ vectorizeScalableVectorPrecondition(Operation *op,
       return failure();
   }
 
-  // Check to not let go the matmul with extended semantic, through this
-  // transform.
-  if (linalgOp.hasUserDefinedMaps())
-    return failure();
-
   // Cond 4: Only the following ops are supported in the
   // presence of scalable vectors
   return success(isElementwise(linalgOp) || isa<linalg::MatmulOp>(op) ||
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index 80edf4a32c6df8de8e66ba04ef493f3a2a1c23f7..79f77822116fd712a70cc7ca3bad726db26e28b5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -729,6 +729,7 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
 
   auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
                        ValueRange args) -> scf::ValueVector {
+    auto context = builder.getContext();
     Value tileHIter = ivs[0];
     Value tileWIter = ivs[1];
     Value NIter = ivs[2];
@@ -740,29 +741,41 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                             FIter, 2, 3, /*loopNorFIdx=*/4,
                             /*loopCorFIdx=*/5, /*heightIdx=*/0, /*widthIdx=*/1);
 
-    TransformMapKeyTy key = {m, r};
-    int64_t retRows = 1;
-    int64_t retCols = 1;
-    int64_t leftScalarFactor = 1;
-    int64_t rightScalarFactor = 1;
+    const TransformMapKeyTy key = {m, r};
+    const TransformMatrix &AMatrix = AMatrices.at(key);
+    const TransformMatrix &ATMatrix = ATMatrices.at(key);
+    int64_t scalarFactor = (rightTransform ? AMatrix.scalarFactor : 1) *
+                           (leftTransform ? ATMatrix.scalarFactor : 1);
+    int64_t retCols = rightTransform ? AMatrix.cols : 1;
+    int64_t retRows = leftTransform ? ATMatrix.rows : 1;
+
     Value matmulRetValue = extractValue;
     Value zero = builder.create<arith::ConstantOp>(
         loc, rewriter.getZeroAttr(elementType));
-    if (leftTransform) {
-      // Get constant transform matrix AT.
-      auto it = ATMatrices.find(key);
-      if (it == ATMatrices.end())
-        return {};
-      const TransformMatrix &ATMatrix = it->second;
 
-      leftScalarFactor = ATMatrix.scalarFactor;
-      retRows = ATMatrix.rows;
+    auto affineMap =
+        AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
+    Value heightOffset =
+        builder.create<affine::AffineApplyOp>(loc, affineMap, tileHIter);
+    Value widthOffset =
+        builder.create<affine::AffineApplyOp>(loc, affineMap, tileWIter);
+
+    Value outInitVal =
+        extract2DDataFrom4D(builder, loc, args[0], NIter, FIter, heightOffset,
+                            widthOffset, retRows, retCols,
+                            /*loopNorFIdx=*/0,
+                            /*loopCorFIdx=*/3, /*heightIdx=*/1,
+                            /*widthIdx=*/2);
+    if (leftTransform) {
       auto matmulType = RankedTensorType::get({retRows, valueW}, elementType);
-      auto empty =
-          builder
-              .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
-              .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      Value init = outInitVal;
+      if (rightTransform || scalarFactor != 1) {
+        auto empty = builder
+                         .create<tensor::EmptyOp>(loc, matmulType.getShape(),
+                                                  elementType)
+                         .getResult();
+        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      }
 
       Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
       // Multiply AT x m.
@@ -772,21 +785,16 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
     }
 
     if (rightTransform) {
-      // Get constant transform matrix T.
-      auto it = AMatrices.find(key);
-      if (it == AMatrices.end())
-        return {};
-      const TransformMatrix &AMatrix = it->second;
-
-      rightScalarFactor = AMatrix.scalarFactor;
       auto matmulType =
           RankedTensorType::get({retRows, AMatrix.cols}, elementType);
-      retCols = AMatrix.cols;
-      auto empty =
-          builder
-              .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
-              .getResult();
-      auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      Value init = outInitVal;
+      if (scalarFactor != 1) {
+        auto empty = builder
+                         .create<tensor::EmptyOp>(loc, matmulType.getShape(),
+                                                  elementType)
+                         .getResult();
+        init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
+      }
 
       Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
       // Multiply y = (AT x m) x A.
@@ -795,48 +803,36 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
       matmulRetValue = matmulOp.getResult(0);
     }
 
-    if (leftScalarFactor * rightScalarFactor != 1) {
-      // Multiply scalar factor.
-      Value scalarFactor = builder.create<arith::ConstantOp>(
-          loc,
-          FloatAttr::get(elementType, leftScalarFactor * rightScalarFactor));
+    if (scalarFactor != 1) {
+      // Multiply by scalar factor and add outInitVal.
+      Value scalarFactorValue = builder.create<arith::ConstantOp>(
+          loc, FloatAttr::get(elementType, scalarFactor));
       auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
-      auto init = builder.create<tensor::EmptyOp>(loc, matmulType.getShape(),
-                                                  elementType);
-
       auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
       SmallVector<AffineMap> affineMaps = {
-          AffineMap::get(2, 0, init.getContext()), identityAffineMap};
-      auto broadcastedScalar =
+          AffineMap::get(2, 0, context), identityAffineMap, identityAffineMap};
+
+      matmulRetValue =
           rewriter
               .create<linalg::GenericOp>(
-                  loc, matmulType, ValueRange{scalarFactor}, ValueRange{init},
-                  affineMaps,
+                  loc, matmulType,
+                  ValueRange{scalarFactorValue, matmulRetValue},
+                  ValueRange{outInitVal}, affineMaps,
                   llvm::ArrayRef<utils::IteratorType>{
                       utils::IteratorType::parallel,
                       utils::IteratorType::parallel},
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
+                    auto mulf = nestedBuilder.create<arith::MulFOp>(
+                        nestedLoc, args[0], args[1]);
+                    auto addf = nestedBuilder.create<arith::AddFOp>(
+                        nestedLoc, mulf.getResult(), args[2]);
+                    nestedBuilder.create<linalg::YieldOp>(nestedLoc,
+                                                          addf.getResult());
                   })
               .getResult(0);
-
-      matmulRetValue = builder
-                           .create<linalg::MulOp>(
-                               loc, matmulType,
-                               ValueRange{broadcastedScalar, matmulRetValue},
-                               ValueRange{init})
-                           .getResult(0);
     }
 
-    auto context = builder.getContext();
-    auto affineMap =
-        AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
-    Value heightOffset =
-        builder.create<affine::AffineApplyOp>(loc, affineMap, tileHIter);
-    Value widthOffset =
-        builder.create<affine::AffineApplyOp>(loc, affineMap, tileWIter);
-
     // Insert (H, W) to (N, H, W, F).
     Value combinedVal =
         insert2DDataTo4D(builder, loc, matmulRetValue, args[0], NIter, FIter,
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index 3c508ed6e324b2bb914b9aec169f794f00c4748b..0c2275bbc4b224accc9b9f916f9757ed639761ae 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -821,12 +821,6 @@ DiagnosedSilenceableFailure transform::RewriteMatmulAsMmaSyncOp::applyToOne(
   bool fail = true;
   // TODO: more robust detection of matmulOp, with transposes etc.
   if (isa_and_nonnull<linalg::MatmulOp>(linalgOp.getOperation())) {
-    // Check to not let go the matmul with extended semantic, through this
-    // transform.
-    if (linalgOp.hasUserDefinedMaps()) {
-      return emitSilenceableError()
-             << "only matmul ops with non-extended semantics are supported";
-    }
     Location loc = linalgOp.getLoc();
     // TODO: more robust computation of laneId, for now assume a single warp.
     Value laneId = rewriter.create<gpu::ThreadIdOp>(
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index d516c8d9e0be6c8a26dff45e5af22c54fe0001cc..c6c6edb8f999fc724b183184aa0dae33a185cbb5 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
@@ -83,6 +84,8 @@ void OpenMPDialect::initialize() {
 #include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
       >();
 
+  declarePromisedInterface<ConvertToLLVMPatternInterface, OpenMPDialect>();
+
   MemRefType::attachInterface<MemRefPointerLikeModel>(*getContext());
   LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(
       *getContext());
@@ -1401,7 +1404,7 @@ static ParseResult parseMembersIndex(OpAsmParser &parser,
     if (parser.parseInteger(value))
       return failure();
     shapeTmp++;
-    values.push_back(APInt(32, value));
+    values.push_back(APInt(32, value, /*isSigned=*/true));
     return success();
   };
 
@@ -1921,27 +1924,23 @@ LogicalResult SingleOp::verify() {
 
 LogicalResult LoopWrapperInterface::verifyImpl() {
   Operation *op = this->getOperation();
+  if (!op->hasTrait<OpTrait::NoTerminator>() ||
+      !op->hasTrait<OpTrait::SingleBlock>())
+    return emitOpError() << "loop wrapper must also have the `NoTerminator` "
+                            "and `SingleBlock` traits";
+
   if (op->getNumRegions() != 1)
-    return emitOpError() << "loop wrapper contains multiple regions";
+    return emitOpError() << "loop wrapper does not contain exactly one region";
 
   Region &region = op->getRegion(0);
-  if (!region.hasOneBlock())
-    return emitOpError() << "loop wrapper contains multiple blocks";
-
-  if (::llvm::range_size(region.getOps()) != 2)
+  if (range_size(region.getOps()) != 1)
     return emitOpError()
-           << "loop wrapper does not contain exactly two nested ops";
+           << "loop wrapper does not contain exactly one nested op";
 
   Operation &firstOp = *region.op_begin();
-  Operation &secondOp = *(std::next(region.op_begin()));
-
-  if (!secondOp.hasTrait<OpTrait::IsTerminator>())
-    return emitOpError()
-           << "second nested op in loop wrapper is not a terminator";
-
-  if (!::llvm::isa<LoopNestOp, LoopWrapperInterface>(firstOp))
-    return emitOpError() << "first nested op in loop wrapper is not "
-                            "another loop wrapper or `omp.loop_nest`";
+  if (!isa<LoopNestOp, LoopWrapperInterface>(firstOp))
+    return emitOpError() << "op nested in loop wrapper is not another loop "
+                            "wrapper or `omp.loop_nest`";
 
   return success();
 }
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopRangeFolding.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopRangeFolding.cpp
index 9ca441f643183594f43193b6d68ad0f03e90a208..0c2a71eb25fe5d8b6e0f3dcf0dc97e63f7721759 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopRangeFolding.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopRangeFolding.cpp
@@ -72,9 +72,11 @@ void ForLoopRangeFolding::runOnOperation() {
         op.setUpperBound(ubFold->getResult(0));
 
       } else if (isa<arith::MulIOp>(user)) {
+        Operation *lbFold = b.clone(*user, lbMap);
         Operation *ubFold = b.clone(*user, ubMap);
         Operation *stepFold = b.clone(*user, stepMap);
 
+        op.setLowerBound(lbFold->getResult(0));
         op.setUpperBound(ubFold->getResult(0));
         op.setStep(stepFold->getResult(0));
       }
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index b21bc1a93036c48f06fce7618855563c99159c65..7b1b1f383e634349cd528fce54959e64acd4e50a 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -1310,7 +1310,7 @@ static LogicalResult verifyPackUnPack(Operation *op, bool requiresStaticShape,
     // The coordinates should be in shape of <? x rank>
     unsigned expCOORank = stt.getLvlRank() - cooStartLvl;
     if (cooTp.getRank() != 2 || expCOORank != cooTp.getShape().back()) {
-      op->emitError("input/output trailing COO level-ranks don't match");
+      return op->emitError("input/output trailing COO level-ranks don't match");
     }
   }
 
@@ -1350,7 +1350,7 @@ static LogicalResult verifyPackUnPack(Operation *op, bool requiresStaticShape,
 }
 
 LogicalResult AssembleOp::verify() {
-  const auto valuesTp = getRankedTensorType(getValues());
+  RankedTensorType valuesTp = getValues().getType();
   const auto lvlsTp = getLevels().getTypes();
   const auto resTp = getSparseTensorType(getResult());
   return verifyPackUnPack(*this, true, resTp, valuesTp, lvlsTp);
@@ -1364,34 +1364,31 @@ LogicalResult DisassembleOp::verify() {
     if (ot.getType() != rt.getType())
       return emitError("output levels and return levels type mismatch");
 
-  const auto valuesTp = getRankedTensorType(getRetValues());
+  RankedTensorType valuesTp = getRetValues().getType();
   const auto lvlsTp = getRetLevels().getTypes();
   const auto srcTp = getSparseTensorType(getTensor());
   return verifyPackUnPack(*this, false, srcTp, valuesTp, lvlsTp);
 }
 
 LogicalResult ConvertOp::verify() {
-  if (auto tp1 = llvm::dyn_cast<RankedTensorType>(getSource().getType())) {
-    if (auto tp2 = llvm::dyn_cast<RankedTensorType>(getDest().getType())) {
-      if (tp1.getRank() != tp2.getRank())
-        return emitError("unexpected conversion mismatch in rank");
-      auto dstEnc =
-          llvm::dyn_cast_or_null<SparseTensorEncodingAttr>(tp2.getEncoding());
-      if (dstEnc && dstEnc.isSlice())
-        return emitError("cannot convert to a sparse tensor slice");
-
-      auto shape1 = tp1.getShape();
-      auto shape2 = tp2.getShape();
-      // Accept size matches between the source and the destination type
-      // (e.g. 10 vs. 10, 10 vs. ?, or ? vs. ?), but reject direct mismatches or
-      // matches that would need a runtime assert (e.g. 10 vs. 20 or ? vs. 10).
-      for (Dimension d = 0, dimRank = tp1.getRank(); d < dimRank; d++)
-        if (shape1[d] != shape2[d] && shape2[d] != ShapedType::kDynamic)
-          return emitError("unexpected conversion mismatch in dimension ") << d;
-      return success();
-    }
-  }
-  return emitError("unexpected type in convert");
+  RankedTensorType tp1 = getSource().getType();
+  RankedTensorType tp2 = getDest().getType();
+  if (tp1.getRank() != tp2.getRank())
+    return emitError("unexpected conversion mismatch in rank");
+  auto dstEnc =
+      llvm::dyn_cast_or_null<SparseTensorEncodingAttr>(tp2.getEncoding());
+  if (dstEnc && dstEnc.isSlice())
+    return emitError("cannot convert to a sparse tensor slice");
+
+  auto shape1 = tp1.getShape();
+  auto shape2 = tp2.getShape();
+  // Accept size matches between the source and the destination type
+  // (e.g. 10 vs. 10, 10 vs. ?, or ? vs. ?), but reject direct mismatches or
+  // matches that would need a runtime assert (e.g. 10 vs. 20 or ? vs. 10).
+  for (Dimension d = 0, dimRank = tp1.getRank(); d < dimRank; d++)
+    if (shape1[d] != shape2[d] && shape2[d] != ShapedType::kDynamic)
+      return emitError("unexpected conversion mismatch in dimension ") << d;
+  return success();
 }
 
 OpFoldResult ConvertOp::fold(FoldAdaptor adaptor) {
@@ -1495,7 +1492,8 @@ LogicalResult LvlOp::verify() {
   if (std::optional<uint64_t> lvl = getConstantLvlIndex()) {
     auto stt = getSparseTensorType(getSource());
     if (static_cast<uint64_t>(lvl.value()) >= stt.getLvlRank())
-      emitError("Level index exceeds the rank of the input sparse tensor");
+      return emitError(
+          "Level index exceeds the rank of the input sparse tensor");
   }
   return success();
 }
@@ -1697,14 +1695,14 @@ LogicalResult ToValuesOp::inferReturnTypes(MLIRContext *ctx,
 }
 
 LogicalResult ToSliceOffsetOp::verify() {
-  auto rank = getRankedTensorType(getSlice()).getRank();
+  auto rank = getSlice().getType().getRank();
   if (rank <= getDim().getSExtValue() || getDim().getSExtValue() < 0)
     return emitError("requested dimension out of bound");
   return success();
 }
 
 LogicalResult ToSliceStrideOp::verify() {
-  auto rank = getRankedTensorType(getSlice()).getRank();
+  auto rank = getSlice().getType().getRank();
   if (rank <= getDim().getSExtValue() || getDim().getSExtValue() < 0)
     return emitError("requested dimension out of bound");
   return success();
@@ -1986,15 +1984,16 @@ LogicalResult ForeachOp::verify() {
   const auto iTp = IndexType::get(getContext());
   for (Dimension d = 0; d < dimRank; d++)
     if (args[d].getType() != iTp)
-      emitError(
+      return emitError(
           llvm::formatv("Expecting Index type for argument at index {0}", d));
 
   const auto elemTp = t.getElementType();
   const auto valueTp = args[dimRank].getType();
   if (elemTp != valueTp)
-    emitError(llvm::formatv("Unmatched element type between input tensor and "
-                            "block argument, expected:{0}, got: {1}",
-                            elemTp, valueTp));
+    return emitError(
+        llvm::formatv("Unmatched element type between input tensor and "
+                      "block argument, expected:{0}, got: {1}",
+                      elemTp, valueTp));
   return success();
 }
 
@@ -2011,15 +2010,15 @@ LogicalResult ReorderCOOOp::verify() {
   SparseTensorType dstStt = getSparseTensorType(getResultCoo());
 
   if (!srcStt.isCOOType() || !dstStt.isCOOType())
-    emitError("Expected COO sparse tensors only");
+    return emitError("Expected COO sparse tensors only");
 
   if (!srcStt.hasSameDimToLvl(dstStt))
-    emitError("Unmatched dim2lvl map between input and result COO");
+    return emitError("Unmatched dim2lvl map between input and result COO");
 
   if (srcStt.getPosType() != dstStt.getPosType() ||
       srcStt.getCrdType() != dstStt.getCrdType() ||
       srcStt.getElementType() != dstStt.getElementType())
-    emitError("Unmatched storage format between input and result COO");
+    return emitError("Unmatched storage format between input and result COO");
 
   return success();
 }
@@ -2044,10 +2043,11 @@ LogicalResult SortOp::verify() {
   AffineMap xPerm = getPermMap();
   uint64_t nx = xPerm.getNumDims();
   if (nx < 1)
-    emitError(llvm::formatv("Expected rank(perm_map) > 1, got {0}", nx));
+    return emitError(llvm::formatv("Expected rank(perm_map) > 1, got {0}", nx));
 
   if (!xPerm.isPermutation())
-    emitError(llvm::formatv("Expected a permutation map, got {0}", xPerm));
+    return emitError(
+        llvm::formatv("Expected a permutation map, got {0}", xPerm));
 
   // We can't check the size of the buffers when n or buffer dimensions aren't
   // compile-time constants.
@@ -2056,19 +2056,24 @@ LogicalResult SortOp::verify() {
     return success();
 
   // Verify dimensions.
-  const auto checkDim = [&](Value v, Size minSize, const char *message) {
+  const auto checkDim = [&](Value v, Size minSize,
+                            const char *message) -> LogicalResult {
     const Size sh = getMemRefType(v).getShape()[0];
     if (!ShapedType::isDynamic(sh) && sh < minSize)
-      emitError(llvm::formatv("{0} got {1} < {2}", message, sh, minSize));
+      return emitError(
+          llvm::formatv("{0} got {1} < {2}", message, sh, minSize));
+    return success();
   };
   uint64_t n = cn.value();
   uint64_t ny = 0;
   if (auto nyAttr = getNyAttr())
     ny = nyAttr.getInt();
-  checkDim(getXy(), n * (nx + ny),
-           "Expected dimension(xy) >= n * (rank(perm_map) + ny)");
+  if (failed(checkDim(getXy(), n * (nx + ny),
+                      "Expected dimension(xy) >= n * (rank(perm_map) + ny)")))
+    return failure();
   for (Value opnd : getYs())
-    checkDim(opnd, n, "Expected dimension(y) >= n");
+    if (failed(checkDim(opnd, n, "Expected dimension(y) >= n")))
+      return failure();
 
   return success();
 }
@@ -2101,8 +2106,8 @@ static ParseResult parseLevelRange(AsmParser &parser, Level &lvlLo,
   }
 
   if (lvlHi <= lvlLo)
-    parser.emitError(parser.getNameLoc(),
-                     "expect larger level upper bound than lower bound");
+    return parser.emitError(parser.getNameLoc(),
+                            "expect larger level upper bound than lower bound");
 
   return success();
 }
diff --git a/mlir/lib/IR/AffineExpr.cpp b/mlir/lib/IR/AffineExpr.cpp
index 0b078966aeb85bfb62867d854b2aebbdec697627..2291d64c50a56045f106ff354dfd4ae83a2e6e7c 100644
--- a/mlir/lib/IR/AffineExpr.cpp
+++ b/mlir/lib/IR/AffineExpr.cpp
@@ -356,8 +356,9 @@ unsigned AffineDimExpr::getPosition() const {
 ///`exprKind` is floordiv and `expr` is also a binary expression of a floordiv
 /// operation, then the commutative property can be used otherwise, the floordiv
 /// operation is not divisible. The same argument holds for ceildiv operation.
-static bool isDivisibleBySymbol(AffineExpr expr, unsigned symbolPos,
-                                AffineExprKind opKind) {
+static bool canSimplifyDivisionBySymbol(AffineExpr expr, unsigned symbolPos,
+                                        AffineExprKind opKind,
+                                        bool fromMul = false) {
   // The argument `opKind` can either be Modulo, Floordiv or Ceildiv only.
   assert((opKind == AffineExprKind::Mod || opKind == AffineExprKind::FloorDiv ||
           opKind == AffineExprKind::CeilDiv) &&
@@ -372,8 +373,9 @@ static bool isDivisibleBySymbol(AffineExpr expr, unsigned symbolPos,
   // Checks divisibility by the given symbol for both operands.
   case AffineExprKind::Add: {
     AffineBinaryOpExpr binaryExpr = cast<AffineBinaryOpExpr>(expr);
-    return isDivisibleBySymbol(binaryExpr.getLHS(), symbolPos, opKind) &&
-           isDivisibleBySymbol(binaryExpr.getRHS(), symbolPos, opKind);
+    return canSimplifyDivisionBySymbol(binaryExpr.getLHS(), symbolPos,
+                                       opKind) &&
+           canSimplifyDivisionBySymbol(binaryExpr.getRHS(), symbolPos, opKind);
   }
   // Checks divisibility by the given symbol for both operands. Consider the
   // expression `(((s1*s0) floordiv w) mod ((s1 * s2) floordiv p)) floordiv s1`,
@@ -382,16 +384,18 @@ static bool isDivisibleBySymbol(AffineExpr expr, unsigned symbolPos,
   // `AffineExprKind::Mod` for this reason.
   case AffineExprKind::Mod: {
     AffineBinaryOpExpr binaryExpr = cast<AffineBinaryOpExpr>(expr);
-    return isDivisibleBySymbol(binaryExpr.getLHS(), symbolPos,
-                               AffineExprKind::Mod) &&
-           isDivisibleBySymbol(binaryExpr.getRHS(), symbolPos,
-                               AffineExprKind::Mod);
+    return canSimplifyDivisionBySymbol(binaryExpr.getLHS(), symbolPos,
+                                       AffineExprKind::Mod) &&
+           canSimplifyDivisionBySymbol(binaryExpr.getRHS(), symbolPos,
+                                       AffineExprKind::Mod);
   }
   // Checks if any of the operand divisible by the given symbol.
   case AffineExprKind::Mul: {
     AffineBinaryOpExpr binaryExpr = cast<AffineBinaryOpExpr>(expr);
-    return isDivisibleBySymbol(binaryExpr.getLHS(), symbolPos, opKind) ||
-           isDivisibleBySymbol(binaryExpr.getRHS(), symbolPos, opKind);
+    return canSimplifyDivisionBySymbol(binaryExpr.getLHS(), symbolPos, opKind,
+                                       true) ||
+           canSimplifyDivisionBySymbol(binaryExpr.getRHS(), symbolPos, opKind,
+                                       true);
   }
   // Floordiv and ceildiv are divisible by the given symbol when the first
   // operand is divisible, and the affine expression kind of the argument expr
@@ -399,14 +403,19 @@ static bool isDivisibleBySymbol(AffineExpr expr, unsigned symbolPos,
   // property of floordiv and ceildiv operations and are as follow:
   // (exp1 floordiv exp2) floordiv exp3 = (exp1 floordiv exp3) floordiv exp2
   // (exp1 ceildiv exp2) ceildiv exp3 = (exp1 ceildiv exp3) ceildiv expr2
-  // It will fail if operations are not same. For example:
-  // (exps1 ceildiv exp2) floordiv exp3 can not be simplified.
+  // It will fail 1.if operations are not same. For example:
+  // (exps1 ceildiv exp2) floordiv exp3 can not be simplified. 2.if there is a
+  // multiplication operation in the expression. For example:
+  //  (exps1 ceildiv exp2) mul exp3 ceildiv exp4 can not be simplified.
   case AffineExprKind::FloorDiv:
   case AffineExprKind::CeilDiv: {
     AffineBinaryOpExpr binaryExpr = cast<AffineBinaryOpExpr>(expr);
     if (opKind != expr.getKind())
       return false;
-    return isDivisibleBySymbol(binaryExpr.getLHS(), symbolPos, expr.getKind());
+    if (fromMul)
+      return false;
+    return canSimplifyDivisionBySymbol(binaryExpr.getLHS(), symbolPos,
+                                       expr.getKind());
   }
   }
   llvm_unreachable("Unknown AffineExpr");
@@ -448,7 +457,7 @@ static AffineExpr symbolicDivide(AffineExpr expr, unsigned symbolPos,
   // Dividing any of the operand by the given symbol.
   case AffineExprKind::Mul: {
     AffineBinaryOpExpr binaryExpr = cast<AffineBinaryOpExpr>(expr);
-    if (!isDivisibleBySymbol(binaryExpr.getLHS(), symbolPos, opKind))
+    if (!canSimplifyDivisionBySymbol(binaryExpr.getLHS(), symbolPos, opKind))
       return binaryExpr.getLHS() *
              symbolicDivide(binaryExpr.getRHS(), symbolPos, opKind);
     return symbolicDivide(binaryExpr.getLHS(), symbolPos, opKind) *
@@ -583,7 +592,8 @@ static AffineExpr simplifySemiAffine(AffineExpr expr, unsigned numDims,
     if (!symbolExpr)
       return getAffineBinaryOpExpr(expr.getKind(), sLHS, sRHS);
     unsigned symbolPos = symbolExpr.getPosition();
-    if (!isDivisibleBySymbol(binaryExpr.getLHS(), symbolPos, expr.getKind()))
+    if (!canSimplifyDivisionBySymbol(binaryExpr.getLHS(), symbolPos,
+                                     expr.getKind()))
       return getAffineBinaryOpExpr(expr.getKind(), sLHS, sRHS);
     if (expr.getKind() == AffineExprKind::Mod)
       return getAffineConstantExpr(0, expr.getContext());
diff --git a/mlir/lib/IR/Builders.cpp b/mlir/lib/IR/Builders.cpp
index a9bc3c0ef65a23cc609173e63a23e240dd02513b..5359432a04f3e572db76709e4cf16b467536dda6 100644
--- a/mlir/lib/IR/Builders.cpp
+++ b/mlir/lib/IR/Builders.cpp
@@ -238,7 +238,10 @@ DenseIntElementsAttr Builder::getIndexTensorAttr(ArrayRef<int64_t> values) {
 }
 
 IntegerAttr Builder::getI32IntegerAttr(int32_t value) {
-  return IntegerAttr::get(getIntegerType(32), APInt(32, value));
+  // The APInt always uses isSigned=true here because we accept the value
+  // as int32_t.
+  return IntegerAttr::get(getIntegerType(32),
+                          APInt(32, value, /*isSigned=*/true));
 }
 
 IntegerAttr Builder::getSI32IntegerAttr(int32_t value) {
@@ -256,14 +259,19 @@ IntegerAttr Builder::getI16IntegerAttr(int16_t value) {
 }
 
 IntegerAttr Builder::getI8IntegerAttr(int8_t value) {
-  return IntegerAttr::get(getIntegerType(8), APInt(8, value));
+  // The APInt always uses isSigned=true here because we accept the value
+  // as int8_t.
+  return IntegerAttr::get(getIntegerType(8),
+                          APInt(8, value, /*isSigned=*/true));
 }
 
 IntegerAttr Builder::getIntegerAttr(Type type, int64_t value) {
   if (type.isIndex())
     return IntegerAttr::get(type, APInt(64, value));
-  return IntegerAttr::get(
-      type, APInt(type.getIntOrFloatBitWidth(), value, type.isSignedInteger()));
+  // TODO: Avoid implicit trunc?
+  return IntegerAttr::get(type, APInt(type.getIntOrFloatBitWidth(), value,
+                                      type.isSignedInteger(),
+                                      /*implicitTrunc=*/true));
 }
 
 IntegerAttr Builder::getIntegerAttr(Type type, const APInt &value) {
diff --git a/mlir/lib/TableGen/AttrOrTypeDef.cpp b/mlir/lib/TableGen/AttrOrTypeDef.cpp
index 9b9d9fd2317d9922e5bc08bd139e6f3225e664c1..e72ca155bcf7654a8ac7a949518908852b7914bd 100644
--- a/mlir/lib/TableGen/AttrOrTypeDef.cpp
+++ b/mlir/lib/TableGen/AttrOrTypeDef.cpp
@@ -40,7 +40,7 @@ AttrOrTypeDef::AttrOrTypeDef(const llvm::Record *def) : def(def) {
   auto *builderList =
       dyn_cast_or_null<llvm::ListInit>(def->getValueInit("builders"));
   if (builderList && !builderList->empty()) {
-    for (llvm::Init *init : builderList->getValues()) {
+    for (const llvm::Init *init : builderList->getValues()) {
       AttrOrTypeBuilder builder(cast<llvm::DefInit>(init)->getDef(),
                                 def->getLoc());
 
@@ -58,8 +58,8 @@ AttrOrTypeDef::AttrOrTypeDef(const llvm::Record *def) : def(def) {
   if (auto *traitList = def->getValueAsListInit("traits")) {
     SmallPtrSet<const llvm::Init *, 32> traitSet;
     traits.reserve(traitSet.size());
-    llvm::unique_function<void(llvm::ListInit *)> processTraitList =
-        [&](llvm::ListInit *traitList) {
+    llvm::unique_function<void(const llvm::ListInit *)> processTraitList =
+        [&](const llvm::ListInit *traitList) {
           for (auto *traitInit : *traitList) {
             if (!traitSet.insert(traitInit).second)
               continue;
@@ -335,7 +335,9 @@ std::optional<StringRef> AttrOrTypeParameter::getDefaultValue() const {
   return result && !result->empty() ? result : std::nullopt;
 }
 
-llvm::Init *AttrOrTypeParameter::getDef() const { return def->getArg(index); }
+const llvm::Init *AttrOrTypeParameter::getDef() const {
+  return def->getArg(index);
+}
 
 std::optional<Constraint> AttrOrTypeParameter::getConstraint() const {
   if (auto *param = dyn_cast<llvm::DefInit>(getDef()))
@@ -349,7 +351,7 @@ std::optional<Constraint> AttrOrTypeParameter::getConstraint() const {
 //===----------------------------------------------------------------------===//
 
 bool AttributeSelfTypeParameter::classof(const AttrOrTypeParameter *param) {
-  llvm::Init *paramDef = param->getDef();
+  const llvm::Init *paramDef = param->getDef();
   if (auto *paramDefInit = dyn_cast<llvm::DefInit>(paramDef))
     return paramDefInit->getDef()->isSubClassOf("AttributeSelfTypeParameter");
   return false;
diff --git a/mlir/lib/TableGen/Attribute.cpp b/mlir/lib/TableGen/Attribute.cpp
index de930cb400703215cf468c535d62c6bf2fbe1515..887553bca66102dd71620b98fcbf5d1f90e98a76 100644
--- a/mlir/lib/TableGen/Attribute.cpp
+++ b/mlir/lib/TableGen/Attribute.cpp
@@ -126,7 +126,7 @@ StringRef Attribute::getDerivedCodeBody() const {
 Dialect Attribute::getDialect() const {
   const llvm::RecordVal *record = def->getValue("dialect");
   if (record && record->getValue()) {
-    if (DefInit *init = dyn_cast<DefInit>(record->getValue()))
+    if (const DefInit *init = dyn_cast<DefInit>(record->getValue()))
       return Dialect(init->getDef());
   }
   return Dialect(nullptr);
diff --git a/mlir/lib/TableGen/Dialect.cpp b/mlir/lib/TableGen/Dialect.cpp
index 081f6e56f9ded468cc9e4aa9efd9f6da3d973a9f..ef39818e439b3e393b61104e84a1c725762226e0 100644
--- a/mlir/lib/TableGen/Dialect.cpp
+++ b/mlir/lib/TableGen/Dialect.cpp
@@ -106,7 +106,7 @@ bool Dialect::usePropertiesForAttributes() const {
   return def->getValueAsBit("usePropertiesForAttributes");
 }
 
-llvm::DagInit *Dialect::getDiscardableAttributes() const {
+const llvm::DagInit *Dialect::getDiscardableAttributes() const {
   return def->getValueAsDag("discardableAttrs");
 }
 
diff --git a/mlir/lib/TableGen/Interfaces.cpp b/mlir/lib/TableGen/Interfaces.cpp
index a209b003b0f3bbe3b30d1bba52823b96e87bbc8a..4a6709a43d0a8f741f6ebeea87dbdcd81cd8bb70 100644
--- a/mlir/lib/TableGen/Interfaces.cpp
+++ b/mlir/lib/TableGen/Interfaces.cpp
@@ -22,7 +22,7 @@ using namespace mlir::tblgen;
 //===----------------------------------------------------------------------===//
 
 InterfaceMethod::InterfaceMethod(const llvm::Record *def) : def(def) {
-  llvm::DagInit *args = def->getValueAsDag("arguments");
+  const llvm::DagInit *args = def->getValueAsDag("arguments");
   for (unsigned i = 0, e = args->getNumArgs(); i != e; ++i) {
     arguments.push_back(
         {llvm::cast<llvm::StringInit>(args->getArg(i))->getValue(),
@@ -78,7 +78,7 @@ Interface::Interface(const llvm::Record *def) : def(def) {
 
   // Initialize the interface methods.
   auto *listInit = dyn_cast<llvm::ListInit>(def->getValueInit("methods"));
-  for (llvm::Init *init : listInit->getValues())
+  for (const llvm::Init *init : listInit->getValues())
     methods.emplace_back(cast<llvm::DefInit>(init)->getDef());
 
   // Initialize the interface base classes.
@@ -98,7 +98,7 @@ Interface::Interface(const llvm::Record *def) : def(def) {
         baseInterfaces.push_back(std::make_unique<Interface>(baseInterface));
         basesAdded.insert(baseInterface.getName());
       };
-  for (llvm::Init *init : basesInit->getValues())
+  for (const llvm::Init *init : basesInit->getValues())
     addBaseInterfaceFn(Interface(cast<llvm::DefInit>(init)->getDef()));
 }
 
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index 6a33ff5ecd67214b2bfd32ba4ff967c26db5675c..86670e9f87127c4752ff7e3c96171db4e6bb1794 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -161,7 +161,7 @@ std::string Operator::getQualCppClassName() const {
 StringRef Operator::getCppNamespace() const { return cppNamespace; }
 
 int Operator::getNumResults() const {
-  DagInit *results = def.getValueAsDag("results");
+  const DagInit *results = def.getValueAsDag("results");
   return results->getNumArgs();
 }
 
@@ -198,12 +198,12 @@ auto Operator::getResults() const -> const_value_range {
 }
 
 TypeConstraint Operator::getResultTypeConstraint(int index) const {
-  DagInit *results = def.getValueAsDag("results");
+  const DagInit *results = def.getValueAsDag("results");
   return TypeConstraint(cast<DefInit>(results->getArg(index)));
 }
 
 StringRef Operator::getResultName(int index) const {
-  DagInit *results = def.getValueAsDag("results");
+  const DagInit *results = def.getValueAsDag("results");
   return results->getArgNameStr(index);
 }
 
@@ -241,7 +241,7 @@ Operator::arg_range Operator::getArgs() const {
 }
 
 StringRef Operator::getArgName(int index) const {
-  DagInit *argumentValues = def.getValueAsDag("arguments");
+  const DagInit *argumentValues = def.getValueAsDag("arguments");
   return argumentValues->getArgNameStr(index);
 }
 
@@ -557,7 +557,7 @@ void Operator::populateOpStructure() {
   auto *opVarClass = recordKeeper.getClass("OpVariable");
   numNativeAttributes = 0;
 
-  DagInit *argumentValues = def.getValueAsDag("arguments");
+  const DagInit *argumentValues = def.getValueAsDag("arguments");
   unsigned numArgs = argumentValues->getNumArgs();
 
   // Mapping from name of to argument or result index. Arguments are indexed
@@ -721,8 +721,8 @@ void Operator::populateOpStructure() {
                   " to precede it in traits list");
     };
 
-    std::function<void(llvm::ListInit *)> insert;
-    insert = [&](llvm::ListInit *traitList) {
+    std::function<void(const llvm::ListInit *)> insert;
+    insert = [&](const llvm::ListInit *traitList) {
       for (auto *traitInit : *traitList) {
         auto *def = cast<DefInit>(traitInit)->getDef();
         if (def->isSubClassOf("TraitList")) {
@@ -780,7 +780,7 @@ void Operator::populateOpStructure() {
   auto *builderList =
       dyn_cast_or_null<llvm::ListInit>(def.getValueInit("builders"));
   if (builderList && !builderList->empty()) {
-    for (llvm::Init *init : builderList->getValues())
+    for (const llvm::Init *init : builderList->getValues())
       builders.emplace_back(cast<llvm::DefInit>(init)->getDef(), def.getLoc());
   } else if (skipDefaultBuilders()) {
     PrintFatalError(
@@ -818,7 +818,8 @@ bool Operator::hasAssemblyFormat() const {
 }
 
 StringRef Operator::getAssemblyFormat() const {
-  return TypeSwitch<llvm::Init *, StringRef>(def.getValueInit("assemblyFormat"))
+  return TypeSwitch<const llvm::Init *, StringRef>(
+             def.getValueInit("assemblyFormat"))
       .Case<llvm::StringInit>([&](auto *init) { return init->getValue(); });
 }
 
@@ -832,7 +833,7 @@ void Operator::print(llvm::raw_ostream &os) const {
   }
 }
 
-auto Operator::VariableDecoratorIterator::unwrap(llvm::Init *init)
+auto Operator::VariableDecoratorIterator::unwrap(const llvm::Init *init)
     -> VariableDecorator {
   return VariableDecorator(cast<llvm::DefInit>(init)->getDef());
 }
diff --git a/mlir/lib/TableGen/Pattern.cpp b/mlir/lib/TableGen/Pattern.cpp
index 6437839ef20849bd430d6de7661770a06f675518..bee20354387fd69da8b7686961c89052ee26b6e1 100644
--- a/mlir/lib/TableGen/Pattern.cpp
+++ b/mlir/lib/TableGen/Pattern.cpp
@@ -700,7 +700,7 @@ int Pattern::getBenefit() const {
   // The initial benefit value is a heuristic with number of ops in the source
   // pattern.
   int initBenefit = getSourcePattern().getNumOps();
-  llvm::DagInit *delta = def.getValueAsDag("benefitDelta");
+  const llvm::DagInit *delta = def.getValueAsDag("benefitDelta");
   if (delta->getNumArgs() != 1 || !isa<llvm::IntInit>(delta->getArg(0))) {
     PrintFatalError(&def,
                     "The 'addBenefit' takes and only takes one integer value");
diff --git a/mlir/lib/TableGen/Type.cpp b/mlir/lib/TableGen/Type.cpp
index cda752297988bbd654dc7690765c3de15e08e3fb..c3b813ec598d0a0ade98dc2729105808c97ac7fa 100644
--- a/mlir/lib/TableGen/Type.cpp
+++ b/mlir/lib/TableGen/Type.cpp
@@ -50,7 +50,7 @@ std::optional<StringRef> TypeConstraint::getBuilderCall() const {
   const llvm::RecordVal *builderCall = baseType->getValue("builderCall");
   if (!builderCall || !builderCall->getValue())
     return std::nullopt;
-  return TypeSwitch<llvm::Init *, std::optional<StringRef>>(
+  return TypeSwitch<const llvm::Init *, std::optional<StringRef>>(
              builderCall->getValue())
       .Case<llvm::StringInit>([&](auto *init) {
         StringRef value = init->getValue();
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
index 46b7b0a473c692812a743ffb9c0b66c0f7129dfa..a8595d14ccf2e5104c8a1e46942ee54d2e749cf8 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
@@ -99,7 +99,8 @@ getOverloadedDeclaration(CallIntrinsicOp op, llvm::Intrinsic::ID id,
   }
 
   ArrayRef<llvm::Type *> overloadedArgTysRef = overloadedArgTys;
-  return llvm::Intrinsic::getDeclaration(module, id, overloadedArgTysRef);
+  return llvm::Intrinsic::getOrInsertDeclaration(module, id,
+                                                 overloadedArgTysRef);
 }
 
 static llvm::OperandBundleDef
@@ -143,7 +144,7 @@ convertCallLLVMIntrinsicOp(CallIntrinsicOp op, llvm::IRBuilderBase &builder,
       return failure();
     fn = *fnOrFailure;
   } else {
-    fn = llvm::Intrinsic::getDeclaration(module, id, {});
+    fn = llvm::Intrinsic::getOrInsertDeclaration(module, id, {});
   }
 
   // Check the result type of the call.
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 19d80fbbd699b041cae8f1ab59f9ddd89f4329ac..4a575f4e57706263e22d6f11016e07d83263d46e 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -480,12 +480,11 @@ makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder,
       [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
                 llvm::Value *lhs, llvm::Value *rhs,
                 llvm::Value *&result) mutable {
-        Region &reductionRegion = decl.getReductionRegion();
-        moduleTranslation.mapValue(reductionRegion.front().getArgument(0), lhs);
-        moduleTranslation.mapValue(reductionRegion.front().getArgument(1), rhs);
+        moduleTranslation.mapValue(decl.getReductionLhsArg(), lhs);
+        moduleTranslation.mapValue(decl.getReductionRhsArg(), rhs);
         builder.restoreIP(insertPoint);
         SmallVector<llvm::Value *> phis;
-        if (failed(inlineConvertOmpRegions(reductionRegion,
+        if (failed(inlineConvertOmpRegions(decl.getReductionRegion(),
                                            "omp.reduction.nonatomic.body",
                                            builder, moduleTranslation, &phis)))
           return llvm::OpenMPIRBuilder::InsertPointTy();
@@ -513,12 +512,11 @@ makeAtomicReductionGen(omp::DeclareReductionOp decl,
   OwningAtomicReductionGen atomicGen =
       [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
                 llvm::Value *lhs, llvm::Value *rhs) mutable {
-        Region &atomicRegion = decl.getAtomicReductionRegion();
-        moduleTranslation.mapValue(atomicRegion.front().getArgument(0), lhs);
-        moduleTranslation.mapValue(atomicRegion.front().getArgument(1), rhs);
+        moduleTranslation.mapValue(decl.getAtomicReductionLhsArg(), lhs);
+        moduleTranslation.mapValue(decl.getAtomicReductionRhsArg(), rhs);
         builder.restoreIP(insertPoint);
         SmallVector<llvm::Value *> phis;
-        if (failed(inlineConvertOmpRegions(atomicRegion,
+        if (failed(inlineConvertOmpRegions(decl.getAtomicReductionRegion(),
                                            "omp.reduction.atomic.body", builder,
                                            moduleTranslation, &phis)))
           return llvm::OpenMPIRBuilder::InsertPointTy();
@@ -1674,9 +1672,10 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
         // argument of the `alloc` region and the second argument of the `copy`
         // region to be the yielded value of the `alloc` region (this is the
         // private clone of the privatized value).
-        copyCloneBuilder.mergeBlocks(
-            &*newCopyRegionFrontBlock, &*oldAllocBackBlock,
-            {allocRegion.getArgument(0), oldAllocYieldOp.getOperand(0)});
+        copyCloneBuilder.mergeBlocks(&*newCopyRegionFrontBlock,
+                                     &*oldAllocBackBlock,
+                                     {mlirPrivatizerClone.getAllocMoldArg(),
+                                      oldAllocYieldOp.getOperand(0)});
 
         // 4. The old terminator of the `alloc` region is not needed anymore, so
         // delete it.
@@ -1686,8 +1685,8 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       // Replace the privatizer block argument with mlir value being privatized.
       // This way, the body of the privatizer will be changed from using the
       // region/block argument to the value being privatized.
-      auto allocRegionArg = allocRegion.getArgument(0);
-      replaceAllUsesInRegionWith(allocRegionArg, mlirPrivVar, allocRegion);
+      replaceAllUsesInRegionWith(mlirPrivatizerClone.getAllocMoldArg(),
+                                 mlirPrivVar, allocRegion);
 
       auto oldIP = builder.saveIP();
       builder.restoreIP(allocaIP);
@@ -3242,11 +3241,6 @@ static bool targetOpSupported(Operation &opInst) {
     return false;
   }
 
-  if (targetOp.getNowait()) {
-    opInst.emitError("Nowait clause not yet supported");
-    return false;
-  }
-
   if (!targetOp.getAllocateVars().empty() ||
       !targetOp.getAllocatorVars().empty()) {
     opInst.emitError("Allocate clause not yet supported");
@@ -3480,10 +3474,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                            " private allocatables is not supported yet");
           bodyGenStatus = failure();
         } else {
-          Region &allocRegion = privatizer.getAllocRegion();
-          BlockArgument allocRegionArg = allocRegion.getArgument(0);
-          moduleTranslation.mapValue(allocRegionArg,
+          moduleTranslation.mapValue(privatizer.getAllocMoldArg(),
                                      moduleTranslation.lookupValue(privVar));
+          Region &allocRegion = privatizer.getAllocRegion();
           SmallVector<llvm::Value *, 1> yieldedValues;
           if (failed(inlineConvertOmpRegions(
                   allocRegion, "omp.targetop.privatizer", builder,
@@ -3569,7 +3562,7 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
   builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTarget(
       ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), entryInfo,
       defaultValTeams, defaultValThreads, kernelInput, genMapInfoCB, bodyCB,
-      argAccessorCB, dds));
+      argAccessorCB, dds, targetOp.getNowait()));
 
   // Remap access operations to declare target reference pointers for the
   // device, essentially generating extra loadop's as necessary
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 4ff1f1135b0a88f042b0964846f2eefec82dc299..bd861f3a69e53c73963ecf11e4566b607075aea9 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -535,6 +535,23 @@ LogicalResult ModuleImport::convertIdentMetadata() {
   return success();
 }
 
+LogicalResult ModuleImport::convertCommandlineMetadata() {
+  for (const llvm::NamedMDNode &nmd : llvmModule->named_metadata()) {
+    // llvm.commandline should have a single operand. That operand is itself an
+    // MDNode with a single string operand.
+    if (nmd.getName() != LLVMDialect::getCommandlineAttrName())
+      continue;
+
+    if (nmd.getNumOperands() == 1)
+      if (auto *md = dyn_cast<llvm::MDNode>(nmd.getOperand(0)))
+        if (md->getNumOperands() == 1)
+          if (auto *mdStr = dyn_cast<llvm::MDString>(md->getOperand(0)))
+            mlirModule->setAttr(LLVMDialect::getCommandlineAttrName(),
+                                builder.getStringAttr(mdStr->getString()));
+  }
+  return success();
+}
+
 LogicalResult ModuleImport::convertMetadata() {
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(mlirModule.getBody());
@@ -565,6 +582,8 @@ LogicalResult ModuleImport::convertMetadata() {
     return failure();
   if (failed(convertIdentMetadata()))
     return failure();
+  if (failed(convertCommandlineMetadata()))
+    return failure();
   return success();
 }
 
@@ -914,14 +933,15 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
 
   // Get the global expression associated with this global variable and convert
   // it.
-  DIGlobalVariableExpressionAttr globalExpressionAttr;
+  SmallVector<Attribute> globalExpressionAttrs;
   SmallVector<llvm::DIGlobalVariableExpression *> globalExpressions;
   globalVar->getDebugInfo(globalExpressions);
 
-  // There should only be a single global expression.
-  if (!globalExpressions.empty())
-    globalExpressionAttr =
-        debugImporter->translateGlobalVariableExpression(globalExpressions[0]);
+  for (llvm::DIGlobalVariableExpression *expr : globalExpressions) {
+    DIGlobalVariableExpressionAttr globalExpressionAttr =
+        debugImporter->translateGlobalVariableExpression(expr);
+    globalExpressionAttrs.push_back(globalExpressionAttr);
+  }
 
   // Workaround to support LLVM's nameless globals. MLIR, in contrast to LLVM,
   // always requires a symbol name.
@@ -935,7 +955,7 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
       valueAttr, alignment, /*addr_space=*/globalVar->getAddressSpace(),
       /*dso_local=*/globalVar->isDSOLocal(),
       /*thread_local=*/globalVar->isThreadLocal(), /*comdat=*/SymbolRefAttr(),
-      /*attrs=*/ArrayRef<NamedAttribute>(), /*dbgExpr=*/globalExpressionAttr);
+      /*attrs=*/ArrayRef<NamedAttribute>(), /*dbgExprs=*/globalExpressionAttrs);
   globalInsertionOp = globalOp;
 
   if (globalVar->hasInitializer() && !valueAttr) {
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index a5de90160c41458761ae4b47826c62f9e042734a..6e005f9ec5df85cb161cf40d7ab53cb4d75cdba7 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -839,7 +839,8 @@ llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
     llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
     ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) {
   llvm::Module *module = builder.GetInsertBlock()->getModule();
-  llvm::Function *fn = llvm::Intrinsic::getDeclaration(module, intrinsic, tys);
+  llvm::Function *fn =
+      llvm::Intrinsic::getOrInsertDeclaration(module, intrinsic, tys);
   return builder.CreateCall(fn, args);
 }
 
@@ -886,8 +887,8 @@ llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
   for (unsigned overloadedOperandIdx : overloadedOperands)
     overloadedTypes.push_back(args[overloadedOperandIdx]->getType());
   llvm::Module *module = builder.GetInsertBlock()->getModule();
-  llvm::Function *llvmIntr =
-      llvm::Intrinsic::getDeclaration(module, intrinsic, overloadedTypes);
+  llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration(
+      module, intrinsic, overloadedTypes);
 
   return builder.CreateCall(llvmIntr, args);
 }
@@ -1055,44 +1056,48 @@ LogicalResult ModuleTranslation::convertGlobals() {
     globalsMapping.try_emplace(op, var);
 
     // Add debug information if present.
-    if (op.getDbgExpr()) {
-      llvm::DIGlobalVariableExpression *diGlobalExpr =
-          debugTranslation->translateGlobalVariableExpression(op.getDbgExpr());
-      llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();
-      var->addDebugInfo(diGlobalExpr);
-
-      // There is no `globals` field in DICompileUnitAttr which can be directly
-      // assigned to DICompileUnit. We have to build the list by looking at the
-      // dbgExpr of all the GlobalOps. The scope of the variable is used to get
-      // the DICompileUnit in which to add it.
-      // But there are cases where the scope of a global does not
-      // directly point to the DICompileUnit and we have to do a bit more work
-      // to get to it. Some of those cases are:
-      //
-      // 1. For the languages that support modules, the scope hierarchy can be
-      // variable -> DIModule -> DICompileUnit
-      //
-      // 2. For the Fortran common block variable, the scope hierarchy can be
-      // variable -> DICommonBlock -> DISubprogram -> DICompileUnit
-      //
-      // 3. For entities like static local variables in C or variable with
-      // SAVE attribute in Fortran, the scope hierarchy can be
-      // variable -> DISubprogram -> DICompileUnit
-      llvm::DIScope *scope = diGlobalVar->getScope();
-      if (auto *mod = dyn_cast_if_present<llvm::DIModule>(scope))
-        scope = mod->getScope();
-      else if (auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(scope)) {
-        if (auto *sp = dyn_cast_if_present<llvm::DISubprogram>(cb->getScope()))
+    if (op.getDbgExprs()) {
+      for (auto exprAttr :
+           op.getDbgExprs()->getAsRange<DIGlobalVariableExpressionAttr>()) {
+        llvm::DIGlobalVariableExpression *diGlobalExpr =
+            debugTranslation->translateGlobalVariableExpression(exprAttr);
+        llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();
+        var->addDebugInfo(diGlobalExpr);
+
+        // There is no `globals` field in DICompileUnitAttr which can be
+        // directly assigned to DICompileUnit. We have to build the list by
+        // looking at the dbgExpr of all the GlobalOps. The scope of the
+        // variable is used to get the DICompileUnit in which to add it. But
+        // there are cases where the scope of a global does not directly point
+        // to the DICompileUnit and we have to do a bit more work to get to
+        // it. Some of those cases are:
+        //
+        // 1. For the languages that support modules, the scope hierarchy can
+        // be variable -> DIModule -> DICompileUnit
+        //
+        // 2. For the Fortran common block variable, the scope hierarchy can
+        // be variable -> DICommonBlock -> DISubprogram -> DICompileUnit
+        //
+        // 3. For entities like static local variables in C or variable with
+        // SAVE attribute in Fortran, the scope hierarchy can be
+        // variable -> DISubprogram -> DICompileUnit
+        llvm::DIScope *scope = diGlobalVar->getScope();
+        if (auto *mod = dyn_cast_if_present<llvm::DIModule>(scope))
+          scope = mod->getScope();
+        else if (auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(scope)) {
+          if (auto *sp =
+                  dyn_cast_if_present<llvm::DISubprogram>(cb->getScope()))
+            scope = sp->getUnit();
+        } else if (auto *sp = dyn_cast_if_present<llvm::DISubprogram>(scope))
           scope = sp->getUnit();
-      } else if (auto *sp = dyn_cast_if_present<llvm::DISubprogram>(scope))
-        scope = sp->getUnit();
-
-      // Get the compile unit (scope) of the the global variable.
-      if (llvm::DICompileUnit *compileUnit =
-              dyn_cast_if_present<llvm::DICompileUnit>(scope)) {
-        // Update the compile unit with this incoming global variable expression
-        // during the finalizing step later.
-        allGVars[compileUnit].push_back(diGlobalExpr);
+
+        // Get the compile unit (scope) of the the global variable.
+        if (llvm::DICompileUnit *compileUnit =
+                dyn_cast_if_present<llvm::DICompileUnit>(scope)) {
+          // Update the compile unit with this incoming global variable
+          // expression during the finalizing step later.
+          allGVars[compileUnit].push_back(diGlobalExpr);
+        }
       }
     }
   }
@@ -1832,6 +1837,21 @@ LogicalResult ModuleTranslation::createIdentMetadata() {
   return success();
 }
 
+LogicalResult ModuleTranslation::createCommandlineMetadata() {
+  if (auto attr = mlirModule->getAttrOfType<StringAttr>(
+          LLVMDialect::getCommandlineAttrName())) {
+    StringRef cmdLine = attr;
+    llvm::LLVMContext &ctx = llvmModule->getContext();
+    llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata(
+        LLVMDialect::getCommandlineAttrName());
+    llvm::MDNode *md =
+        llvm::MDNode::get(ctx, llvm::MDString::get(ctx, cmdLine));
+    nmd->addOperand(md);
+  }
+
+  return success();
+}
+
 void ModuleTranslation::setLoopMetadata(Operation *op,
                                         llvm::Instruction *inst) {
   LoopAnnotationAttr attr =
@@ -1985,6 +2005,8 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
     return nullptr;
   if (failed(translator.createIdentMetadata()))
     return nullptr;
+  if (failed(translator.createCommandlineMetadata()))
+    return nullptr;
 
   // Convert other top-level operations if possible.
   for (Operation &o : getModuleBody(module).getOperations()) {
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 6c7fe41069824fc4de5fc877a2cc129ec4f1b61d..462d3e326b6c27a9f24b3e3a8d36b5952dc9712b 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -1287,7 +1287,8 @@ LogicalResult spirv::Deserializer::processConstant(ArrayRef<uint32_t> operands,
       } words = {operands[2], operands[3]};
       value = APInt(64, llvm::bit_cast<uint64_t>(words), /*isSigned=*/true);
     } else if (bitwidth <= 32) {
-      value = APInt(bitwidth, operands[2], /*isSigned=*/true);
+      value = APInt(bitwidth, operands[2], /*isSigned=*/true,
+                    /*implicitTrunc=*/true);
     }
 
     auto attr = opBuilder.getIntegerAttr(intType, value);
diff --git a/mlir/lib/Tools/mlir-tblgen/MlirTblgenMain.cpp b/mlir/lib/Tools/mlir-tblgen/MlirTblgenMain.cpp
index 7119324dd125d55ff95de9de9d25aabb89b2eb85..20ad4292a548bfdbb72f7009ddbda9efe77c6634 100644
--- a/mlir/lib/Tools/mlir-tblgen/MlirTblgenMain.cpp
+++ b/mlir/lib/Tools/mlir-tblgen/MlirTblgenMain.cpp
@@ -30,8 +30,8 @@ enum DeprecatedAction { None, Warn, Error };
 static DeprecatedAction actionOnDeprecatedValue;
 
 // Returns if there is a use of `deprecatedInit` in `field`.
-static bool findUse(Init *field, Init *deprecatedInit,
-                    llvm::DenseMap<Init *, bool> &known) {
+static bool findUse(const Init *field, const Init *deprecatedInit,
+                    llvm::DenseMap<const Init *, bool> &known) {
   if (field == deprecatedInit)
     return true;
 
@@ -64,13 +64,13 @@ static bool findUse(Init *field, Init *deprecatedInit,
     if (findUse(dagInit->getOperator(), deprecatedInit, known))
       return memoize(true);
 
-    return memoize(llvm::any_of(dagInit->getArgs(), [&](Init *arg) {
+    return memoize(llvm::any_of(dagInit->getArgs(), [&](const Init *arg) {
       return findUse(arg, deprecatedInit, known);
     }));
   }
 
-  if (ListInit *li = dyn_cast<ListInit>(field)) {
-    return memoize(llvm::any_of(li->getValues(), [&](Init *jt) {
+  if (const ListInit *li = dyn_cast<ListInit>(field)) {
+    return memoize(llvm::any_of(li->getValues(), [&](const Init *jt) {
       return findUse(jt, deprecatedInit, known);
     }));
   }
@@ -83,8 +83,8 @@ static bool findUse(Init *field, Init *deprecatedInit,
 }
 
 // Returns if there is a use of `deprecatedInit` in `record`.
-static bool findUse(Record &record, Init *deprecatedInit,
-                    llvm::DenseMap<Init *, bool> &known) {
+static bool findUse(Record &record, const Init *deprecatedInit,
+                    llvm::DenseMap<const Init *, bool> &known) {
   return llvm::any_of(record.getValues(), [&](const RecordVal &val) {
     return findUse(val.getValue(), deprecatedInit, known);
   });
@@ -100,7 +100,7 @@ static void warnOfDeprecatedUses(const RecordKeeper &records) {
     if (!r || !r->getValue())
       continue;
 
-    llvm::DenseMap<Init *, bool> hasUse;
+    llvm::DenseMap<const Init *, bool> hasUse;
     if (auto *si = dyn_cast<StringInit>(r->getValue())) {
       for (auto &jt : records.getDefs()) {
         // Skip anonymous defs.
diff --git a/mlir/lib/Transforms/SROA.cpp b/mlir/lib/Transforms/SROA.cpp
index aca252b01dce7b63b474ccc7261bd70c51047e50..db8be38a51443e09d280a2bc98c9f8818ff520f8 100644
--- a/mlir/lib/Transforms/SROA.cpp
+++ b/mlir/lib/Transforms/SROA.cpp
@@ -100,10 +100,11 @@ computeDestructuringInfo(DestructurableMemorySlot &slot,
   mlir::getForwardSlice(slot.ptr, &forwardSlice);
   for (Operation *user : forwardSlice) {
     // If the next operation has no blocking uses, everything is fine.
-    if (!info.userToBlockingUses.contains(user))
+    auto it = info.userToBlockingUses.find(user);
+    if (it == info.userToBlockingUses.end())
       continue;
 
-    SmallPtrSet<OpOperand *, 4> &blockingUses = info.userToBlockingUses[user];
+    SmallPtrSet<OpOperand *, 4> &blockingUses = it->second;
     auto promotable = dyn_cast<PromotableOpInterface>(user);
 
     // An operation that has blocking uses must be promoted. If it is not
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 97dd3ab1f48293a64750cbc460fc793dfe4e3dd6..1baddd881f6aa252926516a4a0f4d6d5260cac52 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -683,10 +683,10 @@ enum MaterializationKind {
 /// conversion.
 class UnresolvedMaterializationRewrite : public OperationRewrite {
 public:
-  UnresolvedMaterializationRewrite(
-      ConversionPatternRewriterImpl &rewriterImpl,
-      UnrealizedConversionCastOp op, const TypeConverter *converter = nullptr,
-      MaterializationKind kind = MaterializationKind::Target);
+  UnresolvedMaterializationRewrite(ConversionPatternRewriterImpl &rewriterImpl,
+                                   UnrealizedConversionCastOp op,
+                                   const TypeConverter *converter,
+                                   MaterializationKind kind, Type originalType);
 
   static bool classof(const IRRewrite *rewrite) {
     return rewrite->getKind() == Kind::UnresolvedMaterialization;
@@ -708,11 +708,18 @@ public:
     return converterAndKind.getInt();
   }
 
+  /// Return the original type of the SSA value.
+  Type getOriginalType() const { return originalType; }
+
 private:
   /// The corresponding type converter to use when resolving this
   /// materialization, and the kind of this materialization.
   llvm::PointerIntPair<const TypeConverter *, 2, MaterializationKind>
       converterAndKind;
+
+  /// The original type of the SSA value. Only used for target
+  /// materializations.
+  Type originalType;
 };
 } // namespace
 
@@ -808,6 +815,7 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   Value buildUnresolvedMaterialization(MaterializationKind kind,
                                        OpBuilder::InsertPoint ip, Location loc,
                                        ValueRange inputs, Type outputType,
+                                       Type originalType,
                                        const TypeConverter *converter);
 
   //===--------------------------------------------------------------------===//
@@ -1034,9 +1042,12 @@ void CreateOperationRewrite::rollback() {
 
 UnresolvedMaterializationRewrite::UnresolvedMaterializationRewrite(
     ConversionPatternRewriterImpl &rewriterImpl, UnrealizedConversionCastOp op,
-    const TypeConverter *converter, MaterializationKind kind)
+    const TypeConverter *converter, MaterializationKind kind, Type originalType)
     : OperationRewrite(Kind::UnresolvedMaterialization, rewriterImpl, op),
-      converterAndKind(converter, kind) {
+      converterAndKind(converter, kind), originalType(originalType) {
+  assert(!originalType ||
+         kind == MaterializationKind::Target &&
+             "original type is valid only for target materializations");
   rewriterImpl.unresolvedMaterializations[op] = this;
 }
 
@@ -1139,7 +1150,7 @@ LogicalResult ConversionPatternRewriterImpl::remapValues(
       Value castValue = buildUnresolvedMaterialization(
           MaterializationKind::Target, computeInsertPoint(newOperand),
           operandLoc, /*inputs=*/newOperand, /*outputType=*/desiredType,
-          currentTypeConverter);
+          /*originalType=*/origType, currentTypeConverter);
       mapping.map(newOperand, castValue);
       newOperand = castValue;
     }
@@ -1255,7 +1266,7 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
           MaterializationKind::Source,
           OpBuilder::InsertPoint(newBlock, newBlock->begin()), origArg.getLoc(),
           /*inputs=*/ValueRange(),
-          /*outputType=*/origArgType, converter);
+          /*outputType=*/origArgType, /*originalType=*/Type(), converter);
       mapping.map(origArg, repl);
       appendRewrite<ReplaceBlockArgRewrite>(block, origArg);
       continue;
@@ -1280,7 +1291,8 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
     Value argMat = buildUnresolvedMaterialization(
         MaterializationKind::Argument,
         OpBuilder::InsertPoint(newBlock, newBlock->begin()), origArg.getLoc(),
-        /*inputs=*/replArgs, origArgType, converter);
+        /*inputs=*/replArgs, /*outputType=*/origArgType,
+        /*originalType=*/Type(), converter);
     mapping.map(origArg, argMat);
 
     Type legalOutputType;
@@ -1299,7 +1311,8 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
     if (legalOutputType && legalOutputType != origArgType) {
       Value targetMat = buildUnresolvedMaterialization(
           MaterializationKind::Target, computeInsertPoint(argMat),
-          origArg.getLoc(), argMat, legalOutputType, converter);
+          origArg.getLoc(), /*inputs=*/argMat, /*outputType=*/legalOutputType,
+          /*originalType=*/origArgType, converter);
       mapping.map(argMat, targetMat);
     }
     appendRewrite<ReplaceBlockArgRewrite>(block, origArg);
@@ -1322,7 +1335,12 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
 /// of input operands.
 Value ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
     MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc,
-    ValueRange inputs, Type outputType, const TypeConverter *converter) {
+    ValueRange inputs, Type outputType, Type originalType,
+    const TypeConverter *converter) {
+  assert(!originalType ||
+         kind == MaterializationKind::Target &&
+             "original type is valid only for target materializations");
+
   // Avoid materializing an unnecessary cast.
   if (inputs.size() == 1 && inputs.front().getType() == outputType)
     return inputs.front();
@@ -1333,7 +1351,8 @@ Value ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
   builder.setInsertionPoint(ip.getBlock(), ip.getPoint());
   auto convertOp =
       builder.create<UnrealizedConversionCastOp>(loc, outputType, inputs);
-  appendRewrite<UnresolvedMaterializationRewrite>(convertOp, converter, kind);
+  appendRewrite<UnresolvedMaterializationRewrite>(convertOp, converter, kind,
+                                                  originalType);
   return convertOp.getResult(0);
 }
 
@@ -1381,7 +1400,8 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(Operation *op,
       newValue = buildUnresolvedMaterialization(
           MaterializationKind::Source, computeInsertPoint(result),
           result.getLoc(), /*inputs=*/ValueRange(),
-          /*outputType=*/result.getType(), currentTypeConverter);
+          /*outputType=*/result.getType(), /*originalType=*/Type(),
+          currentTypeConverter);
     }
 
     // Remap, and check for any result type changes.
@@ -2408,7 +2428,8 @@ legalizeUnresolvedMaterialization(RewriterBase &rewriter,
       [[fallthrough]];
     case MaterializationKind::Target:
       newMaterialization = converter->materializeTargetConversion(
-          rewriter, op->getLoc(), outputType, inputOperands);
+          rewriter, op->getLoc(), outputType, inputOperands,
+          rewrite->getOriginalType());
       break;
     case MaterializationKind::Source:
       newMaterialization = converter->materializeSourceConversion(
@@ -2565,7 +2586,7 @@ void OperationConverter::finalize(ConversionPatternRewriter &rewriter) {
           MaterializationKind::Source, computeInsertPoint(newValue),
           originalValue.getLoc(),
           /*inputs=*/newValue, /*outputType=*/originalValue.getType(),
-          converter);
+          /*originalType=*/Type(), converter);
       rewriterImpl.mapping.map(originalValue, castValue);
       inverseMapping[castValue].push_back(originalValue);
       llvm::erase(inverseMapping[newValue], originalValue);
@@ -2787,15 +2808,39 @@ TypeConverter::convertSignatureArgs(TypeRange types,
   return success();
 }
 
-Value TypeConverter::materializeConversion(
-    ArrayRef<MaterializationCallbackFn> materializations, OpBuilder &builder,
-    Location loc, Type resultType, ValueRange inputs) const {
-  for (const MaterializationCallbackFn &fn : llvm::reverse(materializations))
+Value TypeConverter::materializeArgumentConversion(OpBuilder &builder,
+                                                   Location loc,
+                                                   Type resultType,
+                                                   ValueRange inputs) const {
+  for (const MaterializationCallbackFn &fn :
+       llvm::reverse(argumentMaterializations))
+    if (std::optional<Value> result = fn(builder, resultType, inputs, loc))
+      return *result;
+  return nullptr;
+}
+
+Value TypeConverter::materializeSourceConversion(OpBuilder &builder,
+                                                 Location loc, Type resultType,
+                                                 ValueRange inputs) const {
+  for (const MaterializationCallbackFn &fn :
+       llvm::reverse(sourceMaterializations))
     if (std::optional<Value> result = fn(builder, resultType, inputs, loc))
       return *result;
   return nullptr;
 }
 
+Value TypeConverter::materializeTargetConversion(OpBuilder &builder,
+                                                 Location loc, Type resultType,
+                                                 ValueRange inputs,
+                                                 Type originalType) const {
+  for (const TargetMaterializationCallbackFn &fn :
+       llvm::reverse(targetMaterializations))
+    if (std::optional<Value> result =
+            fn(builder, resultType, inputs, loc, originalType))
+      return *result;
+  return nullptr;
+}
+
 std::optional<TypeConverter::SignatureConversion>
 TypeConverter::convertBlockSignature(Block *block) const {
   SignatureConversion conversion(block->getNumArguments());
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
index d5e79b4d3cb6dd0459ad885683204b05e62b0344..e4a6ec7487bb2f1df71bc09333366d5990eb0d60 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
@@ -383,6 +383,23 @@ def select(
     O[None] = TernaryFn.select(cond[None], lhs[None], rhs[None])
 
 
+@linalg_structured_op
+def matmul(
+    A=TensorDef(T1, S.M, S.K),
+    B=TensorDef(T2, S.K, S.N),
+    C=TensorDef(U, S.M, S.N, output=True),
+    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
+):
+    """Performs a matrix multiplication of two 2D inputs.
+
+    Numeric casting is performed on the operands to the inner multiply, promoting
+    them to the same data type as the accumulator/output.
+    """
+    domain(D.m, D.n, D.k)
+    implements(ContractionOpInterface)
+    C[D.m, D.n] += cast(U, A[D.m, D.k]) * cast(U, B[D.k, D.n])
+
+
 @linalg_structured_op
 def quantized_matmul(
     A=TensorDef(T1, S.M, S.K),
diff --git a/mlir/test/CAPI/execution_engine.c b/mlir/test/CAPI/execution_engine.c
index 81ff8477ffd7b73f39daac1bad2b1b0cd7c32786..18120c6ec802801142cf3a773234f69eb919d71d 100644
--- a/mlir/test/CAPI/execution_engine.c
+++ b/mlir/test/CAPI/execution_engine.c
@@ -103,7 +103,6 @@ void testOmpCreation(void) {
 "        omp.loop_nest (%3) : i32 = (%0) to (%2) step (%1) {                    \n"
 "          omp.yield                                                            \n"
 "        }                                                                      \n"
-"        omp.terminator                                                         \n"
 "      }                                                                        \n"
 "      omp.terminator                                                           \n"
 "    }                                                                          \n"
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index 4d2d738b734ec637ea73c4c3eb68154f4bfc5446..361981605a76b7fde9a473c9e6d72803b041ceb9 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -115,6 +115,7 @@ set(MLIR_TEST_DEPENDS
   mlir-opt
   mlir-query
   mlir-reduce
+  mlir-rewrite
   mlir-tblgen
   mlir-translate
   tblgen-lsp-server
diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir
index dee9cc97a14493b2b68b637214cafe0160d15b2d..fda01974d3fc85587bf94d937bce29ee5216683c 100644
--- a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir
+++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir
@@ -41,6 +41,22 @@ func.func @zero_rank() {
 
 // -----
 
+func.func @zero_dim_rank_1() {
+  // expected-error@+1 {{failed to legalize operation 'memref.alloca'}}
+  %0 = memref.alloca() : memref<0xf32>
+  return
+}
+
+// -----
+
+func.func @zero_dim_rank_3() {
+  // expected-error@+1 {{failed to legalize operation 'memref.alloca'}}
+  %0 = memref.alloca() : memref<2x0x4xf32>
+  return
+}
+
+// -----
+
 // expected-error@+1 {{failed to legalize operation 'memref.global'}}
 memref.global "nested" constant @nested_global : memref<3x7xf32>
 
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index 5ab6802c75700f478119b4eaa904e7770ee7aa67..e496c66f86522b195608f8366666e4fb7109b9ab 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt -convert-openmp-to-llvm -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -convert-to-llvm -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: llvm.func @foo(i64, i64)
 func.func private @foo(index, index)
@@ -95,7 +96,6 @@ func.func @wsloop(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4:
         "test.payload"(%arg6, %arg7) : (index, index) -> ()
         omp.yield
       }
-      omp.terminator
     }) : () -> ()
     omp.terminator
   }
@@ -188,7 +188,6 @@ func.func @loop_nest_block_arg(%val : i32, %ub : i32, %i : index) {
     ^bb3:
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -346,7 +345,6 @@ llvm.func @_QPsb() {
 // CHECK:          %[[ZEXT:.+]] = llvm.zext %[[CMP]] : i1 to i32
 // CHECK:          llvm.store %[[ZEXT]], %[[PRV]] : i32, !llvm.ptr
 // CHECK:          omp.yield
-// CHECK:        omp.terminator
 // CHECK:      omp.terminator
 // CHECK:    llvm.return
 
@@ -388,7 +386,6 @@ llvm.func @_QPsimple_reduction(%arg0: !llvm.ptr {fir.bindc_name = "y"}) {
         llvm.store %14, %prv : i32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -558,7 +555,6 @@ func.func @omp_distribute(%arg0 : index) -> () {
     omp.loop_nest (%iv) : index = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -586,7 +582,6 @@ func.func @omp_ordered(%arg0 : index) -> () {
       omp.ordered depend_vec(%arg0 : index) {doacross_num_loops = 1 : i64}
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -607,7 +602,6 @@ func.func @omp_taskloop(%arg0: index, %arg1 : memref<i32>) {
         "test.payload"(%iv) : (index) -> ()
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
diff --git a/mlir/test/Conversion/SCFToOpenMP/reductions.mlir b/mlir/test/Conversion/SCFToOpenMP/reductions.mlir
index fc6d56559c2618125050b2bcbf61dae87488c903..8e2343df488773e80964a1dccfaa912b6fb9bab3 100644
--- a/mlir/test/Conversion/SCFToOpenMP/reductions.mlir
+++ b/mlir/test/Conversion/SCFToOpenMP/reductions.mlir
@@ -44,7 +44,6 @@ func.func @reduction1(%arg0 : index, %arg1 : index, %arg2 : index,
     }
     // CHECK: omp.yield
   }
-  // CHECK:   omp.terminator
   // CHECK: omp.terminator
   // CHECK: llvm.load %[[BUF]]
   return
@@ -240,7 +239,6 @@ func.func @reduction4(%arg0 : index, %arg1 : index, %arg2 : index,
     }
     // CHECK: omp.yield
   }
-  // CHECK:   omp.terminator
   // CHECK: omp.terminator
   // CHECK: %[[RES1:.*]] = llvm.load %[[BUF1]] : !llvm.ptr -> f32
   // CHECK: %[[RES2:.*]] = llvm.load %[[BUF2]] : !llvm.ptr -> i64
diff --git a/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir b/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
index b2f19d294cb5fe73e8b83f0ef5920383698f262b..a722acbf2c34710b17a67a6c64ad4c67c991a129 100644
--- a/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
+++ b/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
@@ -14,7 +14,6 @@ func.func @parallel(%arg0: index, %arg1: index, %arg2: index,
     // CHECK:   omp.yield
     // CHECK: }
   }
-  // CHECK:     omp.terminator
   // CHECK:   }
   // CHECK:   omp.terminator
   // CHECK: }
@@ -41,10 +40,8 @@ func.func @nested_loops(%arg0: index, %arg1: index, %arg2: index,
     }
     // CHECK:     omp.yield
     // CHECK:   }
-    // CHECK:   omp.terminator
     // CHECK: }
   }
-  // CHECK:     omp.terminator
   // CHECK:   }
   // CHECK:   omp.terminator
   // CHECK: }
@@ -65,7 +62,6 @@ func.func @adjacent_loops(%arg0: index, %arg1: index, %arg2: index,
     // CHECK:   omp.yield
     // CHECK: }
   }
-  // CHECK:     omp.terminator
   // CHECK:   }
   // CHECK:   omp.terminator
   // CHECK: }
@@ -81,7 +77,6 @@ func.func @adjacent_loops(%arg0: index, %arg1: index, %arg2: index,
     // CHECK:   omp.yield
     // CHECK: }
   }
-  // CHECK:     omp.terminator
   // CHECK:   }
   // CHECK:   omp.terminator
   // CHECK: }
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-mask-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-mask-to-llvm.mlir
index d644088080fdefc05487d0f72e3aa697dd057014..82351eb7c98a43af652e0fd2de179a6cecfdd020 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-mask-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-mask-to-llvm.mlir
@@ -46,33 +46,3 @@ func.func @genbool_var_1d_scalable(%arg0: index) -> vector<[11]xi1> {
   %0 = vector.create_mask %arg0 : vector<[11]xi1>
   return %0 : vector<[11]xi1>
 }
-
-// CMP32-LABEL: @transfer_read_1d
-// CMP32: %[[MEM:.*]]: memref<?xf32>, %[[OFF:.*]]: index) -> vector<16xf32> {
-// CMP32: %[[D:.*]] = memref.dim %[[MEM]], %{{.*}} : memref<?xf32>
-// CMP32: %[[S:.*]] = arith.subi %[[D]], %[[OFF]] : index
-// CMP32: %[[C:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]> : vector<16xi32>
-// CMP32: %[[B:.*]] = arith.index_cast %[[S]] : index to i32
-// CMP32: %[[B0:.*]] = llvm.insertelement %[[B]], %{{.*}} : vector<16xi32>
-// CMP32: %[[BV:.*]] = llvm.shufflevector %[[B0]], {{.*}} : vector<16xi32>
-// CMP32: %[[M:.*]] = arith.cmpi slt, %[[C]], %[[BV]] : vector<16xi32>
-// CMP32: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %[[M]], %{{.*}}
-// CMP32: return %[[L]] : vector<16xf32>
-
-// CMP64-LABEL: @transfer_read_1d(
-// CMP64: %[[MEM:.*]]: memref<?xf32>, %[[OFF:.*]]: index) -> vector<16xf32> {
-// CMP64: %[[D:.*]] = memref.dim %[[MEM]], %{{.*}} : memref<?xf32>
-// CMP64: %[[S:.*]] = arith.subi %[[D]], %[[OFF]] : index
-// CMP64: %[[C:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]> : vector<16xi64>
-// CMP64: %[[B:.*]] = arith.index_cast %[[S]] : index to i64
-// CMP64: %[[B0:.*]] = llvm.insertelement %[[B]], %{{.*}} : vector<16xi64>
-// CMP64: %[[BV:.*]] = llvm.shufflevector %[[B0]], {{.*}} : vector<16xi64>
-// CMP64: %[[M:.*]] = arith.cmpi slt, %[[C]], %[[BV]] : vector<16xi64>
-// CMP64: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %[[M]], %{{.*}}
-// CMP64: return %[[L]] : vector<16xf32>
-
-func.func @transfer_read_1d(%A : memref<?xf32>, %i: index) -> vector<16xf32> {
-  %d = arith.constant -1.0: f32
-  %f = vector.transfer_read %A[%i], %d {permutation_map = affine_map<(d0) -> (d0)>} : memref<?xf32>, vector<16xf32>
-  return %f : vector<16xf32>
-}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index ae1d6fe8bd1672d06e4f19e8ce8903a48877ed10..eb6da71b063273dbd4709e1b252ff8dfb4773c6e 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -2001,6 +2001,37 @@ func.func @vector_fma(%a: vector<8xf32>, %b: vector<2x4xf32>, %c: vector<1x1x1xf
   return %0, %1, %2, %3: vector<8xf32>, vector<2x4xf32>, vector<1x1x1xf32>, vector<f32>
 }
 
+func.func @vector_fma_scalable(%a: vector<[8]xf32>, %b: vector<2x[4]xf32>, %c: vector<1x1x[1]xf32>, %d: vector<f32>) -> (vector<[8]xf32>, vector<2x[4]xf32>, vector<1x1x[1]xf32>) {
+  // CHECK-LABEL: @vector_fma_scalable
+  //  CHECK-SAME: %[[A:.*]]: vector<[8]xf32>
+  //  CHECK-SAME: %[[B:.*]]: vector<2x[4]xf32>
+  //  CHECK-SAME: %[[C:.*]]: vector<1x1x[1]xf32>
+  //       CHECK: %[[BL:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<2x[4]xf32> to !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: llvm.intr.fmuladd
+  //  CHECK-SAME:   (vector<[8]xf32>, vector<[8]xf32>, vector<[8]xf32>) -> vector<[8]xf32>
+  %0 = vector.fma %a, %a, %a : vector<[8]xf32>
+
+  //       CHECK: %[[b00:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[b01:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[b02:.*]] = llvm.extractvalue %[[BL]][0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[B0:.*]] = llvm.intr.fmuladd(%[[b00]], %[[b01]], %[[b02]]) :
+  //  CHECK-SAME: (vector<[4]xf32>, vector<[4]xf32>, vector<[4]xf32>) -> vector<[4]xf32>
+  //       CHECK: llvm.insertvalue %[[B0]], {{.*}}[0] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[b10:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[b11:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[b12:.*]] = llvm.extractvalue %[[BL]][1] : !llvm.array<2 x vector<[4]xf32>>
+  //       CHECK: %[[B1:.*]] = llvm.intr.fmuladd(%[[b10]], %[[b11]], %[[b12]]) :
+  //  CHECK-SAME: (vector<[4]xf32>, vector<[4]xf32>, vector<[4]xf32>) -> vector<[4]xf32>
+  //       CHECK: llvm.insertvalue %[[B1]], {{.*}}[1] : !llvm.array<2 x vector<[4]xf32>>
+  %1 = vector.fma %b, %b, %b : vector<2x[4]xf32>
+
+  //       CHECK: %[[C0:.*]] = llvm.intr.fmuladd
+  //  CHECK-SAME:   (vector<[1]xf32>, vector<[1]xf32>, vector<[1]xf32>) -> vector<[1]xf32>
+  %2 = vector.fma %c, %c, %c : vector<1x1x[1]xf32>
+
+  return %0, %1, %2: vector<[8]xf32>, vector<2x[4]xf32>, vector<1x1x[1]xf32>
+}
+
 // -----
 
 func.func @reduce_0d_f32(%arg0: vector<f32>) -> f32 {
@@ -2028,6 +2059,17 @@ func.func @reduce_f16(%arg0: vector<16xf16>) -> f16 {
 // CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f16, vector<16xf16>) -> f16
 //      CHECK: return %[[V]] : f16
 
+func.func @reduce_f16_scalable(%arg0: vector<[16]xf16>) -> f16 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xf16> into f16
+  return %0 : f16
+}
+// CHECK-LABEL: @reduce_f16_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf16>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f16) : f16
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f16, vector<[16]xf16>) -> f16
+//      CHECK: return %[[V]] : f16
+
 // -----
 
 func.func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
@@ -2041,6 +2083,17 @@ func.func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
 // CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<16xf32>) -> f32
 //      CHECK: return %[[V]] : f32
 
+func.func @reduce_f32_scalable(%arg0: vector<[16]xf32>) -> f32 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f32, vector<[16]xf32>) -> f32
+//      CHECK: return %[[V]] : f32
+
 // -----
 
 func.func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
@@ -2054,6 +2107,17 @@ func.func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
 // CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f64, vector<16xf64>) -> f64
 //      CHECK: return %[[V]] : f64
 
+func.func @reduce_f64_scalable(%arg0: vector<[16]xf64>) -> f64 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xf64> into f64
+  return %0 : f64
+}
+// CHECK-LABEL: @reduce_f64_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf64>)
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : f64
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.fadd"(%[[C]], %[[A]])
+// CHECK-SAME: <{fastmathFlags = #llvm.fastmath<none>}> : (f64, vector<[16]xf64>) -> f64
+//      CHECK: return %[[V]] : f64
+
 // -----
 
 func.func @reduce_i8(%arg0: vector<16xi8>) -> i8 {
@@ -2065,6 +2129,15 @@ func.func @reduce_i8(%arg0: vector<16xi8>) -> i8 {
 //      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
 //      CHECK: return %[[V]] : i8
 
+func.func @reduce_i8_scalable(%arg0: vector<[16]xi8>) -> i8 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xi8> into i8
+  return %0 : i8
+}
+// CHECK-LABEL: @reduce_i8_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xi8>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i8
+
 // -----
 
 func.func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2076,6 +2149,15 @@ func.func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
 //      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
 //      CHECK: return %[[V]] : i32
 
+func.func @reduce_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_i32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2088,6 +2170,16 @@ func.func @reduce_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.add %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <add>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.add %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_mul_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2099,6 +2191,15 @@ func.func @reduce_mul_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_mul_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <mul>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_mul_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_mul_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2111,6 +2212,16 @@ func.func @reduce_mul_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.mul %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_mul_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <mul>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_mul_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.mul"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.mul %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_fmaximum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
@@ -2123,6 +2234,16 @@ func.func @reduce_fmaximum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.maximum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+func.func @reduce_fmaximum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <maximumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmaximum_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmaximum(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.maximum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
 // -----
 
 func.func @reduce_fminimum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
@@ -2135,6 +2256,16 @@ func.func @reduce_fminimum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.minimum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+func.func @reduce_fminimum_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <minimumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fminimum_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fminimum(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.minimum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
 // -----
 
 func.func @reduce_fmax_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
@@ -2147,6 +2278,16 @@ func.func @reduce_fmax_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.maxnum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+func.func @reduce_fmax_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <maxnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmax_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmax(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.maxnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
 // -----
 
 func.func @reduce_fmin_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
@@ -2159,6 +2300,16 @@ func.func @reduce_fmin_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
 //      CHECK: %[[R:.*]] = llvm.intr.minnum(%[[V]], %[[B]]) : (f32, f32) -> f32
 //      CHECK: return %[[R]] : f32
 
+func.func @reduce_fmin_f32_scalable(%arg0: vector<[16]xf32>, %arg1: f32) -> f32 {
+  %0 = vector.reduction <minnumf>, %arg0, %arg1 : vector<[16]xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmin_f32_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xf32>, %[[B:.*]]: f32)
+//      CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmin(%[[A]]) : (vector<[16]xf32>) -> f32
+//      CHECK: %[[R:.*]] = llvm.intr.minnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+//      CHECK: return %[[R]] : f32
+
 // -----
 
 func.func @reduce_minui_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2170,6 +2321,15 @@ func.func @reduce_minui_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_minui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <minui>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minui_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_minui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2183,6 +2343,17 @@ func.func @reduce_minui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_minui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <minui>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minui_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umin"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "ule" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_maxui_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2194,6 +2365,15 @@ func.func @reduce_maxui_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_maxui_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <maxui>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxui_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_maxui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2207,6 +2387,17 @@ func.func @reduce_maxui_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_maxui_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <maxui>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxui_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.umax"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "uge" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_minsi_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2218,6 +2409,15 @@ func.func @reduce_minsi_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_minsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <minsi>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minsi_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_minsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2231,6 +2431,17 @@ func.func @reduce_minsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_minsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <minsi>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_minsi_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smin"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "sle" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_maxsi_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2242,6 +2453,15 @@ func.func @reduce_maxsi_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_maxsi_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <maxsi>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxsi_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_maxsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2255,6 +2475,17 @@ func.func @reduce_maxsi_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_maxsi_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <maxsi>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_maxsi_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.smax"(%[[A]])
+//       CHECK: %[[S:.*]] = llvm.icmp "sge" %[[ACC]], %[[R]]
+//       CHECK: %[[V:.*]] = llvm.select %[[S]], %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_and_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2266,6 +2497,15 @@ func.func @reduce_and_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_and_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <and>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_and_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_and_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2278,6 +2518,16 @@ func.func @reduce_and_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.and %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_and_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <and>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_and_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.and"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.and %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_or_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2289,6 +2539,15 @@ func.func @reduce_or_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_or_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <or>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_or_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_or_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2301,6 +2560,16 @@ func.func @reduce_or_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.or %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_or_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <or>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_or_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.or"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.or %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_xor_i32(%arg0: vector<16xi32>) -> i32 {
@@ -2312,6 +2581,15 @@ func.func @reduce_xor_i32(%arg0: vector<16xi32>) -> i32 {
 //       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_xor_i32_scalable(%arg0: vector<[16]xi32>) -> i32 {
+  %0 = vector.reduction <xor>, %arg0 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_xor_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>)
+//       CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_xor_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
@@ -2324,6 +2602,16 @@ func.func @reduce_xor_acc_i32(%arg0: vector<16xi32>, %arg1 : i32) -> i32 {
 //       CHECK: %[[V:.*]] = llvm.xor %[[ACC]], %[[R]]
 //       CHECK: return %[[V]] : i32
 
+func.func @reduce_xor_acc_i32_scalable(%arg0: vector<[16]xi32>, %arg1 : i32) -> i32 {
+  %0 = vector.reduction <xor>, %arg0, %arg1 : vector<[16]xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: @reduce_xor_acc_i32_scalable(
+//  CHECK-SAME: %[[A:.*]]: vector<[16]xi32>, %[[ACC:.*]]: i32)
+//       CHECK: %[[R:.*]] = "llvm.intr.vector.reduce.xor"(%[[A]])
+//       CHECK: %[[V:.*]] = llvm.xor %[[ACC]], %[[R]]
+//       CHECK: return %[[V]] : i32
+
 // -----
 
 func.func @reduce_i64(%arg0: vector<16xi64>) -> i64 {
@@ -2335,6 +2623,15 @@ func.func @reduce_i64(%arg0: vector<16xi64>) -> i64 {
 //      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
 //      CHECK: return %[[V]] : i64
 
+func.func @reduce_i64_scalable(%arg0: vector<[16]xi64>) -> i64 {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xi64> into i64
+  return %0 : i64
+}
+// CHECK-LABEL: @reduce_i64_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xi64>)
+//      CHECK: %[[V:.*]] = "llvm.intr.vector.reduce.add"(%[[A]])
+//      CHECK: return %[[V]] : i64
+
 // -----
 
 func.func @reduce_index(%arg0: vector<16xindex>) -> index {
@@ -2348,6 +2645,17 @@ func.func @reduce_index(%arg0: vector<16xindex>) -> index {
 //      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
 //      CHECK: return %[[T2]] : index
 
+func.func @reduce_index_scalable(%arg0: vector<[16]xindex>) -> index {
+  %0 = vector.reduction <add>, %arg0 : vector<[16]xindex> into index
+  return %0 : index
+}
+// CHECK-LABEL: @reduce_index_scalable(
+// CHECK-SAME: %[[A:.*]]: vector<[16]xindex>)
+//      CHECK: %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<[16]xindex> to vector<[16]xi64>
+//      CHECK: %[[T1:.*]] = "llvm.intr.vector.reduce.add"(%[[T0]])
+//      CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T1]] : i64 to index
+//      CHECK: return %[[T2]] : index
+
 //                          4x16                16x3               4x3
 // -----
 
@@ -2377,224 +2685,6 @@ func.func @matrix_ops_index(%A: vector<64xindex>, %B: vector<48xindex>) -> vecto
 
 // -----
 
-func.func @transfer_read_1d(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
-  %f7 = arith.constant 7.0: f32
-  %f = vector.transfer_read %A[%base], %f7
-      {permutation_map = affine_map<(d0) -> (d0)>} :
-    memref<?xf32>, vector<17xf32>
-  vector.transfer_write %f, %A[%base]
-      {permutation_map = affine_map<(d0) -> (d0)>} :
-    vector<17xf32>, memref<?xf32>
-  return %f: vector<17xf32>
-}
-// CHECK-LABEL: func @transfer_read_1d
-//  CHECK-SAME: %[[MEM:.*]]: memref<?xf32>,
-//  CHECK-SAME: %[[BASE:.*]]: index) -> vector<17xf32>
-//       CHECK: %[[C7:.*]] = arith.constant 7.0
-//
-// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
-//       CHECK: %[[C0:.*]] = arith.constant 0 : index
-//       CHECK: %[[DIM:.*]] = memref.dim %[[MEM]], %[[C0]] : memref<?xf32>
-//       CHECK: %[[BOUND:.*]] = arith.subi %[[DIM]],  %[[BASE]] : index
-//
-// 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
-//       CHECK: %[[linearIndex:.*]] = arith.constant dense
-//  CHECK-SAME: <[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> :
-//  CHECK-SAME: vector<17xi32>
-//
-// 3. Create bound vector to compute in-bound mask:
-//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
-//       CHECK: %[[btrunc:.*]] = arith.index_cast %[[BOUND]] : index to i32
-//       CHECK: %[[boundVecInsert:.*]] = llvm.insertelement %[[btrunc]]
-//       CHECK: %[[boundVect:.*]] = llvm.shufflevector %[[boundVecInsert]]
-//       CHECK: %[[mask:.*]] = arith.cmpi slt, %[[linearIndex]], %[[boundVect]]
-//  CHECK-SAME: : vector<17xi32>
-//
-// 4. Create pass-through vector.
-//       CHECK: %[[PASS_THROUGH:.*]] = arith.constant dense<7.{{.*}}> : vector<17xf32>
-//
-// 5. Bitcast to vector form.
-//       CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}} :
-//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
-//
-// 6. Rewrite as a masked read.
-//       CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[gep]], %[[mask]],
-//  CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} :
-//  CHECK-SAME: -> vector<17xf32>
-//
-// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
-//       CHECK: %[[C0_b:.*]] = arith.constant 0 : index
-//       CHECK: %[[DIM_b:.*]] = memref.dim %[[MEM]], %[[C0_b]] : memref<?xf32>
-//       CHECK: %[[BOUND_b:.*]] = arith.subi %[[DIM_b]], %[[BASE]] : index
-//
-// 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
-//       CHECK: %[[linearIndex_b:.*]] = arith.constant dense
-//  CHECK-SAME: <[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> :
-//  CHECK-SAME: vector<17xi32>
-//
-// 3. Create bound vector to compute in-bound mask:
-//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
-//       CHECK: %[[btrunc_b:.*]] = arith.index_cast %[[BOUND_b]] : index to i32
-//       CHECK: %[[boundVecInsert_b:.*]] = llvm.insertelement %[[btrunc_b]]
-//       CHECK: %[[boundVect_b:.*]] = llvm.shufflevector %[[boundVecInsert_b]]
-//       CHECK: %[[mask_b:.*]] = arith.cmpi slt, %[[linearIndex_b]],
-//  CHECK-SAME: %[[boundVect_b]] : vector<17xi32>
-//
-// 4. Bitcast to vector form.
-//       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
-//
-// 5. Rewrite as a masked write.
-//       CHECK: llvm.intr.masked.store %[[loaded]], %[[gep_b]], %[[mask_b]]
-//  CHECK-SAME: {alignment = 4 : i32} :
-//  CHECK-SAME: vector<17xf32>, vector<17xi1> into !llvm.ptr
-
-// -----
-
-func.func @transfer_read_index_1d(%A : memref<?xindex>, %base: index) -> vector<17xindex> {
-  %f7 = arith.constant 7: index
-  %f = vector.transfer_read %A[%base], %f7
-      {permutation_map = affine_map<(d0) -> (d0)>} :
-    memref<?xindex>, vector<17xindex>
-  vector.transfer_write %f, %A[%base]
-      {permutation_map = affine_map<(d0) -> (d0)>} :
-    vector<17xindex>, memref<?xindex>
-  return %f: vector<17xindex>
-}
-// CHECK-LABEL: func @transfer_read_index_1d
-//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xindex>
-//       CHECK: %[[SPLAT:.*]] = arith.constant dense<7> : vector<17xindex>
-//       CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[SPLAT]] : vector<17xindex> to vector<17xi64>
-
-//       CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} :
-//  CHECK-SAME: (!llvm.ptr, vector<17xi1>, vector<17xi64>) -> vector<17xi64>
-
-//       CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i32} :
-//  CHECK-SAME: vector<17xi64>, vector<17xi1> into !llvm.ptr
-
-// -----
-
-func.func @transfer_read_2d_to_1d(%A : memref<?x?xf32>, %base0: index, %base1: index) -> vector<17xf32> {
-  %f7 = arith.constant 7.0: f32
-  %f = vector.transfer_read %A[%base0, %base1], %f7
-      {permutation_map = affine_map<(d0, d1) -> (d1)>} :
-    memref<?x?xf32>, vector<17xf32>
-  return %f: vector<17xf32>
-}
-// CHECK-LABEL: func @transfer_read_2d_to_1d
-//  CHECK-SAME: %[[BASE_0:[a-zA-Z0-9]*]]: index, %[[BASE_1:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
-//       CHECK: %[[c1:.*]] = arith.constant 1 : index
-//       CHECK: %[[DIM:.*]] = memref.dim %{{.*}}, %[[c1]] : memref<?x?xf32>
-//
-// Compute the in-bound index (dim - offset)
-//       CHECK: %[[BOUND:.*]] = arith.subi %[[DIM]], %[[BASE_1]] : index
-//
-// Create a vector with linear indices [ 0 .. vector_length - 1 ].
-//       CHECK: %[[linearIndex:.*]] = arith.constant dense
-//  CHECK-SAME: <[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> :
-//  CHECK-SAME: vector<17xi32>
-//
-// Create bound vector to compute in-bound mask:
-//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
-//       CHECK: %[[btrunc:.*]] = arith.index_cast %[[BOUND]] : index to i32
-//       CHECK: %[[boundVecInsert:.*]] = llvm.insertelement %[[btrunc]]
-//       CHECK: %[[boundVect:.*]] = llvm.shufflevector %[[boundVecInsert]]
-//       CHECK: %[[mask:.*]] = arith.cmpi slt, %[[linearIndex]], %[[boundVect]]
-
-// -----
-
-func.func @transfer_read_1d_non_zero_addrspace(%A : memref<?xf32, 3>, %base: index) -> vector<17xf32> {
-  %f7 = arith.constant 7.0: f32
-  %f = vector.transfer_read %A[%base], %f7
-      {permutation_map = affine_map<(d0) -> (d0)>} :
-    memref<?xf32, 3>, vector<17xf32>
-  vector.transfer_write %f, %A[%base]
-      {permutation_map = affine_map<(d0) -> (d0)>} :
-    vector<17xf32>, memref<?xf32, 3>
-  return %f: vector<17xf32>
-}
-// CHECK-LABEL: func @transfer_read_1d_non_zero_addrspace
-//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
-//
-// 1. Check address space for GEP is correct.
-//       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
-//
-// 2. Check address space of the memref is correct.
-//       CHECK: %[[c0:.*]] = arith.constant 0 : index
-//       CHECK: %[[DIM:.*]] = memref.dim %{{.*}}, %[[c0]] : memref<?xf32, 3>
-//
-// 3. Check address space for GEP is correct.
-//       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
-
-// -----
-
-func.func @transfer_read_1d_inbounds(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
-  %f7 = arith.constant 7.0: f32
-  %f = vector.transfer_read %A[%base], %f7 {in_bounds = [true]} :
-    memref<?xf32>, vector<17xf32>
-  return %f: vector<17xf32>
-}
-// CHECK-LABEL: func @transfer_read_1d_inbounds
-//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
-//
-// 1. Bitcast to vector form.
-//       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
-//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
-//
-// 2. Rewrite as a load.
-//       CHECK: %[[loaded:.*]] = llvm.load %[[gep]] {alignment = 4 : i64} : !llvm.ptr -> vector<17xf32>
-
-// -----
-
-// CHECK-LABEL: func @transfer_read_1d_mask
-// CHECK: %[[mask1:.*]] = arith.constant dense<[false, false, true, false, true]>
-// CHECK: %[[cmpi:.*]] = arith.cmpi slt
-// CHECK: %[[mask2:.*]] = arith.andi %[[cmpi]], %[[mask1]]
-// CHECK: %[[r:.*]] = llvm.intr.masked.load %{{.*}}, %[[mask2]]
-// CHECK: return %[[r]]
-func.func @transfer_read_1d_mask(%A : memref<?xf32>, %base : index) -> vector<5xf32> {
-  %m = arith.constant dense<[0, 0, 1, 0, 1]> : vector<5xi1>
-  %f7 = arith.constant 7.0: f32
-  %f = vector.transfer_read %A[%base], %f7, %m : memref<?xf32>, vector<5xf32>
-  return %f: vector<5xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transfer_read_1d_scalable_mask
-// CHECK: %[[passtru:.*]] = arith.constant dense<0.000000e+00> : vector<[4]xf32>
-// CHECK: %[[r:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %[[passtru]] {alignment = 4 : i32} : (!llvm.ptr, vector<[4]xi1>, vector<[4]xf32>) -> vector<[4]xf32>
-// CHECK: return %[[r]] : vector<[4]xf32>
-func.func @transfer_read_1d_scalable_mask(%arg0: memref<1x?xf32>, %mask: vector<[4]xi1>) -> vector<[4]xf32> {
-  %c0 = arith.constant 0 : index
-  %pad = arith.constant 0.0 : f32
-  %vec = vector.transfer_read %arg0[%c0, %c0], %pad, %mask {in_bounds = [true]} : memref<1x?xf32>, vector<[4]xf32>
-  return %vec : vector<[4]xf32>
-}
-
-// -----
-// CHECK-LABEL: func @transfer_write_1d_scalable_mask
-// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : vector<[4]xf32>, vector<[4]xi1> into !llvm.ptr
-func.func @transfer_write_1d_scalable_mask(%arg0: memref<1x?xf32>, %vec: vector<[4]xf32>, %mask: vector<[4]xi1>) {
-  %c0 = arith.constant 0 : index
-  vector.transfer_write %vec, %arg0[%c0, %c0], %mask {in_bounds = [true]} : vector<[4]xf32>, memref<1x?xf32>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: func @transfer_write_tensor
-//       CHECK:   vector.transfer_write
-func.func @transfer_write_tensor(%arg0: vector<4xf32>,%arg1: tensor<?xf32>) -> tensor<?xf32> {
-  %c0 = arith.constant 0 : index
-  %0 = vector.transfer_write %arg0, %arg1[%c0] : vector<4xf32>, tensor<?xf32>
-  return %0 : tensor<?xf32>
-}
-
-// -----
-
 func.func @genbool_0d_f() -> vector<i1> {
   %0 = vector.constant_mask [0] : vector<i1>
   return %0 : vector<i1>
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..9cfd6885fba95870a435e24d386d0ef52e78e4e1
--- /dev/null
+++ b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
@@ -0,0 +1,378 @@
+// RUN: mlir-opt %s -convert-vector-to-llvm -split-input-file | FileCheck -D$IDX_TYPE=i32 %s
+// RUN: mlir-opt %s --convert-vector-to-llvm='force-32bit-vector-indices=0' | FileCheck  -D$IDX_TYPE=i64 %s
+
+func.func @transfer_read_write_1d(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    memref<?xf32>, vector<17xf32>
+  vector.transfer_write %f, %A[%base]
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    vector<17xf32>, memref<?xf32>
+  return %f: vector<17xf32>
+}
+// CHECK-LABEL: func @transfer_read_write_1d
+//  CHECK-SAME: %[[MEM:.*]]: memref<?xf32>,
+//  CHECK-SAME: %[[BASE:.*]]: index) -> vector<17xf32>
+//       CHECK: %[[C7:.*]] = arith.constant 7.0
+//
+// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
+//       CHECK: %[[C0:.*]] = arith.constant 0 : index
+//       CHECK: %[[DIM:.*]] = memref.dim %[[MEM]], %[[C0]] : memref<?xf32>
+//       CHECK: %[[BOUND:.*]] = arith.subi %[[DIM]],  %[[BASE]] : index
+//
+// 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
+//       CHECK: %[[linearIndex:.*]] = arith.constant dense
+//  CHECK-SAME: <[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> : vector<17x[[$IDX_TYPE]]>
+//
+// 3. Create bound vector to compute in-bound mask:
+//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
+//       CHECK: %[[btrunc:.*]] = arith.index_cast %[[BOUND]] :
+//  CMP32-SAME: index to i32
+//  CMP64-SAME: index to i64
+//       CHECK: %[[boundVecInsert:.*]] = llvm.insertelement %[[btrunc]]
+//       CHECK: %[[boundVect:.*]] = llvm.shufflevector %[[boundVecInsert]]
+//       CHECK: %[[mask:.*]] = arith.cmpi slt, %[[linearIndex]], %[[boundVect]] : vector<17x[[$IDX_TYPE]]>
+//  CMP64-SAME: : vector<17xi64>
+//
+// 4. Create pass-through vector.
+//       CHECK: %[[PASS_THROUGH:.*]] = arith.constant dense<7.{{.*}}> : vector<17xf32>
+//
+// 5. Bitcast to vector form.
+//       CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}} :
+//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
+//
+// 6. Rewrite as a masked read.
+//       CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[gep]], %[[mask]],
+//  CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} :
+//  CHECK-SAME: -> vector<17xf32>
+//
+// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
+//       CHECK: %[[C0_b:.*]] = arith.constant 0 : index
+//       CHECK: %[[DIM_b:.*]] = memref.dim %[[MEM]], %[[C0_b]] : memref<?xf32>
+//       CHECK: %[[BOUND_b:.*]] = arith.subi %[[DIM_b]], %[[BASE]] : index
+//
+// 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
+//       CHECK: %[[linearIndex_b:.*]] = arith.constant dense
+//  CHECK-SAME: <[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> : vector<17x[[$IDX_TYPE]]>
+//
+// 3. Create bound vector to compute in-bound mask:
+//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
+//       CHECK: %[[btrunc_b:.*]] = arith.index_cast %[[BOUND_b]]
+//  CMP32-SAME: index to i32
+//       CHECK: %[[boundVecInsert_b:.*]] = llvm.insertelement %[[btrunc_b]]
+//       CHECK: %[[boundVect_b:.*]] = llvm.shufflevector %[[boundVecInsert_b]]
+//       CHECK: %[[mask_b:.*]] = arith.cmpi slt, %[[linearIndex_b]],
+//  CHECK-SAME: %[[boundVect_b]] : vector<17x[[$IDX_TYPE]]>
+//
+// 4. Bitcast to vector form.
+//       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
+//
+// 5. Rewrite as a masked write.
+//       CHECK: llvm.intr.masked.store %[[loaded]], %[[gep_b]], %[[mask_b]]
+//  CHECK-SAME: {alignment = 4 : i32} :
+//  CHECK-SAME: vector<17xf32>, vector<17xi1> into !llvm.ptr
+
+func.func @transfer_read_write_1d_scalable(%A : memref<?xf32>, %base: index) -> vector<[17]xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    memref<?xf32>, vector<[17]xf32>
+  vector.transfer_write %f, %A[%base]
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    vector<[17]xf32>, memref<?xf32>
+  return %f: vector<[17]xf32>
+}
+// CHECK-LABEL: func @transfer_read_write_1d_scalable
+//  CHECK-SAME: %[[MEM:.*]]: memref<?xf32>,
+//  CHECK-SAME: %[[BASE:.*]]: index) -> vector<[17]xf32>
+//       CHECK: %[[C7:.*]] = arith.constant 7.0
+//
+// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
+//       CHECK: %[[C0:.*]] = arith.constant 0 : index
+//       CHECK: %[[DIM:.*]] = memref.dim %[[MEM]], %[[C0]] : memref<?xf32>
+//       CHECK: %[[BOUND:.*]] = arith.subi %[[DIM]],  %[[BASE]] : index
+//
+// 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
+//       CHECK: %[[linearIndex:.*]] = llvm.intr.stepvector : vector<[17]x[[$IDX_TYPE]]>
+//
+// 3. Create bound vector to compute in-bound mask:
+//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
+//       CHECK: %[[btrunc:.*]] = arith.index_cast %[[BOUND]] : index to [[$IDX_TYPE]]
+//       CHECK: %[[boundVecInsert:.*]] = llvm.insertelement %[[btrunc]]
+//       CHECK: %[[boundVect:.*]] = llvm.shufflevector %[[boundVecInsert]]
+//       CHECK: %[[mask:.*]] = arith.cmpi slt, %[[linearIndex]], %[[boundVect]]
+//  CHECK-SAME: : vector<[17]x[[$IDX_TYPE]]>
+//
+// 4. Create pass-through vector.
+//       CHECK: %[[PASS_THROUGH:.*]] = arith.constant dense<7.{{.*}}> : vector<[17]xf32>
+//
+// 5. Bitcast to vector form.
+//       CHECK: %[[gep:.*]] = llvm.getelementptr %{{.*}} :
+//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
+//
+// 6. Rewrite as a masked read.
+//       CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[gep]], %[[mask]],
+//  CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} :
+//  CHECK-SAME: -> vector<[17]xf32>
+//
+// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
+//       CHECK: %[[C0_b:.*]] = arith.constant 0 : index
+//       CHECK: %[[DIM_b:.*]] = memref.dim %[[MEM]], %[[C0_b]] : memref<?xf32>
+//       CHECK: %[[BOUND_b:.*]] = arith.subi %[[DIM_b]], %[[BASE]] : index
+//
+// 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
+//       CHECK: %[[linearIndex_b:.*]] = llvm.intr.stepvector : vector<[17]x[[$IDX_TYPE]]>
+//
+// 3. Create bound vector to compute in-bound mask:
+//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
+//       CHECK: %[[btrunc_b:.*]] = arith.index_cast %[[BOUND_b]] : index to [[$IDX_TYPE]]
+//       CHECK: %[[boundVecInsert_b:.*]] = llvm.insertelement %[[btrunc_b]]
+//       CHECK: %[[boundVect_b:.*]] = llvm.shufflevector %[[boundVecInsert_b]]
+//       CHECK: %[[mask_b:.*]] = arith.cmpi slt, %[[linearIndex_b]],
+//  CHECK-SAME: %[[boundVect_b]] : vector<[17]x[[$IDX_TYPE]]>
+//
+// 4. Bitcast to vector form.
+//       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
+//
+// 5. Rewrite as a masked write.
+//       CHECK: llvm.intr.masked.store %[[loaded]], %[[gep_b]], %[[mask_b]]
+//  CHECK-SAME: {alignment = 4 : i32} :
+//  CHECK-SAME: vector<[17]xf32>, vector<[17]xi1> into !llvm.ptr
+
+// -----
+
+func.func @transfer_read_write_index_1d(%A : memref<?xindex>, %base: index) -> vector<17xindex> {
+  %f7 = arith.constant 7: index
+  %f = vector.transfer_read %A[%base], %f7
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    memref<?xindex>, vector<17xindex>
+  vector.transfer_write %f, %A[%base]
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    vector<17xindex>, memref<?xindex>
+  return %f: vector<17xindex>
+}
+// CHECK-LABEL: func @transfer_read_write_index_1d
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xindex>
+//       CHECK: %[[SPLAT:.*]] = arith.constant dense<7> : vector<17xindex>
+//       CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[SPLAT]] : vector<17xindex> to vector<17xi64>
+
+//       CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+//  CHECK-SAME: (!llvm.ptr, vector<17xi1>, vector<17xi64>) -> vector<17xi64>
+
+//       CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+//  CHECK-SAME: vector<17xi64>, vector<17xi1> into !llvm.ptr
+
+func.func @transfer_read_write_index_1d_scalable(%A : memref<?xindex>, %base: index) -> vector<[17]xindex> {
+  %f7 = arith.constant 7: index
+  %f = vector.transfer_read %A[%base], %f7
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    memref<?xindex>, vector<[17]xindex>
+  vector.transfer_write %f, %A[%base]
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    vector<[17]xindex>, memref<?xindex>
+  return %f: vector<[17]xindex>
+}
+// CHECK-LABEL: func @transfer_read_write_index_1d
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<[17]xindex>
+//       CHECK: %[[SPLAT:.*]] = arith.constant dense<7> : vector<[17]xindex>
+//       CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[SPLAT]] : vector<[17]xindex> to vector<[17]xi64>
+
+//       CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+//  CHECK-SAME: (!llvm.ptr, vector<[17]xi1>, vector<[17]xi64>) -> vector<[17]xi64>
+
+//       CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+//  CHECK-SAME: vector<[17]xi64>, vector<[17]xi1> into !llvm.ptr
+
+// -----
+
+func.func @transfer_read_2d_to_1d(%A : memref<?x?xf32>, %base0: index, %base1: index) -> vector<17xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base0, %base1], %f7
+      {permutation_map = affine_map<(d0, d1) -> (d1)>} :
+    memref<?x?xf32>, vector<17xf32>
+  return %f: vector<17xf32>
+}
+// CHECK-LABEL: func @transfer_read_2d_to_1d
+//  CHECK-SAME: %[[BASE_0:[a-zA-Z0-9]*]]: index, %[[BASE_1:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
+//       CHECK: %[[c1:.*]] = arith.constant 1 : index
+//       CHECK: %[[DIM:.*]] = memref.dim %{{.*}}, %[[c1]] : memref<?x?xf32>
+//
+// Compute the in-bound index (dim - offset)
+//       CHECK: %[[BOUND:.*]] = arith.subi %[[DIM]], %[[BASE_1]] : index
+//
+// Create a vector with linear indices [ 0 .. vector_length - 1 ].
+//       CHECK: %[[linearIndex:.*]] = arith.constant dense
+//  CHECK-SAME: <[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]> :
+//  CHECK-SAME: vector<17x[[$IDX_TYPE]]>
+//
+// Create bound vector to compute in-bound mask:
+//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
+//       CHECK: %[[btrunc:.*]] = arith.index_cast %[[BOUND]] : index to [[$IDX_TYPE]]
+//       CHECK: %[[boundVecInsert:.*]] = llvm.insertelement %[[btrunc]]
+//       CHECK: %[[boundVect:.*]] = llvm.shufflevector %[[boundVecInsert]]
+//       CHECK: %[[mask:.*]] = arith.cmpi slt, %[[linearIndex]], %[[boundVect]]
+
+func.func @transfer_read_2d_to_1d_scalable(%A : memref<?x?xf32>, %base0: index, %base1: index) -> vector<[17]xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base0, %base1], %f7
+      {permutation_map = affine_map<(d0, d1) -> (d1)>} :
+    memref<?x?xf32>, vector<[17]xf32>
+  return %f: vector<[17]xf32>
+}
+// CHECK-LABEL: func @transfer_read_2d_to_1d
+//  CHECK-SAME: %[[BASE_0:[a-zA-Z0-9]*]]: index, %[[BASE_1:[a-zA-Z0-9]*]]: index) -> vector<[17]xf32>
+//       CHECK: %[[c1:.*]] = arith.constant 1 : index
+//       CHECK: %[[DIM:.*]] = memref.dim %{{.*}}, %[[c1]] : memref<?x?xf32>
+//
+// Compute the in-bound index (dim - offset)
+//       CHECK: %[[BOUND:.*]] = arith.subi %[[DIM]], %[[BASE_1]] : index
+//
+// Create a vector with linear indices [ 0 .. vector_length - 1 ].
+//       CHECK: %[[linearIndex:.*]] = llvm.intr.stepvector : vector<[17]x[[$IDX_TYPE]]>
+//
+// Create bound vector to compute in-bound mask:
+//    [ 0 .. vector_length - 1 ] < [ dim - offset .. dim - offset ]
+//       CHECK: %[[btrunc:.*]] = arith.index_cast %[[BOUND]] : index to [[$IDX_TYPE]]
+//       CHECK: %[[boundVecInsert:.*]] = llvm.insertelement %[[btrunc]]
+//       CHECK: %[[boundVect:.*]] = llvm.shufflevector %[[boundVecInsert]]
+//       CHECK: %[[mask:.*]] = arith.cmpi slt, %[[linearIndex]], %[[boundVect]]
+
+// -----
+
+func.func @transfer_read_write_1d_non_zero_addrspace(%A : memref<?xf32, 3>, %base: index) -> vector<17xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    memref<?xf32, 3>, vector<17xf32>
+  vector.transfer_write %f, %A[%base]
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    vector<17xf32>, memref<?xf32, 3>
+  return %f: vector<17xf32>
+}
+// CHECK-LABEL: func @transfer_read_write_1d_non_zero_addrspace
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
+//
+// 1. Check address space for GEP is correct.
+//       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
+//
+// 2. Check address space of the memref is correct.
+//       CHECK: %[[c0:.*]] = arith.constant 0 : index
+//       CHECK: %[[DIM:.*]] = memref.dim %{{.*}}, %[[c0]] : memref<?xf32, 3>
+//
+// 3. Check address space for GEP is correct.
+//       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
+
+func.func @transfer_read_write_1d_non_zero_addrspace_scalable(%A : memref<?xf32, 3>, %base: index) -> vector<[17]xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    memref<?xf32, 3>, vector<[17]xf32>
+  vector.transfer_write %f, %A[%base]
+      {permutation_map = affine_map<(d0) -> (d0)>} :
+    vector<[17]xf32>, memref<?xf32, 3>
+  return %f: vector<[17]xf32>
+}
+// CHECK-LABEL: func @transfer_read_write_1d_non_zero_addrspace_scalable
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<[17]xf32>
+//
+// 1. Check address space for GEP is correct.
+//       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
+//
+// 2. Check address space of the memref is correct.
+//       CHECK: %[[c0:.*]] = arith.constant 0 : index
+//       CHECK: %[[DIM:.*]] = memref.dim %{{.*}}, %[[c0]] : memref<?xf32, 3>
+//
+// 3. Check address space for GEP is correct.
+//       CHECK: %[[gep_b:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, f32
+
+// -----
+
+func.func @transfer_read_1d_inbounds(%A : memref<?xf32>, %base: index) -> vector<17xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7 {in_bounds = [true]} :
+    memref<?xf32>, vector<17xf32>
+  return %f: vector<17xf32>
+}
+// CHECK-LABEL: func @transfer_read_1d_inbounds
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
+//
+// 1. Bitcast to vector form.
+//       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
+//
+// 2. Rewrite as a load.
+//       CHECK: %[[loaded:.*]] = llvm.load %[[gep]] {alignment = 4 : i64} : !llvm.ptr -> vector<17xf32>
+
+func.func @transfer_read_1d_inbounds_scalable(%A : memref<?xf32>, %base: index) -> vector<[17]xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7 {in_bounds = [true]} :
+    memref<?xf32>, vector<[17]xf32>
+  return %f: vector<[17]xf32>
+}
+// CHECK-LABEL: func @transfer_read_1d_inbounds_scalable
+//  CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<[17]xf32>
+//
+// 1. Bitcast to vector form.
+//       CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+//  CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
+//
+// 2. Rewrite as a load.
+//       CHECK: %[[loaded:.*]] = llvm.load %[[gep]] {alignment = 4 : i64} : !llvm.ptr -> vector<[17]xf32>
+
+// -----
+
+// CHECK-LABEL: func @transfer_read_write_1d_mask
+// CHECK: %[[mask1:.*]] = arith.constant dense<[false, false, true, false, true]>
+// CHECK: %[[cmpi:.*]] = arith.cmpi slt
+// CHECK: %[[mask2:.*]] = arith.andi %[[cmpi]], %[[mask1]]
+// CHECK: %[[r:.*]] = llvm.intr.masked.load %{{.*}}, %[[mask2]]
+// CHECK: %[[cmpi_1:.*]] = arith.cmpi slt
+// CHECK: %[[mask3:.*]] = arith.andi %[[cmpi_1]], %[[mask1]]
+// CHECK: llvm.intr.masked.store %[[r]], %{{.*}}, %[[mask3]]
+// CHECK: return %[[r]]
+func.func @transfer_read_write_1d_mask(%A : memref<?xf32>, %base : index) -> vector<5xf32> {
+  %m = arith.constant dense<[0, 0, 1, 0, 1]> : vector<5xi1>
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7, %m : memref<?xf32>, vector<5xf32>
+  vector.transfer_write %f, %A[%base], %m : vector<5xf32>, memref<?xf32>
+  return %f: vector<5xf32>
+}
+
+// CHECK-LABEL: func @transfer_read_write_1d_mask_scalable
+// CHECK-SAME: %[[mask:[a-zA-Z0-9]*]]: vector<[5]xi1>
+// CHECK: %[[cmpi:.*]] = arith.cmpi slt
+// CHECK: %[[mask1:.*]] = arith.andi %[[cmpi]], %[[mask]]
+// CHECK: %[[r:.*]] = llvm.intr.masked.load %{{.*}}, %[[mask1]]
+// CHECK: %[[cmpi_1:.*]] = arith.cmpi slt
+// CHECK: %[[mask2:.*]] = arith.andi %[[cmpi_1]], %[[mask]]
+// CHECK: llvm.intr.masked.store %[[r]], %{{.*}}, %[[mask2]]
+// CHECK: return %[[r]]
+func.func @transfer_read_write_1d_mask_scalable(%A : memref<?xf32>, %base : index, %m : vector<[5]xi1>) -> vector<[5]xf32> {
+  %f7 = arith.constant 7.0: f32
+  %f = vector.transfer_read %A[%base], %f7, %m : memref<?xf32>, vector<[5]xf32>
+  vector.transfer_write %f, %A[%base], %m : vector<[5]xf32>, memref<?xf32>
+  return %f: vector<[5]xf32>
+}
+
+// -----
+
+// Can't lower xfer_read/xfer_write on tensors, but this shouldn't crash
+
+// CHECK-LABEL: func @transfer_read_write_tensor
+//       CHECK:   vector.transfer_read
+//       CHECK:   vector.transfer_write
+func.func @transfer_read_write_tensor(%A: tensor<?xf32>, %base : index) -> vector<4xf32> {
+  %f7 = arith.constant 7.0: f32
+  %c0 = arith.constant 0: index
+  %f = vector.transfer_read %A[%base], %f7 : tensor<?xf32>, vector<4xf32>
+  %w = vector.transfer_write %f, %A[%c0] : vector<4xf32>, tensor<?xf32>
+  "test.some_use"(%w) : (tensor<?xf32>) -> ()
+  return %f : vector<4xf32>
+}
diff --git a/mlir/test/Dialect/Affine/simplify-structures.mlir b/mlir/test/Dialect/Affine/simplify-structures.mlir
index 92d3d86bc93068f523313507c4138dafbcdb9df8..d1f34f20fa5dad5bd9f2b475439dd51e6f826560 100644
--- a/mlir/test/Dialect/Affine/simplify-structures.mlir
+++ b/mlir/test/Dialect/Affine/simplify-structures.mlir
@@ -308,10 +308,26 @@ func.func @semiaffine_ceildiv(%arg0: index, %arg1: index) -> index {
 }
 
 // Tests the simplification of a semi-affine expression with a nested ceildiv operation and further simplifications after performing ceildiv.
-// CHECK-LABEL: func @semiaffine_composite_floor
-func.func @semiaffine_composite_floor(%arg0: index, %arg1: index) -> index {
+// CHECK-LABEL: func @semiaffine_composite_ceildiv
+func.func @semiaffine_composite_ceildiv(%arg0: index, %arg1: index) -> index {
+  %a = affine.apply affine_map<(d0)[s0] ->((((s0 * 2) ceildiv 4) + s0 * 42) ceildiv s0)> (%arg0)[%arg1]
+  // CHECK:       %[[CST:.*]] = arith.constant 43
+  return %a : index
+}
+
+// Tests the do not simplification of a semi-affine expression with a nested ceildiv-mul-ceildiv operation.
+// CHECK-LABEL: func @semiaffine_composite_ceildiv
+func.func @semiaffine_composite_ceildiv_mul_ceildiv(%arg0: index, %arg1: index) -> index {
   %a = affine.apply affine_map<(d0)[s0] ->(((((s0 * 2) ceildiv 4) * 5) + s0 * 42) ceildiv s0)> (%arg0)[%arg1]
-  // CHECK:       %[[CST:.*]] = arith.constant 47
+  // CHECK-NOT:       arith.constant
+  return %a : index
+}
+
+// Tests the do not simplification of a semi-affine expression with a nested floordiv_mul_floordiv operation
+// CHECK-LABEL: func @semiaffine_composite_floordiv
+func.func @semiaffine_composite_floordiv_mul_floordiv(%arg0: index, %arg1: index) -> index {
+  %a = affine.apply affine_map<(d0)[s0] ->(((((s0 * 2) floordiv 4) * 5) + s0 * 42) floordiv s0)> (%arg0)[%arg1]
+  // CHECK-NOT:       arith.constant
   return %a : index
 }
 
diff --git a/mlir/test/Dialect/Arith/emulate-wide-int-unsupported.mlir b/mlir/test/Dialect/Arith/emulate-wide-int-unsupported.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..091dd15dcd8517c00ea2eea0a7fa2282a6669bb4
--- /dev/null
+++ b/mlir/test/Dialect/Arith/emulate-wide-int-unsupported.mlir
@@ -0,0 +1,43 @@
+// RUN: mlir-opt --arith-emulate-wide-int="widest-int-supported=32" \
+// RUN:   --split-input-file --verify-diagnostics %s
+
+// Make sure we do not crash on unsupported types.
+
+// Unsupported result type in `arith.extsi`.
+func.func @unsupported_result_integer(%arg0: i64) -> i64 {
+  // expected-error@+1 {{failed to legalize operation 'arith.extsi' that was explicitly marked illegal}}
+  %0 = arith.extsi %arg0: i64 to i128
+  %2 = arith.muli %0, %0 : i128
+  %3 = arith.trunci %2 : i128 to i64
+  return %3 : i64
+}
+
+// -----
+
+// Unsupported result type in `arith.extsi`.
+func.func @unsupported_result_vector(%arg0: vector<4xi64>) -> vector<4xi64> {
+  // expected-error@+1 {{failed to legalize operation 'arith.extsi' that was explicitly marked illegal}}
+  %0 = arith.extsi %arg0: vector<4xi64> to vector<4xi128>
+  %2 = arith.muli %0, %0 : vector<4xi128>
+  %3 = arith.trunci %2 : vector<4xi128> to vector<4xi64>
+  return %3 : vector<4xi64>
+}
+
+// -----
+
+// Unsupported function return type.
+// expected-error@+1 {{failed to legalize operation 'func.func' that was explicitly marked illegal}}
+func.func @unsupported_return_type(%arg0: vector<4xi64>) -> vector<4xi128> {
+  %0 = arith.extsi %arg0: vector<4xi64> to vector<4xi128>
+  return %0 : vector<4xi128>
+}
+
+// -----
+
+// Unsupported function argument type.
+// expected-error@+1 {{failed to legalize operation 'func.func' that was explicitly marked illegal}}
+func.func @unsupported_argument_type(%arg0: vector<4xi128>) -> vector<4xi64> {
+  %0 = arith.trunci %arg0: vector<4xi128> to vector<4xi64>
+  return %0 : vector<4xi64>
+}
+
diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir
index 8475ec6c3510db1e9760b088bf7db33d7dfb2376..dafb3bcef740f046c64a437c4d32016281d1647c 100644
--- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir
+++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir
@@ -162,7 +162,18 @@
  file = #file, line = 2, type = #int0>
 #var_expression = #llvm.di_global_variable_expression<var = #global_var,
  expr = <>>
-llvm.mlir.global common @block_() {dbg_expr = #var_expression} : i64
+#global_var1 = #llvm.di_global_variable<scope = #di_common_block, name = "b",
+ file = #file, line = 3, type = #int0>
+#var_expression1 = #llvm.di_global_variable_expression<var = #global_var1,
+ expr = <>>
+llvm.mlir.global @data() {dbg_exprs = [#var_expression, #var_expression1]} : i64
+
+// CHECK-DAG: llvm.mlir.global external @data() {{{.*}}dbg_exprs = [#[[EXP1:.*]], #[[EXP2:.*]]]} : i64
+// CHECK-DAG: #[[EXP1]] = #llvm.di_global_variable_expression<var = #[[GV1:.*]], expr = <>>
+// CHECK-DAG: #[[EXP2]] = #llvm.di_global_variable_expression<var = #[[GV2:.*]], expr = <>>
+// CHECK-DAG: #[[GV1]] = #llvm.di_global_variable<{{.*}}name = "a"{{.*}}>
+// CHECK-DAG: #[[GV2]] = #llvm.di_global_variable<{{.*}}name = "b"{{.*}}>
+
 
 // CHECK: llvm.func @addr(%[[ARG:.*]]: i64)
 llvm.func @addr(%arg: i64) {
diff --git a/mlir/test/Dialect/LLVMIR/global.mlir b/mlir/test/Dialect/LLVMIR/global.mlir
index 3fa7636d4dd6864c795a219eb604c5ff6fc1c2db..79d1cafabfbed3b2a46c7de0f751fc6cfee9ca6b 100644
--- a/mlir/test/Dialect/LLVMIR/global.mlir
+++ b/mlir/test/Dialect/LLVMIR/global.mlir
@@ -272,15 +272,15 @@ llvm.mlir.global @target_fail(0 : i64) : !llvm.target<"spirv.Image", i32, 0>
 // CHECK-DAG: #[[EXPR1:.*]] = #llvm.di_global_variable_expression<var = #[[GVAR1]], expr = <[DW_OP_push_object_address, DW_OP_deref]>>
 // CHECK-DAG: #[[EXPR2:.*]] = #llvm.di_global_variable_expression<var = #[[GVAR2]], expr = <[DW_OP_LLVM_arg(0), DW_OP_LLVM_arg(1), DW_OP_plus]>>
 // CHECK-DAG: #[[EXPR3:.*]] = #llvm.di_global_variable_expression<var = #[[GVAR3]], expr = <[DW_OP_LLVM_convert(16, DW_ATE_signed)]>>
-// CHECK-DAG:   llvm.mlir.global external @global_with_expr1() {addr_space = 0 : i32, dbg_expr = #[[EXPR0]]} : i64
-// CHECK-DAG:   llvm.mlir.global external @global_with_expr2() {addr_space = 0 : i32, dbg_expr = #[[EXPR1]]} : i64
-// CHECK-DAG:   llvm.mlir.global external @global_with_expr3() {addr_space = 0 : i32, dbg_expr = #[[EXPR2]]} : i64
-// CHECK-DAG:   llvm.mlir.global external @global_with_expr4() {addr_space = 0 : i32, dbg_expr = #[[EXPR3]]} : i64
+// CHECK-DAG:   llvm.mlir.global external @global_with_expr1() {addr_space = 0 : i32, dbg_expr = [#[[EXPR0]]]} : i64
+// CHECK-DAG:   llvm.mlir.global external @global_with_expr2() {addr_space = 0 : i32, dbg_expr = [#[[EXPR1]]]} : i64
+// CHECK-DAG:   llvm.mlir.global external @global_with_expr3() {addr_space = 0 : i32, dbg_expr = [#[[EXPR2]]]} : i64
+// CHECK-DAG:   llvm.mlir.global external @global_with_expr4() {addr_space = 0 : i32, dbg_expr = [#[[EXPR3]]]} : i64
 
 #di_file = #llvm.di_file<"not" in "existence">
 #di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_C, file = #di_file, producer = "MLIR", isOptimized = true, emissionKind = Full>
 #di_basic_type = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "uint64_t", sizeInBits = 64, encoding = DW_ATE_unsigned>
-llvm.mlir.global external @global_with_expr1() {addr_space = 0 : i32, dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_1", linkageName = "global_with_expr_1", file = #di_file, line = 370, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <>>} : i64
-llvm.mlir.global external @global_with_expr2() {addr_space = 0 : i32, dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_2", linkageName = "global_with_expr_2", file = #di_file, line = 371, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <[DW_OP_push_object_address, DW_OP_deref]>>} : i64
-llvm.mlir.global external @global_with_expr3() {addr_space = 0 : i32, dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_3", linkageName = "global_with_expr_3", file = #di_file, line = 372, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <[DW_OP_LLVM_arg(0), DW_OP_LLVM_arg(1), DW_OP_plus]>>} : i64
-llvm.mlir.global external @global_with_expr4() {addr_space = 0 : i32, dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_4", linkageName = "global_with_expr_4", file = #di_file, line = 373, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <[DW_OP_LLVM_convert(16, DW_ATE_signed)]>>} : i64
+llvm.mlir.global external @global_with_expr1() {addr_space = 0 : i32, dbg_expr = [#llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_1", linkageName = "global_with_expr_1", file = #di_file, line = 370, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <>>]} : i64
+llvm.mlir.global external @global_with_expr2() {addr_space = 0 : i32, dbg_expr = [#llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_2", linkageName = "global_with_expr_2", file = #di_file, line = 371, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <[DW_OP_push_object_address, DW_OP_deref]>>]} : i64
+llvm.mlir.global external @global_with_expr3() {addr_space = 0 : i32, dbg_expr = [#llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_3", linkageName = "global_with_expr_3", file = #di_file, line = 372, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <[DW_OP_LLVM_arg(0), DW_OP_LLVM_arg(1), DW_OP_plus]>>]} : i64
+llvm.mlir.global external @global_with_expr4() {addr_space = 0 : i32, dbg_expr = [#llvm.di_global_variable_expression<var = <scope = #di_compile_unit, name = "global_with_expr_4", linkageName = "global_with_expr_4", file = #di_file, line = 373, type = #di_basic_type, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <[DW_OP_LLVM_convert(16, DW_ATE_signed)]>>]} : i64
diff --git a/mlir/test/Dialect/LLVMIR/legalize-for-export.mlir b/mlir/test/Dialect/LLVMIR/legalize-for-export.mlir
index b1b06740f194422ee4a50c96cd0024949f13a4df..7b4557d47bde2f5b5b724a6d059ca8212399c786 100644
--- a/mlir/test/Dialect/LLVMIR/legalize-for-export.mlir
+++ b/mlir/test/Dialect/LLVMIR/legalize-for-export.mlir
@@ -42,7 +42,6 @@ llvm.func @repeated_successor_openmp(%arg0: i64, %arg1: i64, %arg2: i64, %arg3:
     // CHECK: ^[[BB2]](%[[ARG:.*]]: i64):
     // CHECK:  llvm.br ^[[BB1]](%[[ARG]] : i64)
     }
-    omp.terminator
   }
   llvm.return
 }
diff --git a/mlir/test/Dialect/Linalg/generalize-named-ops.mlir b/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
index aba26c35931fd388f47164455b28f377695bd656..1e8f1435ca0fa5c1395b38bbacfa247c0c18c5ea 100644
--- a/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
@@ -29,34 +29,6 @@ func.func @generalize_matmul_buffer(%A : memref<16x8xf32>, %B: memref<8x32xf32>,
 
 // -----
 
-func.func @matmul_bcast_a(%arg0: memref<5xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5xf32>, memref<5x7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-// CHECK-LABEL:   func.func @matmul_bcast_a(
-// CHECK-SAME:                              %[[VAL_0:.*]]: memref<5xf32>,
-// CHECK-SAME:                              %[[VAL_1:.*]]: memref<5x7xf32>,
-// CHECK-SAME:                              %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]], iterator_types = ["parallel", "parallel", "reduction"]} ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<5x7xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) {
-// CHECK:           ^bb0(%[[VAL_3:.*]]: f32, %[[VAL_4:.*]]: f32, %[[VAL_5:.*]]: f32):
-// CHECK:             %[[VAL_6:.*]] = arith.mulf %[[VAL_3]], %[[VAL_4]] : f32
-// CHECK:             %[[VAL_7:.*]] = arith.addf %[[VAL_5]], %[[VAL_6]] : f32
-// CHECK:             linalg.yield %[[VAL_7]] : f32
-// CHECK:           }
-// CHECK:           return
-// CHECK:         }
-
-// -----
-
 func.func @generalize_matmul_tensor(%A : tensor<16x8xf32>, %B: tensor<8x32xf32>, %C: tensor<16x32xf32>) -> tensor<16x32xf32> {
   %0 = linalg.matmul ins(%A, %B: tensor<16x8xf32>, tensor<8x32xf32>)
                     outs(%C: tensor<16x32xf32>) -> tensor<16x32xf32>
@@ -919,86 +891,3 @@ func.func @fill_tensor(%f: f32, %v: vector<2x4xf32>) -> (tensor<f32>, tensor<vec
 
   return %0, %1: tensor<f32>, tensor<vector<2x4xf32>>
 }
-
-// -----
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2, d0)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @matmul_transpose_a_explicit(
-// CHECK-SAME:                                  %[[VAL_0:.*]]: memref<5x3xf32>,
-// CHECK-SAME:                                  %[[VAL_1:.*]]: memref<5x7xf32>,
-// CHECK-SAME:                                  %[[VAL_2:.*]]: memref<3x7xf32>) {
-
-// CHECK:           linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]], iterator_types = ["parallel", "parallel", "reduction"]}
-// CHECK:           arith.mulf
-// CHECK:           arith.addf
-
-func.func @matmul_transpose_a_explicit(%arg0: memref<5x3xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%arg0, %arg1 : memref<5x3xf32>, memref<5x7xf32>)
-                      outs(%arg2: memref<3x7xf32>)
-                      
-  return
-}
-
-// -----
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-// CHECK-LABEL:   func.func @matmul_transpose_b_explicit(
-// CHECK-SAME:                                           %[[VAL_0:.*]]: memref<3x5xf32>,
-// CHECK-SAME:                                           %[[VAL_1:.*]]: memref<7x5xf32>,
-// CHECK-SAME:                                           %[[VAL_2:.*]]: memref<3x7xf32>) {
-
-// CHECK:           linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]], iterator_types = ["parallel", "parallel", "reduction"]}
-// CHECK:           arith.mulf
-// CHECK:           arith.addf
-
-func.func @matmul_transpose_b_explicit(%arg0: memref<3x5xf32>, %arg1: memref<7x5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%arg0, %arg1 : memref<3x5xf32>, memref<7x5xf32>)
-                      outs(%arg2: memref<3x7xf32>)
-                      
-  return
-}
-
-// -----
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2, d0)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @matmul_transpose_a_b_explicit(
-// CHECK-SAME:                                             %[[VAL_0:.*]]: memref<5x3xf32>,
-// CHECK-SAME:                                             %[[VAL_1:.*]]: memref<7x5xf32>,
-// CHECK-SAME:                                             %[[VAL_2:.*]]: memref<3x7xf32>) {
-
-// CHECK:           linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]], iterator_types = ["parallel", "parallel", "reduction"]}
-// CHECK:           arith.mulf
-// CHECK:           arith.addf
-
-func.func @matmul_transpose_a_b_explicit(%arg0: memref<5x3xf32>, %arg1: memref<7x5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%arg0, %arg1 : memref<5x3xf32>, memref<7x5xf32>)
-                      outs(%arg2: memref<3x7xf32>)
-                      
-  return
-}
-
-// -----
-
diff --git a/mlir/test/Dialect/Linalg/inline-scalar-operands.mlir b/mlir/test/Dialect/Linalg/inline-scalar-operands.mlir
index 93d5b8779c7461f344fbe9a258df43cf0af98a8b..8384b307d2dfbdb8b53081e52aa2a6f760d7e28a 100644
--- a/mlir/test/Dialect/Linalg/inline-scalar-operands.mlir
+++ b/mlir/test/Dialect/Linalg/inline-scalar-operands.mlir
@@ -46,3 +46,27 @@ func.func @inline_oned(%arg0: tensor<4xf32>, %scalar: tensor<1xf32>) -> tensor<4
     } -> tensor<4xf32>
   return %1 : tensor<4xf32>
 }
+
+// -----
+
+// CHECK: #[[MAP:.*]] = affine_map<(d0) -> (d0)>
+#map2 = affine_map<(d0) -> (d0)>
+#map3 = affine_map<(d0) -> ()>
+
+// CHECK: func @inline_scalar(%[[ARG:.*]]: tensor<4xf32>, %[[SCALAR:.*]]: f32)
+func.func @inline_scalar(%arg0: tensor<4xf32>, %scalar: f32) -> tensor<4xf32> {
+    %0 = tensor.empty() : tensor<4xf32>
+    // CHECK: linalg.generic {indexing_maps = [#[[MAP]], #[[MAP]]],
+    // CHECK-SAME: iterator_types = ["parallel"]} ins(%[[ARG]] : tensor<4xf32>)
+    %1 = linalg.generic {indexing_maps = [#map2, #map3, #map2],
+                         iterator_types = ["parallel"]}
+                         ins(%arg0, %scalar : tensor<4xf32>, f32)
+                         outs(%0 : tensor<4xf32>) {
+    // CHECK: ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32)
+    ^bb0(%arg1: f32, %arg2: f32, %arg3: f32):
+      // CHECK: arith.divf %[[IN]], %[[SCALAR]] : f32
+      %2 = arith.divf %arg1, %arg2 : f32
+      linalg.yield %2 : f32
+    } -> tensor<4xf32>
+  return %1 : tensor<4xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index b2869893b8042dcc4e5ebb87b6f26203057365a4..c481a723c5623c4ecd4e88abcee49bd503a8b76f 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -361,165 +361,6 @@ func.func @invalid_static_matmul(%arg0: memref<2x4xf32>, %arg1: memref<3x4xf32>,
 
 // -----
 
-func.func @invalid_indexing_maps_matmul(%arg0: memref<2x4xf32>, %arg1: memref<3x4xf32>, %arg2: memref<2x4xf32>) {
-  // expected-error @+1 {{expected attribute value}}
-  linalg.matmul indexing_maps = [
-                       ,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%arg0, %arg1 : memref<2x4xf32>, memref<3x4xf32>)
-                      outs(%arg2 :memref<2x4xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_matmul_dim_a(%arg0: memref<5x5xf32>, %arg1: memref<5x5xf32>, %arg2: memref<5x5xf32>) {
-  // expected-error @+1 {{Unexpected dim expression in map result}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5x5xf32>, memref<5x5xf32>) outs(%arg2: memref<5x5xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_matmul_dim_b(%arg0: memref<5x5xf32>, %arg1: memref<5x5xf32>, %arg2: memref<5x5xf32>) {
-  // expected-error @+1 {{Unexpected dim expression in map result}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5x5xf32>, memref<5x5xf32>) outs(%arg2: memref<5x5xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_transpose_a_matmul(%lhs: tensor<4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) -> tensor<4x64xf32> {
-  // expected-error @+1 {{inferred input/output operand #1 has shape's dimension #0 to be 4, but found 1}}
-  %0 = linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%lhs, %rhs : tensor<4x1xf32>, tensor<1x64xf32>)
-                      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
-  return %0: tensor<4x64xf32>
-}
-
-// -----
-
-func.func @invalid_transpose_b_matmul(%lhs: tensor<4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) -> tensor<4x64xf32> {
-  // expected-error @+1 {{inferred input/output operand #1 has shape's dimension #1 to be 1, but found 64}}
-  %0 = linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%lhs, %rhs : tensor<4x1xf32>, tensor<1x64xf32>)
-                      outs(%init : tensor<4x64xf32>) -> tensor<4x64xf32>
-  return %0: tensor<4x64xf32>
-}
-
-// -----
-
-func.func @invalid_bcast_a(%arg0: memref<3xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  // expected-error @+1 {{'linalg.matmul' op Invalid broadcast requested, should be (d2)}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0)>,
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<3xf32>, memref<5x7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_bcast_b(%arg0: memref<3x5xf32>, %arg1: memref<7xf32>, %arg2: memref<3x7xf32>) {
-  // expected-error @+1 {{'linalg.matmul' op Invalid broadcast requested, should be (d2)}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_bcast_a_rank_mismatch(%arg0: memref<3x5xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  // expected-error @+1 {{'linalg.matmul' op expected operand rank (2) to match the result rank of indexing_map #0 (1)}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<5x7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_bcast_b_rank_mismatch(%arg0: memref<3x5xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  // expected-error @+1 {{'linalg.matmul' op expected operand rank (2) to match the result rank of indexing_map #1 (1)}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<5x7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_matmul_bcast_b_transpose_a(%arg0: memref<5x3xf32>, %arg1: memref<7xf32>, %arg2: memref<3x7xf32>) {
-  // expected-error @+1 {{inferred input/output operand #1 has shape's dimension #0 to be 5, but found 7}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5x3xf32>, memref<7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_matmul_bcast_b_transpose_a_wrong_dim(%arg0: memref<3x5xf32>, %arg1: memref<5xf32>, %arg2: memref<3x7xf32>) {
-  // expected-error @+1 {{'linalg.matmul' op Unexpected dim expression in map result.}}
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<5xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
-func.func @invalid_indexing_maps_placement_matmul(%lhs: tensor<4x1xf32>, %rhs: tensor<1x64xf32>, %init: tensor<4x64xf32>) {
-  // expected-error @+2 {{custom op 'indexing_maps' is unknown (tried 'func.indexing_maps' as well)}}
-  linalg.matmul ins(%lhs, %rhs : tensor<4x1xf32>, tensor<1x64xf32>) outs(%init : tensor<4x64xf32>)
-                        indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-  return
-}
-
-// -----
-
 func.func @invalid_static_2d_conv(%input : memref<1x3x4x2xf32>, %filter: memref<3x2x2x1xf32>, %output: memref<1x2x3x1xf32>) {
   // expected-error @+1 {{inferred input/output operand #0 has shape's dimension #1 to be greater than or equal to 4, but found 3}}
   linalg.conv_2d_nhwc_hwcf
diff --git a/mlir/test/Dialect/Linalg/named-ops.mlir b/mlir/test/Dialect/Linalg/named-ops.mlir
index 65c18de8424771aecef5eec15f08426ce755fd5a..02ecbed232c8b56fb450cafb8b587c13845e5ff9 100644
--- a/mlir/test/Dialect/Linalg/named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/named-ops.mlir
@@ -1201,249 +1201,6 @@ func.func @matmul_transpose_a(%arg0: memref<5x3xf32>, %arg1: memref<5x7xf32>, %a
 
 // -----
 
-// CHECK-LABEL: func @matmul_transpose_a_explicit
-//       CHECK:   linalg.matmul
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5x3xf32>, memref<5x7xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
-func.func @matmul_transpose_a_explicit(%arg0: memref<5x3xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%arg0, %arg1 : memref<5x3xf32>, memref<5x7xf32>)
-                      outs(%arg2: memref<3x7xf32>)
-                      
-  return
-}
-
-// -----
-
-func.func @matmul_transpose_b_explicit(%arg0: memref<3x5xf32>, %arg1: memref<7x5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%arg0, %arg1 : memref<3x5xf32>, memref<7x5xf32>)
-                      outs(%arg2: memref<3x7xf32>)
-                      
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @matmul_transpose_b_explicit(
-// CHECK-SAME:                                           %[[VAL_0:.*]]: memref<3x5xf32>,
-// CHECK-SAME:                                           %[[VAL_1:.*]]: memref<7x5xf32>,
-// CHECK-SAME:                                           %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<3x5xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
-// CHECK:           return
-// CHECK:         }
-
-// -----
-
-func.func @matmul_transpose_a_b_explicit(%arg0: memref<5x3xf32>, %arg1: memref<7x5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                      ]
-                      ins(%arg0, %arg1 : memref<5x3xf32>, memref<7x5xf32>)
-                      outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2, d0)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @matmul_transpose_a_b_explicit(
-// CHECK-SAME:                                             %[[VAL_0:.*]]: memref<5x3xf32>,
-// CHECK-SAME:                                             %[[VAL_1:.*]]: memref<7x5xf32>,
-// CHECK-SAME:                                             %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5x3xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
-// CHECK:           return
-// CHECK:         }
-
-// -----
-
-func.func @matmul_bcast_a(%arg0: memref<5xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5xf32>, memref<5x7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-// CHECK-LABEL: func @matmul_bcast_a
-//       CHECK:   linalg.matmul
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5xf32>, memref<5x7xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
-
-// -----
-
-func.func @matmul_bcast_a_dim1(%arg0: memref<5xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5xf32>, memref<5x7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-// CHECK-LABEL: func @matmul_bcast_a_dim1
-//       CHECK:   linalg.matmul
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5xf32>, memref<5x7xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
-
-// -----
-
-func.func @matmul_bcast_b(%arg0: memref<3x5xf32>, %arg1: memref<5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<5xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-// CHECK-LABEL: func @matmul_bcast_b
-//       CHECK:   linalg.matmul
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<3x5xf32>, memref<5xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
-
-// -----
-
-func.func @matmul_bcast_a_b(%arg0: memref<5xf32>, %arg1: memref<5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5xf32>, memref<5xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @matmul_bcast_a_b(
-// CHECK-SAME:                                %[[VAL_0:.*]]: memref<5xf32>, %[[VAL_1:.*]]: memref<5xf32>,
-// CHECK-SAME:                                %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_0]], #[[$ATTR_1]]]
-// CHECK:           return
-// CHECK:         }
-
-// -----
-
-func.func @matmul_bcast_b_dim1(%arg0: memref<3x5xf32>, %arg1: memref<5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d0, d2)>,
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<3x5xf32>, memref<5xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-// CHECK-LABEL: func @matmul_bcast_b_dim1
-//       CHECK:   linalg.matmul
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<3x5xf32>, memref<5xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
-
-// -----
-
-func.func @dynamic_matmul_bcast_a(%arg0: memref<?xf32>, %arg1: memref<?x?xf32>, %arg2: memref<?x?xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d2, d1)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<?xf32>, memref<?x?xf32>) outs(%arg2: memref<?x?xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @dynamic_matmul_bcast_a(
-// CHECK-SAME:                                      %[[VAL_0:.*]]: memref<?xf32>,
-// CHECK-SAME:                                      %[[VAL_1:.*]]: memref<?x?xf32>,
-// CHECK-SAME:                                      %[[VAL_2:.*]]: memref<?x?xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<?xf32>, memref<?x?xf32>) outs(%[[VAL_2]] : memref<?x?xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
-// CHECK:           return
-// CHECK:         }
-
-// -----
-
-func.func @matmul_bcast_a_transpose_b(%arg0: memref<5xf32>, %arg1: memref<7x5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d1, d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5xf32>, memref<7x5xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1, d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @matmul_bcast_a_transpose_b(
-// CHECK-SAME:                                  %[[VAL_0:.*]]: memref<5xf32>,
-// CHECK-SAME:                                  %[[VAL_1:.*]]: memref<7x5xf32>,
-// CHECK-SAME:                                  %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5xf32>, memref<7x5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
-// CHECK:           return
-// CHECK:         }
-
-// -----
-
-func.func @matmul_bcast_b_transpose_a(%arg0: memref<5x3xf32>, %arg1: memref<5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul indexing_maps = [
-                       affine_map<(d0, d1, d2) -> (d2, d0)>,
-                       affine_map<(d0, d1, d2) -> (d2)>,
-                       affine_map<(d0, d1, d2) -> (d0, d1)>
-                     ]
-                     ins(%arg0, %arg1 : memref<5x3xf32>, memref<5xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d2, d0)>
-// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d2)>
-// CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
-
-// CHECK-LABEL:   func.func @matmul_bcast_b_transpose_a(
-// CHECK-SAME:                                          %[[VAL_0:.*]]: memref<5x3xf32>,
-// CHECK-SAME:                                          %[[VAL_1:.*]]: memref<5xf32>,
-// CHECK-SAME:                                          %[[VAL_2:.*]]: memref<3x7xf32>) {
-// CHECK:           linalg.matmul ins(%[[VAL_0]], %[[VAL_1]] : memref<5x3xf32>, memref<5xf32>) outs(%[[VAL_2]] : memref<3x7xf32>) indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_2]]]
-// CHECK:           return
-// CHECK:         }
-
-// -----
-
 // CHECK-LABEL: func @matmul_transpose_b
 //       CHECK:   linalg.matmul_transpose_b
 //  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<3x5xf32>, memref<7x5xf32>)
diff --git a/mlir/test/Dialect/Linalg/roundtrip-broadcast.mlir b/mlir/test/Dialect/Linalg/roundtrip-broadcast.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..d6915ec8fbbf6f4714e5f5302901a7df79245df3
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/roundtrip-broadcast.mlir
@@ -0,0 +1,32 @@
+// RUN: mlir-opt %s -linalg-generalize-named-ops | mlir-opt --linalg-specialize-generic-ops | FileCheck %s
+
+// CHECK-LABEL: broadcast_first_dimension
+// CHECK-SAME:   %[[A:.+]]: tensor<?x?xf32>, %[[Out:.+]]: tensor<?x?x?xf32>)
+// CHECK-NOT:     linalg.generic
+// CHECK:         %broadcasted = linalg.broadcast ins(%[[A]] : tensor<?x?xf32>) outs(%[[Out]] : tensor<?x?x?xf32>) dimensions = [0]
+//
+func.func @broadcast_first_dimension(%A: tensor<?x?xf32>, %Out: tensor<?x?x?xf32>) -> tensor<?x?x?xf32> {
+   %res = linalg.broadcast ins(%A: tensor<?x?xf32>) outs(%Out: tensor<?x?x?xf32>) dimensions = [0]
+  return %res : tensor<?x?x?xf32>
+}
+
+// CHECK-LABEL: broadcast_mid_dimension
+// CHECK-SAME:   %[[A:.+]]: tensor<3x5xf32>, %[[Out:.+]]: tensor<3x4x5xf32>)
+// CHECK-NOT:     linalg.generic
+// CHECK:         %broadcasted = linalg.broadcast ins(%[[A]] : tensor<3x5xf32>) outs(%[[Out]] : tensor<3x4x5xf32>) dimensions = [1]
+//
+func.func @broadcast_mid_dimension(%A: tensor<3x5xf32>, %Out: tensor<3x4x5xf32>) -> tensor<3x4x5xf32> {
+   %res = linalg.broadcast ins(%A: tensor<3x5xf32>) outs(%Out: tensor<3x4x5xf32>) dimensions = [1]
+  return %res : tensor<3x4x5xf32>
+}
+
+
+// CHECK-LABEL: broadcast_multiple_dimensions
+// CHECK-SAME:   %[[A:.+]]: tensor<4x5x7xf32>, %[[Out:.+]]: tensor<3x4x5x6x7x8x9xf32>)
+// CHECK-NOT:     linalg.generic
+// CHECK:         %broadcasted = linalg.broadcast ins(%[[A]] : tensor<4x5x7xf32>) outs(%[[Out]] : tensor<3x4x5x6x7x8x9xf32>) dimensions = [0, 3, 5, 6]
+//
+func.func @broadcast_multiple_dimensions(%A: tensor<4x5x7xf32>, %Out: tensor<3x4x5x6x7x8x9xf32>) -> tensor<3x4x5x6x7x8x9xf32> {
+   %res = linalg.broadcast ins(%A: tensor<4x5x7xf32>) outs(%Out: tensor<3x4x5x6x7x8x9xf32>) dimensions = [0,3,5,6]
+  return %res : tensor<3x4x5x6x7x8x9xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/roundtrip-transpose.mlir b/mlir/test/Dialect/Linalg/roundtrip-transpose.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..21b7b348f1c7f8c7c6ab8f4090698e2c19a9a1b0
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/roundtrip-transpose.mlir
@@ -0,0 +1,22 @@
+// RUN: mlir-opt %s -linalg-generalize-named-ops | mlir-opt --linalg-specialize-generic-ops | FileCheck %s
+
+// CHECK-LABEL: transpose2D
+// CHECK-SAME:  %[[A:.+]]: tensor<16x64xf32>, %[[Out:.+]]: tensor<64x16xf32>
+// CHECK-NOT:   linalg.generic
+// CHECK:  %transposed = linalg.transpose ins(%[[A]] : tensor<16x64xf32>) outs(%[[Out]] : tensor<64x16xf32>) permutation = [1, 0]
+//
+func.func @transpose2D(%A: tensor<16x64xf32>, %Out: tensor<64x16xf32>) -> tensor<64x16xf32> {
+  %res = linalg.transpose ins(%A: tensor<16x64xf32>) outs(%Out: tensor<64x16xf32>) permutation = [1,0]
+  return %res : tensor<64x16xf32>
+}
+
+
+// CHECK-LABEL: transpose3D
+// CHECK-SAME:  %[[A:.+]]: tensor<7x8x9xf32>, %[[Out:.+]]: tensor<9x7x8xf32>
+// CHECK-NOT:   linalg.generic
+// CHECK:  %transposed = linalg.transpose ins(%[[A]] : tensor<7x8x9xf32>) outs(%[[Out]] : tensor<9x7x8xf32>) permutation = [2, 0, 1]
+//
+func.func @transpose3D(%arg0: tensor<7x8x9xf32>, %arg1: tensor<9x7x8xf32>) -> tensor<9x7x8xf32> {
+  %transposed = linalg.transpose ins(%arg0 : tensor<7x8x9xf32>) outs(%arg1 : tensor<9x7x8xf32>) permutation = [2, 0, 1]
+  return %transposed : tensor<9x7x8xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/specialize-generic-ops-fail.mlir b/mlir/test/Dialect/Linalg/specialize-generic-ops-fail.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..542a7ed4a198b8015ec782a4f00b02e29a25e9fc
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/specialize-generic-ops-fail.mlir
@@ -0,0 +1,16 @@
+// RUN: mlir-opt %s -split-input-file --linalg-specialize-generic-ops | FileCheck %s
+
+#map = affine_map<(d0, d1, d2) -> (d1, d0)>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+// This test checks that linalg.generic does not get incorrectly specialized to transform or broadcast.
+// CHECK-LABEL: @transpose_and_broadcast
+// CHECK: linalg.generic
+func.func @transpose_and_broadcast(%arg0: tensor<7x8xf32>, %arg1: tensor<8x7x9xf32>) -> tensor<8x7x9xf32> {
+  %0 = linalg.generic
+        {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]}
+        ins(%arg0 : tensor<7x8xf32>) outs(%arg1 : tensor<8x7x9xf32>) {
+        ^bb0(%in: f32, %out: f32):
+           linalg.yield %in : f32
+  } -> tensor<8x7x9xf32>
+  return %0 : tensor<8x7x9xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/transform-op-specialize.mlir b/mlir/test/Dialect/Linalg/transform-op-specialize.mlir
index 35679db7412f30d10aaca95ca99d42025813e0f6..31f2f6b1ab513f20923a41247fe71027494034b1 100644
--- a/mlir/test/Dialect/Linalg/transform-op-specialize.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-specialize.mlir
@@ -4,18 +4,6 @@
 #map1 = affine_map<(d0, d1) -> (d0)>
 #map2 = affine_map<(d0, d1) -> (d1, d0)>
 
-func.func @broadcast_copy_expect_no_match(%arg0: memref<?xf32>, %arg1: memref<?x?xf32>) {
-  // expected-note @below {{when applied to this op}}
-  linalg.generic {
-    indexing_maps = [#map1, #map], 
-    iterator_types = ["parallel", "parallel"]}
-    ins(%arg0 : memref<?xf32>) outs(%arg1 : memref<?x?xf32>) {
-    ^bb0(%in: f32, %out: f32):
-      linalg.yield %in : f32
-  }
-  return
-}
-
 func.func @not_a_copy_expect_no_match(%arg0: memref<?x?xf32>, %arg1: memref<?x?xf32>) {
   // expected-note @below {{when applied to this op}}
   linalg.generic {
diff --git a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
index c5760acf94a88a5c8d2c382c4c7fe574de816055..776dc5b748c846170662c09330302b611f148c40 100644
--- a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
+++ b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
@@ -85,31 +85,32 @@ module attributes {transform.with_named_sequence} {
 // CHECK:    scf.yield %[[S9]]
 // CHECK:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
 // CHECK:  %[[COLLAPSED_6:.*]] = tensor.collapse_shape %[[S4]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:  %[[S7:.*]] = tensor.empty()
 // CHECK:  %[[S6:.*]] = linalg.batch_matmul
 // CHECK:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 2, 2, 2, 2]
-// CHECK:  %[[S7:.*]] = tensor.empty() : tensor<2x8x8x2xf32>
-// CHECK:  %[[S8:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S7]])
+// CHECK:  %[[S8:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[ARG2]])
 // CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
 // CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1]
 // CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
 // CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
-// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[ARG2]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[ARG6]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
 // CHECK:      %[[S12:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[EXTRACTED_SLICE_7]])
 // CHECK:        %[[S15:.*]] = scf.for %[[ARG9:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG10:.*]] = %[[ARG8]])
 // CHECK:          %[[EXTRACTED_SLICE_8:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][0, 0, 0, 0, %[[ARG7]], %[[ARG9]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          %[[S25:.*]] = tensor.extract_slice %[[ARG10]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1]
 // CHECK:          %[[S16:.*]] = tensor.empty() : tensor<4x6xf32>
 // CHECK:          %[[S17:.*]] = linalg.fill ins(%[[CST_6]] : f32) outs(%[[S16]] : tensor<4x6xf32>) -> tensor<4x6xf32>
 // CHECK:          %[[S18:.*]] = linalg.matmul ins(%[[CST_1]], %[[EXTRACTED_SLICE_8]] : tensor<4x6xf32>, tensor<6x6xf32>) outs(%[[S17]] : tensor<4x6xf32>) -> tensor<4x6xf32>
 // CHECK:          %[[S19:.*]] = tensor.empty() : tensor<4x4xf32>
 // CHECK:          %[[S20:.*]] = linalg.fill ins(%[[CST_6]] : f32) outs(%[[S19]] : tensor<4x4xf32>) -> tensor<4x4xf32>
 // CHECK:          %[[S21:.*]] = linalg.matmul ins(%[[S18]], %[[CST_0]] : tensor<4x6xf32>, tensor<6x4xf32>) outs(%[[S20]] : tensor<4x4xf32>) -> tensor<4x4xf32>
-// CHECK:          %[[S22:.*]] = tensor.empty() : tensor<4x4xf32>
-// CHECK:          %[[S23:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S22]] : tensor<4x4xf32>) {
-// CHECK:          ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
-// CHECK:            linalg.yield %[[IN]] : f32
+// CHECK:          %[[S23:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]], %[[S21]] : f32, tensor<4x4xf32>) outs(%[[S25]] : tensor<4x4xf32>) {
+// CHECK:          ^bb0(%[[IN1:.*]]: f32, %[[IN2:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:             %[[VAL_90:.*]] = arith.mulf %[[IN1]], %[[IN2]] : f32
+// CHECK:             %[[VAL_91:.*]] = arith.addf %[[VAL_90]], %[[OUT]] : f32
+/// CHECK:            linalg.yield %[[VAL_91]] : f32
 // CHECK:          } -> tensor<4x4xf32>
-// CHECK:          %[[S24:.*]] = linalg.mul ins(%[[S23]], %[[S21]] : tensor<4x4xf32>, tensor<4x4xf32>) outs(%[[S22]] : tensor<4x4xf32>) -> tensor<4x4xf32>
-// CHECK:          %[[INSERTED_SLICE_9:.*]] = tensor.insert_slice %[[S24]] into %[[ARG10]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1]
+// CHECK:          %[[INSERTED_SLICE_9:.*]] = tensor.insert_slice %[[S23]] into %[[ARG10]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1]
 // CHECK:          scf.yield %[[INSERTED_SLICE_9]]
 // CHECK:        scf.yield %[[S15]]
 // CHECK:      %[[S13:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
@@ -218,32 +219,33 @@ module attributes {transform.with_named_sequence} {
 // CHECK:    scf.yield %[[S9]]
 // CHECK:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
 // CHECK:  %[[COLLAPSED_7:.*]] = tensor.collapse_shape %[[S4]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:  %[[S7:.*]] = tensor.empty()
 // CHECK:  %[[S6:.*]] = linalg.batch_matmul
 // CHECK:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 3, 3, 2, 2]
 // CHECK:  %[[PADDED_8:.*]] = tensor.pad %[[ARG2]] low[0, 0, 0, 0] high[0, 3, 3, 0]
-// CHECK:  %[[S7:.*]] = tensor.empty() : tensor<2x12x12x2xf32>
-// CHECK:  %[[S8:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S7]])
+// CHECK:  %[[S8:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[PADDED_8]])
 // CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
 // CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1]
 // CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
 // CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
-// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[PADDED_8]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG7]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
 // CHECK:      %[[S12:.*]] = scf.for %[[ARG8:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG9:.*]] = %[[EXTRACTED_SLICE_10]])
 // CHECK:        %[[S15:.*]] = scf.for %[[ARG10:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG11:.*]] = %[[ARG9]])
 // CHECK:          %[[EXTRACTED_SLICE_11:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE_9]][0, 0, 0, 0, %[[ARG8]], %[[ARG10]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          %[[S26:.*]] = tensor.extract_slice %[[ARG11]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 4, 4, 1] [1, 1, 1, 1]
 // CHECK:          %[[S17:.*]] = tensor.empty() : tensor<4x6xf32>
 // CHECK:          %[[S18:.*]] = linalg.fill ins(%[[CST_6]] : f32) outs(%[[S17]] : tensor<4x6xf32>) -> tensor<4x6xf32>
 // CHECK:          %[[S19:.*]] = linalg.matmul ins(%[[CST_1]], %[[EXTRACTED_SLICE_11]] : tensor<4x6xf32>, tensor<6x6xf32>) outs(%[[S18]] : tensor<4x6xf32>) -> tensor<4x6xf32>
 // CHECK:          %[[S20:.*]] = tensor.empty() : tensor<4x4xf32>
 // CHECK:          %[[S21:.*]] = linalg.fill ins(%[[CST_6]] : f32) outs(%[[S20]] : tensor<4x4xf32>) -> tensor<4x4xf32>
 // CHECK:          %[[S22:.*]] = linalg.matmul ins(%[[S19]], %[[CST_0]] : tensor<4x6xf32>, tensor<6x4xf32>) outs(%[[S21]] : tensor<4x4xf32>) -> tensor<4x4xf32>
-// CHECK:          %[[S23:.*]] = tensor.empty() : tensor<4x4xf32>
-// CHECK:          %[[S24:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S23]] : tensor<4x4xf32>) {
-// CHECK:          ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
-// CHECK:            linalg.yield %[[IN]] : f32
+// CHECK:          %[[S24:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]], %[[S22]] : f32, tensor<4x4xf32>) outs(%[[S26]] : tensor<4x4xf32>) {
+// CHECK:          ^bb0(%[[IN1:.*]]: f32, %[[IN2:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:             %[[VAL_104:.*]] = arith.mulf %[[IN1]], %[[IN2]] : f32
+// CHECK:             %[[VAL_105:.*]] = arith.addf %[[VAL_104]], %[[OUT]] : f32
+/// CHECK:            linalg.yield %[[VAL_105]] : f32
 // CHECK:          } -> tensor<4x4xf32>
-// CHECK:          %[[S25:.*]] = linalg.mul ins(%[[S24]], %[[S22]] : tensor<4x4xf32>, tensor<4x4xf32>) outs(%[[S23]] : tensor<4x4xf32>) -> tensor<4x4xf32>
-// CHECK:          %[[INSERTED_SLICE_12:.*]] = tensor.insert_slice %[[S25]] into %[[ARG11]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 4, 4, 1] [1, 1, 1, 1]
+// CHECK:          %[[INSERTED_SLICE_12:.*]] = tensor.insert_slice %[[S24]] into %[[ARG11]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 4, 4, 1] [1, 1, 1, 1]
 // CHECK:          scf.yield %[[INSERTED_SLICE_12]]
 // CHECK:        scf.yield %[[S15]] : tensor<2x4x4x2xf32>
 // CHECK:      %[[S13:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
@@ -330,16 +332,17 @@ module attributes {transform.with_named_sequence} {
 // CHECK:   %[[S6:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[ARG2]])
 // CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
 // CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, 0, 0, %[[ARG3]], %[[ARG5]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:       %[[S15:.*]] = tensor.extract_slice %[[ARG6]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 4, 1, 1] [1, 1, 1, 1]
 // CHECK:       %[[S9:.*]] = tensor.empty() : tensor<4x1xf32>
 // CHECK:       %[[S10:.*]] = linalg.fill ins(%[[CST_3]] : f32) outs(%[[S9]] : tensor<4x1xf32>) -> tensor<4x1xf32>
 // CHECK:       %[[S11:.*]] = linalg.matmul ins(%[[CST_0]], %[[EXTRACTED_SLICE]] : tensor<4x6xf32>, tensor<6x1xf32>) outs(%[[S10]] : tensor<4x1xf32>) -> tensor<4x1xf32>
-// CHECK:       %[[S12:.*]] = tensor.empty() : tensor<4x1xf32>
-// CHECK:       %[[S13:.*]] = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S12]] : tensor<4x1xf32>) {
-// CHECK:       ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
-// CHECK:         linalg.yield %[[IN]] : f32
+// CHECK:       %[[S13:.*]] = linalg.generic {indexing_maps = [#map, #map1, #map1], iterator_types = ["parallel", "parallel"]} ins(%[[CST]], %[[S11]] : f32, tensor<4x1xf32>) outs(%[[S15]] : tensor<4x1xf32>) {
+// CHECK:       ^bb0(%[[IN1:.*]]: f32, %[[IN2:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:          %[[VAL_57:.*]] = arith.mulf %[[IN1]], %[[IN2]] : f32
+// CHECK:          %[[VAL_58:.*]] = arith.addf %[[VAL_57]], %[[OUT]] : f32
+/// CHECK:         linalg.yield %[[VAL_58]] : f32
 // CHECK:       } -> tensor<4x1xf32>
-// CHECK:       %[[S14:.*]] = linalg.mul ins(%[[S13]], %[[S11]] : tensor<4x1xf32>, tensor<4x1xf32>) outs(%[[S12]] : tensor<4x1xf32>) -> tensor<4x1xf32>
-// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S14]] into %[[ARG6]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 4, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S13]] into %[[ARG6]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 4, 1, 1] [1, 1, 1, 1]
 // CHECK:       scf.yield %[[INSERTED_SLICE]]
 // CHECK:     scf.yield %[[S7]]
 // CHECK:   return %[[S6]]
diff --git a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
index 21522a2083b463e79d0efd3f1f51e5e9dc2df613..9598c434aadb8fa25160901547db11b648b3c85c 100644
--- a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
+++ b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
@@ -279,14 +279,14 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG:   %[[C2_1:.*]] = arith.constant 2 : index
 // CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
-// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
-// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[ARG1]]) -> (tensor<2x8x8x2xf32>)
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]] iter_args(%[[ARG6:.*]] = %[[ARG5]]) -> (tensor<2x8x8x2xf32>)
 // CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x2xf32> to tensor<6x6x1x1x2x2xf32>
 // CHECK:       %[[S3:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
 // CHECK:       %[[S4:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
 // CHECK:       %[[S5:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:       %[[S6:.*]] = affine.apply #[[$MAP1]]()
-// CHECK:       %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG1]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 2] [1, 1, 1, 1] : tensor<2x8x8x2xf32> to tensor<2x?x?x2xf32>
+// CHECK:       %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG6]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 2] [1, 1, 1, 1] : tensor<2x8x8x2xf32> to tensor<2x?x?x2xf32>
 
 // -----
 
@@ -321,10 +321,10 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG:    %[[C2_3:.*]] = arith.constant 2 : index
 // CHECK-DAG:    %[[C2_5:.*]] = arith.constant 2 : index
 // CHECK-DAG:    %[[C2_7:.*]] = arith.constant 2 : index
-// CHECK:    %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]]
-// CHECK:      %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]]
-// CHECK:        %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C3]] step %[[C2_5]]
-// CHECK:          %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C2_7]]
+// CHECK:    %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]] iter_args(%[[ARG9:.*]] = %[[ARG1]]) -> (tensor<3x8x8x5xf32>)
+// CHECK:      %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]] iter_args(%[[ARG10:.*]] = %[[ARG9]]) -> (tensor<3x8x8x5xf32>)
+// CHECK:        %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C3]] step %[[C2_5]] iter_args(%[[ARG11:.*]] = %[[ARG10]])
+// CHECK:          %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C2_7]] iter_args(%[[ARG12:.*]] = %[[ARG11]])
 // CHECK:            %[[C3_8:.*]] = arith.constant 3 : index
 // CHECK:            %[[S5:.*]] = affine.min #[[$MAP0]](%[[ARG6]])
 // CHECK:            %[[C5_9:.*]] = arith.constant 5 : index
@@ -334,7 +334,7 @@ module attributes {transform.with_named_sequence} {
 // CHECK:            %[[S8:.*]] = affine.apply #[[$MAP2]](%[[ARG4]])
 // CHECK:            %[[S9:.*]] = affine.apply #[[$MAP3]]()
 // CHECK:            %[[S10:.*]] = affine.apply #[[$MAP3]]()
-// CHECK:            %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG6]], %[[S7]], %[[S8]], %[[ARG8]]] [%[[S5]], %[[S9]], %[[S10]], %[[S6]]] [1, 1, 1, 1] : tensor<3x8x8x5xf32> to tensor<?x?x?x?xf32>
+// CHECK:            %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG12]][%[[ARG6]], %[[S7]], %[[S8]], %[[ARG8]]] [%[[S5]], %[[S9]], %[[S10]], %[[S6]]] [1, 1, 1, 1] : tensor<3x8x8x5xf32> to tensor<?x?x?x?xf32>
 
 // -----
 
@@ -367,14 +367,14 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[C1_4:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[C1_6:.*]] = arith.constant 1 : index
-// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
-// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C1_1]] step %[[C1_2]]
-// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C3]] step %[[C1_4]]
-// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_5]] to %[[C5]] step %[[C1_6]]
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG9:.*]] = %[[ARG1]]) -> (tensor<3x8x1x5xf32>)
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C1_1]] step %[[C1_2]] iter_args(%[[ARG10:.*]] = %[[ARG9]]) -> (tensor<3x8x1x5xf32>)
+// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C3]] step %[[C1_4]] iter_args(%[[ARG11:.*]] = %[[ARG10]]) -> (tensor<3x8x1x5xf32>)
+// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_5]] to %[[C5]] step %[[C1_6]] iter_args(%[[ARG12:.*]] = %[[ARG11]]) -> (tensor<3x8x1x5xf32>)
 // CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x1x2x1x3x5xf32> to tensor<6x1x1x1x1x1xf32>
 // CHECK:           %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
 // CHECK:           %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
 // CHECK:           %[[S7:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:           %[[S8:.*]] = affine.apply #[[$MAP1]]()
-// CHECK:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG6]], %[[S5]], 0, %[[ARG8]]] [1, %[[S7]], 1, 1] [1, 1, 1, 1] : tensor<3x8x1x5xf32> to tensor<1x?x1x1xf32>
+// CHECK:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG12]][%[[ARG6]], %[[S5]], 0, %[[ARG8]]] [1, %[[S7]], 1, 1] [1, 1, 1, 1] : tensor<3x8x1x5xf32> to tensor<1x?x1x1xf32>
 // CHECK:           %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<6x1x1x1x1x1xf32>) outs(%[[EXTRACTED_SLICE_9]] : tensor<1x?x1x1xf32>) -> tensor<1x?x1x1xf32>
diff --git a/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir b/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir
index 4369f5f1eab4cab33c8dc74ccf56c52720a3e2fb..16d06a747327299bbee26fd3b5158a836a234029 100644
--- a/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir
+++ b/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir
@@ -100,21 +100,22 @@ func.func @conv2d(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg
 // CHECK-NEXT:       %[[S8:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[ARG6]]) -> (tensor<2x12x12x2xf32>) {
 // CHECK-NEXT:         %[[S9:.*]] = scf.for %[[ARG9:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG10:.*]] = %[[ARG8]]) -> (tensor<2x12x12x2xf32>) {
 // CHECK-NEXT:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, %[[ARG3]], %[[ARG5]], %[[ARG7]], %[[ARG9]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x3x3x2x2xf32> to tensor<6x6xf32>
+// CHECK-NEXT:           %[[S20:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
+// CHECK-NEXT:           %[[S21:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
+// CHECK-NEXT:           %[[S22:.*]] = tensor.extract_slice %[[ARG10]][%[[ARG7]], %[[S20]], %[[S21]], %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1] : tensor<2x12x12x2xf32> to tensor<4x4xf32>
 // CHECK-NEXT:           %[[S11:.*]] = tensor.empty() : tensor<4x6xf32>
 // CHECK-NEXT:           %[[S12:.*]] = linalg.fill ins(%[[CST_6]] : f32) outs(%[[S11]] : tensor<4x6xf32>) -> tensor<4x6xf32>
 // CHECK-NEXT:           %[[S13:.*]] = linalg.matmul ins(%[[CST_1]], %[[EXTRACTED_SLICE_9]] : tensor<4x6xf32>, tensor<6x6xf32>) outs(%[[S12]] : tensor<4x6xf32>) -> tensor<4x6xf32>
 // CHECK-NEXT:           %[[S14:.*]] = tensor.empty() : tensor<4x4xf32>
 // CHECK-NEXT:           %[[S15:.*]] = linalg.fill ins(%[[CST_6]] : f32) outs(%[[S14]] : tensor<4x4xf32>) -> tensor<4x4xf32>
 // CHECK-NEXT:           %[[S16:.*]] = linalg.matmul ins(%[[S13]], %[[CST_0]] : tensor<4x6xf32>, tensor<6x4xf32>) outs(%[[S15]] : tensor<4x4xf32>) -> tensor<4x4xf32>
-// CHECK-NEXT:           %[[S17:.*]] = tensor.empty() : tensor<4x4xf32>
-// CHECK-NEXT:           %[[S18:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S17]] : tensor<4x4xf32>) {
-// CHECK-NEXT:           ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
-// CHECK-NEXT:             linalg.yield %[[IN]] : f32
+// CHECK-NEXT:           %[[S18:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]], %[[S16]] : f32, tensor<4x4xf32>) outs(%[[S22]] : tensor<4x4xf32>) {
+// CHECK-NEXT:           ^bb0(%[[IN1:.*]]: f32, %[[IN2:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK-NEXT:             %[[VAL_98:.*]] = arith.mulf %[[IN1]], %[[IN2]] : f32
+// CHECK-NEXT:             %[[VAL_99:.*]] = arith.addf %[[VAL_98]], %[[OUT]] : f32
+// CHECK-NEXT:             linalg.yield %[[VAL_99]] : f32
 // CHECK-NEXT:           } -> tensor<4x4xf32>
-// CHECK-NEXT:           %[[S19:.*]] = linalg.mul ins(%[[S18]], %[[S16]] : tensor<4x4xf32>, tensor<4x4xf32>) outs(%[[S17]] : tensor<4x4xf32>) -> tensor<4x4xf32>
-// CHECK-NEXT:           %[[S20:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
-// CHECK-NEXT:           %[[S21:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
-// CHECK-NEXT:           %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S19]] into %[[ARG10]][%[[ARG7]], %[[S20]], %[[S21]], %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1] : tensor<4x4xf32> into tensor<2x12x12x2xf32>
+// CHECK-NEXT:           %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S18]] into %[[ARG10]][%[[ARG7]], %[[S20]], %[[S21]], %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1] : tensor<4x4xf32> into tensor<2x12x12x2xf32>
 // CHECK-NEXT:           scf.yield %[[INSERTED_SLICE]] : tensor<2x12x12x2xf32>
 // CHECK-NEXT:         }
 // CHECK-NEXT:         scf.yield %[[S9]] : tensor<2x12x12x2xf32>
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index a780efe6d22e163415a82ad5f9b8f063c62c162d..f7a87713aca35191aecae008e0a904580c9b7bfe 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -103,7 +103,6 @@ func.func @type_mismatch(%lb : index, %ub : index, %step : index) {
     ^bb0(%iv2: i32):
       omp.yield
     }) : (index, index, index) -> ()
-    omp.terminator
   }
 }
 
@@ -116,20 +115,18 @@ func.func @iv_number_mismatch(%lb : index, %ub : index, %step : index) {
     ^bb0(%iv1 : index, %iv2 : index):
       omp.yield
     }) : (index, index, index) -> ()
-    omp.terminator
   }
 }
 
 // -----
 
 func.func @no_wrapper(%lb : index, %ub : index, %step : index) {
-  // expected-error @below {{op loop wrapper does not contain exactly two nested ops}}
+  // expected-error @below {{op loop wrapper does not contain exactly one nested op}}
   omp.wsloop {
     %0 = arith.constant 0 : i32
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 
@@ -142,9 +139,7 @@ func.func @invalid_nested_wrapper(%lb : index, %ub : index, %step : index) {
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   } {omp.composite}
 }
 
@@ -157,7 +152,6 @@ func.func @no_loops(%lb : index, %ub : index, %step : index) {
     ^bb0():
       omp.yield
     }) : () -> ()
-    omp.terminator
   }
 }
 
@@ -169,7 +163,6 @@ func.func @inclusive_not_a_clause(%lb : index, %ub : index, %step : index) {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 
@@ -181,7 +174,6 @@ func.func @order_value(%lb : index, %ub : index, %step : index) {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 
@@ -192,7 +184,6 @@ func.func @reproducible_order(%lb : index, %ub : index, %step : index) {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 // -----
@@ -202,7 +193,6 @@ func.func @unconstrained_order(%lb : index, %ub : index, %step : index) {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 // -----
@@ -213,7 +203,6 @@ func.func @if_not_allowed(%lb : index, %ub : index, %step : index, %bool_var : i
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 
@@ -225,7 +214,6 @@ func.func @num_threads_not_allowed(%lb : index, %ub : index, %step : index, %int
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 
@@ -237,7 +225,6 @@ func.func @proc_bind_not_allowed(%lb : index, %ub : index, %step : index) {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 }
 
@@ -249,7 +236,6 @@ llvm.func @test_omp_wsloop_dynamic_bad_modifier(%lb : i64, %ub : i64, %step : i6
     omp.loop_nest (%iv) : i64 = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -262,7 +248,6 @@ llvm.func @test_omp_wsloop_dynamic_many_modifier(%lb : i64, %ub : i64, %step : i
     omp.loop_nest (%iv) : i64 = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -275,7 +260,6 @@ llvm.func @test_omp_wsloop_dynamic_wrong_modifier(%lb : i64, %ub : i64, %step :
     omp.loop_nest (%iv) : i64 = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -288,7 +272,6 @@ llvm.func @test_omp_wsloop_dynamic_wrong_modifier2(%lb : i64, %ub : i64, %step :
     omp.loop_nest (%iv) : i64 = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -301,7 +284,6 @@ llvm.func @test_omp_wsloop_dynamic_wrong_modifier3(%lb : i64, %ub : i64, %step :
     omp.loop_nest (%iv) : i64 = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -309,10 +291,8 @@ llvm.func @test_omp_wsloop_dynamic_wrong_modifier3(%lb : i64, %ub : i64, %step :
 // -----
 
 func.func @omp_simd() -> () {
-  // expected-error @below {{op loop wrapper does not contain exactly two nested ops}}
-  omp.simd {
-    omp.terminator
-  }
+  // expected-error @below {{op loop wrapper does not contain exactly one nested op}}
+  omp.simd {}
   return
 }
 
@@ -325,9 +305,7 @@ func.func @omp_simd_nested_wrapper(%lb : index, %ub : index, %step : index) -> (
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     }
-    omp.terminator
   }
   return
 }
@@ -355,7 +333,6 @@ func.func @omp_simd_aligned_mismatch(%arg0 : index, %arg1 : index,
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }) {alignments = [128],
       operandSegmentSizes = array<i32: 2, 0, 0, 0, 0, 0, 0>} : (memref<i32>, memref<i32>) -> ()
   return
@@ -371,7 +348,6 @@ func.func @omp_simd_aligned_negative(%arg0 : index, %arg1 : index,
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }) {alignments = [-1, 128], operandSegmentSizes = array<i32: 2, 0, 0, 0, 0, 0, 0>} : (memref<i32>, memref<i32>) -> ()
   return
 }
@@ -386,7 +362,6 @@ func.func @omp_simd_unexpected_alignment(%arg0 : index, %arg1 : index,
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }) {alignments = [1, 128]} : () -> ()
   return
 }
@@ -401,7 +376,6 @@ func.func @omp_simd_aligned_float(%arg0 : index, %arg1 : index,
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }) {alignments = [1.5, 128], operandSegmentSizes = array<i32: 2, 0, 0, 0, 0, 0, 0>} : (memref<i32>, memref<i32>) -> ()
   return
 }
@@ -416,7 +390,6 @@ func.func @omp_simd_aligned_the_same_var(%arg0 : index, %arg1 : index,
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }) {alignments = [1, 128], operandSegmentSizes = array<i32: 2, 0, 0, 0, 0, 0, 0>} : (memref<i32>, memref<i32>) -> ()
   return
 }
@@ -431,7 +404,6 @@ func.func @omp_simd_nontemporal_the_same_var(%arg0 : index,  %arg1 : index,
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0, 0, 0, 0, 2, 0, 0>} : (memref<i32>, memref<i32>) -> ()
   return
 }
@@ -444,7 +416,6 @@ func.func @omp_simd_order_value(%lb : index, %ub : index, %step : index) {
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -457,7 +428,6 @@ func.func @omp_simd_reproducible_order(%lb : index, %ub : index, %step : index)
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -468,7 +438,6 @@ func.func @omp_simd_unconstrained_order(%lb : index, %ub : index, %step : index)
     omp.loop_nest (%iv) : index = (%arg0) to (%arg1) step (%arg2) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -479,7 +448,6 @@ func.func @omp_simd_pretty_simdlen(%lb : index, %ub : index, %step : index) -> (
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -492,7 +460,6 @@ func.func @omp_simd_pretty_safelen(%lb : index, %ub : index, %step : index) -> (
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -505,7 +472,6 @@ func.func @omp_simd_pretty_simdlen_safelen(%lb : index, %ub : index, %step : ind
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -724,7 +690,6 @@ func.func @foo(%lb : index, %ub : index, %step : index) {
       %2 = arith.constant 2.0 : f32
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -753,7 +718,6 @@ func.func @foo(%lb : index, %ub : index, %step : index) {
       %2 = arith.constant 2.0 : f32
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -787,7 +751,6 @@ func.func @foo(%lb : index, %ub : index, %step : index, %mem : memref<1xf32>) {
       %2 = arith.constant 2.0 : f32
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -828,7 +791,6 @@ func.func @omp_ordered_region1(%x : i32) -> () {
       }
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -844,7 +806,6 @@ func.func @omp_ordered_region2(%x : i32) -> () {
       }
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -860,7 +821,6 @@ func.func @omp_ordered_region3(%x : i32) -> () {
       }
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -882,7 +842,6 @@ func.func @omp_ordered2(%arg1 : i32, %arg2 : i32, %arg3 : i32, %vec0 : i64) -> (
       omp.ordered depend_type(dependsink) depend_vec(%vec0 : i64) {doacross_num_loops = 1 : i64}
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -896,7 +855,6 @@ func.func @omp_ordered3(%arg1 : i32, %arg2 : i32, %arg3 : i32, %vec0 : i64) -> (
       omp.ordered depend_type(dependsink) depend_vec(%vec0 : i64) {doacross_num_loops = 1 : i64}
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -910,7 +868,6 @@ func.func @omp_ordered4(%arg1 : i32, %arg2 : i32, %arg3 : i32, %vec0 : i64) -> (
       omp.ordered depend_type(dependsink) depend_vec(%vec0 : i64) {doacross_num_loops = 1 : i64}
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -924,7 +881,6 @@ func.func @omp_ordered5(%arg1 : i32, %arg2 : i32, %arg3 : i32, %vec0 : i64, %vec
       omp.ordered depend_type(dependsource) depend_vec(%vec0, %vec1 : i64, i64) {doacross_num_loops = 2 : i64}
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -1796,7 +1752,6 @@ func.func @omp_cancel3(%arg1 : i32, %arg2 : i32, %arg3 : i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -1811,7 +1766,6 @@ func.func @omp_cancel4(%arg1 : i32, %arg2 : i32, %arg3 : i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -1876,7 +1830,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>} : (memref<i32>) -> ()
   return
 }
@@ -1892,7 +1845,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0, 0, 0, 0, 0, 0, 0, 0, 0, 2>, reduction_syms = [@add_f32]} : (!llvm.ptr, !llvm.ptr) -> ()
   return
 }
@@ -1907,7 +1859,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>, reduction_syms = [@add_f32, @add_f32]} : (!llvm.ptr) -> ()
   return
 }
@@ -1923,7 +1874,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }) {in_reduction_syms = [@add_f32], operandSegmentSizes = array<i32: 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>} : (!llvm.ptr, !llvm.ptr) -> ()
   return
 }
@@ -1938,7 +1888,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }) {in_reduction_syms = [@add_f32, @add_f32], operandSegmentSizes = array<i32: 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>} : (!llvm.ptr) -> ()
   return
 }
@@ -1965,7 +1914,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -1991,7 +1939,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -2005,7 +1952,6 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
     omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -2013,10 +1959,9 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
 // -----
 
 func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
-  // expected-error @below {{op first nested op in loop wrapper is not another loop wrapper or `omp.loop_nest`}}
+  // expected-error @below {{op nested in loop wrapper is not another loop wrapper or `omp.loop_nest`}}
   omp.taskloop {
     %0 = arith.constant 0 : i32
-    omp.terminator
   }
   return
 }
@@ -2030,9 +1975,7 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
       omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   } {omp.composite}
   return
 }
@@ -2209,7 +2152,6 @@ func.func @omp_distribute_schedule(%chunk_size : i32, %lb : i32, %ub : i32, %ste
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       "omp.yield"() : () -> ()
     }
-    "omp.terminator"() : () -> ()
   }) : (i32) -> ()
 }
 
@@ -2221,24 +2163,11 @@ func.func @omp_distribute_allocate(%data_var : memref<i32>, %lb : i32, %ub : i32
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       "omp.yield"() : () -> ()
     }
-    "omp.terminator"() : () -> ()
   }) : (memref<i32>) -> ()
 }
 
 // -----
 
-func.func @omp_distribute_wrapper(%lb: index, %ub: index, %step: index) -> () {
-  // expected-error @below {{op second nested op in loop wrapper is not a terminator}}
-  omp.distribute {
-    omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
-      "omp.yield"() : () -> ()
-    }
-    %0 = arith.constant 0 : i32
-  }
-}
-
-// -----
-
 func.func @omp_distribute_nested_wrapper(%lb: index, %ub: index, %step: index) -> () {
   // expected-error @below {{an 'omp.wsloop' nested wrapper is only allowed when 'omp.parallel' is the direct parent}}
   omp.distribute {
@@ -2246,9 +2175,7 @@ func.func @omp_distribute_nested_wrapper(%lb: index, %ub: index, %step: index) -
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         "omp.yield"() : () -> ()
       }
-      "omp.terminator"() : () -> ()
     }) {omp.composite} : () -> ()
-    "omp.terminator"() : () -> ()
   } {omp.composite}
 }
 
@@ -2261,9 +2188,7 @@ func.func @omp_distribute_nested_wrapper2(%lb: index, %ub: index, %step: index)
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         "omp.yield"() : () -> ()
       }
-      "omp.terminator"() : () -> ()
     }) {omp.composite} : () -> ()
-    "omp.terminator"() : () -> ()
   } {omp.composite}
 }
 
@@ -2276,9 +2201,7 @@ func.func @omp_distribute_nested_wrapper3(%lb: index, %ub: index, %step: index)
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         "omp.yield"() : () -> ()
       }
-      "omp.terminator"() : () -> ()
     }) {omp.composite} : () -> ()
-    "omp.terminator"() : () -> ()
   }
 }
 
@@ -2500,9 +2423,7 @@ func.func @omp_parallel_missing_composite(%lb: index, %ub: index, %step: index)
         omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
           omp.yield
         }
-        omp.terminator
       } {omp.composite}
-      omp.terminator
     } {omp.composite}
     omp.terminator
   }
@@ -2517,7 +2438,6 @@ func.func @omp_parallel_invalid_composite(%lb: index, %ub: index, %step: index)
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   } {omp.composite}
@@ -2534,9 +2454,7 @@ func.func @omp_parallel_invalid_composite2(%lb: index, %ub: index, %step: index)
         omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
           omp.yield
         }
-        omp.terminator
       } {omp.composite}
-      omp.terminator
     } {omp.composite}
     omp.terminator
   } {omp.composite}
@@ -2551,9 +2469,7 @@ func.func @omp_wsloop_missing_composite(%lb: index, %ub: index, %step: index) ->
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   }
   return
 }
@@ -2565,7 +2481,6 @@ func.func @omp_wsloop_invalid_composite(%lb: index, %ub: index, %step: index) ->
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   } {omp.composite}
   return
 }
@@ -2579,9 +2494,7 @@ func.func @omp_wsloop_missing_composite_2(%lb: index, %ub: index, %step: index)
         omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
           omp.yield
         }
-        omp.terminator
       }
-      omp.terminator
     } {omp.composite}
     omp.terminator
   } {omp.composite}
@@ -2596,9 +2509,7 @@ func.func @omp_simd_missing_composite(%lb: index, %ub: index, %step: index) -> (
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     }
-    omp.terminator
   } {omp.composite}
   return
 }
@@ -2610,7 +2521,6 @@ func.func @omp_simd_invalid_composite(%lb: index, %ub: index, %step: index) -> (
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   } {omp.composite}
   return
 }
@@ -2624,9 +2534,7 @@ func.func @omp_distribute_missing_composite(%lb: index, %ub: index, %step: index
         omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
           omp.yield
         }
-        omp.terminator
       } {omp.composite}
-      omp.terminator
     }
     omp.terminator
   } {omp.composite}
@@ -2640,7 +2548,6 @@ func.func @omp_distribute_invalid_composite(%lb: index, %ub: index, %step: index
     omp.loop_nest (%0) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   } {omp.composite}
   return
 }
@@ -2653,9 +2560,7 @@ func.func @omp_taskloop_missing_composite(%lb: index, %ub: index, %step: index)
       omp.loop_nest (%i) : index = (%lb) to (%ub) step (%step)  {
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   }
   return
 }
@@ -2667,7 +2572,6 @@ func.func @omp_taskloop_invalid_composite(%lb: index, %ub: index, %step: index)
     omp.loop_nest (%i) : index = (%lb) to (%ub) step (%step)  {
       omp.yield
     }
-    omp.terminator
   } {omp.composite}
   return
 }
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index a4423782a723bfbb624f03e62920b324db25376e..6f11b451fa00a3c3ffb53f8ec130927d1b6bfee9 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -110,9 +110,7 @@ func.func @omp_parallel(%data_var : memref<i32>, %if_cond : i1, %num_threads : i
         omp.loop_nest (%iv) : index = (%idx) to (%idx) step (%idx) {
           omp.yield
         }
-        omp.terminator
       } {omp.composite}
-      omp.terminator
     } {omp.composite}
     omp.terminator
   } {omp.composite}
@@ -130,11 +128,8 @@ func.func @omp_parallel(%data_var : memref<i32>, %if_cond : i1, %num_threads : i
           omp.loop_nest (%iv) : index = (%idx) to (%idx) step (%idx) {
             omp.yield
           }
-          omp.terminator
         } {omp.composite}
-        omp.terminator
       } {omp.composite}
-      omp.terminator
     } {omp.composite}
     omp.terminator
   } {omp.composite}
@@ -197,7 +192,6 @@ func.func @omp_loop_nest(%lb : index, %ub : index, %step : index) -> () {
     ^bb0(%iv: index):
       omp.yield
     }) : (index, index, index) -> ()
-    omp.terminator
   }
 
   omp.wsloop {
@@ -208,7 +202,6 @@ func.func @omp_loop_nest(%lb : index, %ub : index, %step : index) -> () {
     ^bb0(%iv: index):
       omp.yield
     }) {loop_inclusive} : (index, index, index) -> ()
-    omp.terminator
   }
 
   omp.wsloop {
@@ -219,7 +212,6 @@ func.func @omp_loop_nest(%lb : index, %ub : index, %step : index) -> () {
     ^bb0(%iv: index, %iv3: index):
       omp.yield
     }) : (index, index, index, index, index, index) -> ()
-    omp.terminator
   }
 
   omp.wsloop {
@@ -235,7 +227,6 @@ func.func @omp_loop_nest(%lb : index, %ub : index, %step : index) -> () {
       // CHECK: omp.yield
       omp.yield
     }) : (index, index, index) -> ()
-    omp.terminator
   }
 
   return
@@ -250,7 +241,6 @@ func.func @omp_loop_nest_pretty(%lb : index, %ub : index, %step : index) -> () {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop {
@@ -260,7 +250,6 @@ func.func @omp_loop_nest_pretty(%lb : index, %ub : index, %step : index) -> () {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) inclusive step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop {
@@ -270,7 +259,6 @@ func.func @omp_loop_nest_pretty(%lb : index, %ub : index, %step : index) -> () {
     omp.loop_nest (%iv1, %iv2) : index = (%lb, %lb) to (%ub, %ub) step (%step, %step) {
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop {
@@ -285,7 +273,6 @@ func.func @omp_loop_nest_pretty(%lb : index, %ub : index, %step : index) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   return
@@ -304,7 +291,6 @@ func.func @omp_loop_nest_pretty_multi_block(%lb : index, %ub : index,
       memref.store %arg, %data1[%iv] : memref<?xi32>
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop {
@@ -322,7 +308,6 @@ func.func @omp_loop_nest_pretty_multi_block(%lb : index, %ub : index,
     ^bb3:
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop {
@@ -338,7 +323,6 @@ func.func @omp_loop_nest_pretty_multi_block(%lb : index, %ub : index,
       memref.store %arg1, %data2[%iv] : memref<?xi32>
       omp.yield
     }
-    omp.terminator
   }
 
   return
@@ -358,7 +342,6 @@ func.func @omp_loop_nest_pretty_non_index(%lb1 : i32, %ub1 : i32, %step1 : i32,
       memref.store %iv1, %data1[%arg1] : memref<?xi32>
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop {
@@ -370,7 +353,6 @@ func.func @omp_loop_nest_pretty_non_index(%lb1 : i32, %ub1 : i32, %step1 : i32,
       memref.store %iv, %data2[%arg2] : memref<?xi64>
       omp.yield
     }
-    omp.terminator
   }
 
   return
@@ -389,7 +371,6 @@ func.func @omp_loop_nest_pretty_multiple(%lb1 : i32, %ub1 : i32, %step1 : i32,
       memref.store %iv2, %data1[%2] : memref<?xi32>
       omp.yield
     }
-    omp.terminator
   }
 
   return
@@ -404,7 +385,6 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0,0,0,0,0,0,0>, ordered = 1} :
     () -> ()
 
@@ -414,7 +394,6 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0,0,1,1,0,0,0>, schedule_kind = #omp<schedulekind static>} :
     (memref<i32>, i32) -> ()
 
@@ -424,7 +403,6 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0,0,2,2,0,0,0>, schedule_kind = #omp<schedulekind static>} :
     (memref<i32>, memref<i32>, i32, i32) -> ()
 
@@ -434,7 +412,6 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0,0,1,1,0,0,1>, schedule_kind = #omp<schedulekind dynamic>, ordered = 2} :
     (memref<i32>, i32, i32) -> ()
 
@@ -444,7 +421,6 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }) {operandSegmentSizes = array<i32: 0,0,0,0,0,0,0>, nowait, schedule_kind = #omp<schedulekind auto>} :
     () -> ()
 
@@ -456,9 +432,7 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   }) {omp.composite} : () -> ()
 
   return
@@ -473,7 +447,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop linear(%{{.*}} = %{{.*}} : memref<i32>) schedule(static) {
@@ -482,7 +455,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop linear(%{{.*}} = %{{.*}} : memref<i32>) ordered(2) schedule(static = %{{.*}} : i32) {
@@ -491,7 +463,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop linear(%{{.*}} = %{{.*}} : memref<i32>) ordered(2) schedule(dynamic = %{{.*}} : i32, nonmonotonic) {
@@ -500,7 +471,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step)  {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop linear(%{{.*}} = %{{.*}} : memref<i32>) ordered(2) schedule(dynamic = %{{.*}} : i16, monotonic) {
@@ -509,7 +479,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop {
@@ -518,7 +487,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop nowait {
@@ -527,7 +495,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop nowait order(concurrent) {
@@ -536,7 +503,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.wsloop nowait order(reproducible:concurrent) {
@@ -545,7 +511,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.wsloop nowait order(unconstrained:concurrent) {
   // CHECK-NEXT: omp.loop_nest
@@ -553,7 +518,6 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.wsloop {
   // CHECK-NEXT: omp.simd
@@ -563,9 +527,7 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
       omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   } {omp.composite}
 
   return
@@ -579,7 +541,6 @@ func.func @omp_simd(%lb : index, %ub : index, %step : index) -> () {
     ^bb1(%iv2: index):
       "omp.yield"() : () -> ()
     }) : (index, index, index) -> ()
-    "omp.terminator"() : () -> ()
   }
 
   return
@@ -596,7 +557,6 @@ func.func @omp_simd_aligned_list(%arg0 : index, %arg1 : index, %arg2 : index,
     ^bb1(%iv2: index):
       "omp.yield"() : () -> ()
     }) : (index, index, index) -> ()
-    "omp.terminator"() : () -> ()
   }) {alignments = [32, 128],
       operandSegmentSizes = array<i32: 2, 0, 0, 0, 0, 0, 0>} : (memref<i32>, memref<i32>) -> ()
   return
@@ -611,7 +571,6 @@ func.func @omp_simd_aligned_single(%arg0 : index, %arg1 : index, %arg2 : index,
     ^bb1(%iv2: index):
       "omp.yield"() : () -> ()
     }) : (index, index, index) -> ()
-    "omp.terminator"() : () -> ()
   }) {alignments = [32],
       operandSegmentSizes = array<i32: 1, 0, 0, 0, 0, 0, 0>} : (memref<i32>) -> ()
   return
@@ -627,7 +586,6 @@ func.func @omp_simd_nontemporal_list(%arg0 : index, %arg1 : index,
     ^bb1(%iv2: index):
       "omp.yield"() : () -> ()
     }) : (index, index, index) -> ()
-    "omp.terminator"() : () -> ()
   }) {operandSegmentSizes = array<i32: 0, 0, 0, 0, 2, 0, 0>} : (memref<i32>, memref<i64>) -> ()
   return
 }
@@ -642,7 +600,6 @@ func.func @omp_simd_nontemporal_single(%arg0 : index, %arg1 : index,
     ^bb1(%iv2: index):
       "omp.yield"() : () -> ()
     }) : (index, index, index) -> ()
-    "omp.terminator"() : () -> ()
   }) {operandSegmentSizes = array<i32: 0, 0, 0, 0, 1, 0, 0>} : (memref<i32>) -> ()
   return
 }
@@ -654,7 +611,6 @@ func.func @omp_simd_pretty(%lb : index, %ub : index, %step : index) -> () {
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -670,7 +626,6 @@ func.func @omp_simd_pretty_aligned(%lb : index, %ub : index, %step : index,
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -682,7 +637,6 @@ func.func @omp_simd_pretty_if(%lb : index, %ub : index, %step : index, %if_cond
     omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -696,7 +650,6 @@ func.func @omp_simd_pretty_nontemporal(%lb : index, %ub : index, %step : index,
     omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -708,21 +661,18 @@ func.func @omp_simd_pretty_order(%lb : index, %ub : index, %step : index) -> ()
     omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.simd order(reproducible:concurrent)
   omp.simd order(reproducible:concurrent) {
     omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.simd order(unconstrained:concurrent)
   omp.simd order(unconstrained:concurrent) {
     omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -734,7 +684,6 @@ func.func @omp_simd_pretty_simdlen(%lb : index, %ub : index, %step : index) -> (
     omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -746,7 +695,6 @@ func.func @omp_simd_pretty_safelen(%lb : index, %ub : index, %step : index) -> (
     omp.loop_nest (%iv): index = (%lb) to (%ub) step (%step) {
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -759,56 +707,48 @@ func.func @omp_distribute(%chunk_size : i32, %data_var : memref<i32>, %arg0 : i3
     ^bb0(%iv: i32):
       "omp.yield"() : () -> ()
     }) : (i32, i32, i32) -> ()
-    "omp.terminator"() : () -> ()
   }) {} : () -> ()
   // CHECK: omp.distribute
   omp.distribute {
     omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.distribute dist_schedule_static
   omp.distribute dist_schedule_static {
     omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.distribute dist_schedule_static dist_schedule_chunk_size(%{{.+}} : i32)
   omp.distribute dist_schedule_static dist_schedule_chunk_size(%chunk_size : i32) {
     omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.distribute order(concurrent)
   omp.distribute order(concurrent) {
     omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.distribute order(reproducible:concurrent)
   omp.distribute order(reproducible:concurrent) {
     omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.distribute order(unconstrained:concurrent)
   omp.distribute order(unconstrained:concurrent) {
     omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.distribute allocate(%{{.+}} : memref<i32> -> %{{.+}} : memref<i32>)
   omp.distribute allocate(%data_var : memref<i32> -> %data_var : memref<i32>) {
     omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
       omp.yield
     }
-    omp.terminator
   }
   // CHECK: omp.distribute
   omp.distribute {
@@ -816,9 +756,7 @@ func.func @omp_distribute(%chunk_size : i32, %data_var : memref<i32>, %arg0 : i3
       omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   } {omp.composite}
   return
 }
@@ -958,7 +896,6 @@ func.func @wsloop_reduction(%lb : index, %ub : index, %step : index) {
       llvm.store %res, %prv :  f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -980,7 +917,6 @@ func.func @wsloop_reduction_byref(%lb : index, %ub : index, %step : index) {
       llvm.store %res, %prv :  f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -1034,8 +970,6 @@ func.func @parallel_wsloop_reduction(%lb : index, %ub : index, %step : index) {
         // CHECK: omp.yield
         omp.yield
       }
-      // CHECK: omp.terminator
-      omp.terminator
     }
     // CHECK: omp.terminator
     omp.terminator
@@ -1197,7 +1131,6 @@ func.func @wsloop_reduction2(%lb : index, %ub : index, %step : index) {
       memref.store %4, %prv[%2] : memref<1xf32>
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -1235,8 +1168,6 @@ func.func @parallel_wsloop_reduction2(%lb : index, %ub : index, %step : index) {
         // CHECK: omp.yield
         omp.yield
       }
-      // CHECK: omp.terminator
-      omp.terminator
     }
     // CHECK: omp.terminator
     omp.terminator
@@ -1317,7 +1248,6 @@ func.func @omp_ordered(%arg1 : i32, %arg2 : i32, %arg3 : i32,
       }
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop ordered(1) {
@@ -1331,7 +1261,6 @@ func.func @omp_ordered(%arg1 : i32, %arg2 : i32, %arg3 : i32,
 
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop ordered(2) {
@@ -1345,7 +1274,6 @@ func.func @omp_ordered(%arg1 : i32, %arg2 : i32, %arg3 : i32,
 
       omp.yield
     }
-    omp.terminator
   }
 
   return
@@ -2213,7 +2141,6 @@ func.func @omp_cancel_wsloop(%lb : index, %ub : index, %step : index) {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -2252,7 +2179,6 @@ func.func @omp_cancellationpoint_wsloop(%lb : index, %ub : index, %step : index)
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
   return
 }
@@ -2340,7 +2266,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   %testbool = "test.bool"() : () -> (i1)
@@ -2351,7 +2276,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop final(%{{[^)]+}}) {
@@ -2360,7 +2284,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop untied {
@@ -2369,7 +2292,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop mergeable {
@@ -2378,7 +2300,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
@@ -2389,7 +2310,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // Checking byref attribute for in_reduction
@@ -2399,7 +2319,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop reduction(byref @add_f32 %{{.+}} -> %{{.+}}, @add_f32 %{{.+}} -> %{{.+}} : !llvm.ptr, !llvm.ptr) {
@@ -2408,7 +2327,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // check byref attrbute for reduction
@@ -2418,7 +2336,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop in_reduction(@add_f32 %{{.+}} -> %{{.+}} : !llvm.ptr) reduction(@add_f32 %{{.+}} -> %{{.+}} : !llvm.ptr) {
@@ -2427,7 +2344,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   %testi32 = "test.i32"() : () -> (i32)
@@ -2437,7 +2353,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   %testmemref = "test.memref"() : () -> (memref<i32>)
@@ -2447,7 +2362,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   %testi64 = "test.i64"() : () -> (i64)
@@ -2457,7 +2371,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop num_tasks(%{{[^:]+}}: i64) {
@@ -2466,7 +2379,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop nogroup {
@@ -2475,7 +2387,6 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
       // CHECK: omp.yield
       omp.yield
     }
-    omp.terminator
   }
 
   // CHECK: omp.taskloop {
@@ -2485,9 +2396,7 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
         // CHECK: omp.yield
         omp.yield
       }
-      omp.terminator
     } {omp.composite}
-    omp.terminator
   } {omp.composite}
 
   // CHECK: return
diff --git a/mlir/test/Dialect/SCF/loop-range.mlir b/mlir/test/Dialect/SCF/loop-range.mlir
index cd3b4861fc18d138fb70a07d105e3b9da20da763..e50db8ae9f35b736297422e00e35eb0eee4d0430 100644
--- a/mlir/test/Dialect/SCF/loop-range.mlir
+++ b/mlir/test/Dialect/SCF/loop-range.mlir
@@ -21,12 +21,13 @@ func.func @fold_one_loop(%arg0: memref<?xi32>, %arg1: index, %arg2: index) {
 // CHECK:       %[[C4:.*]] = arith.constant 4 : index
 // CHECK:       %[[I0:.*]] = arith.addi %[[ARG2]], %[[C0]] : index
 // CHECK:       %[[I1:.*]] = arith.addi %[[ARG2]], %[[ARG1]] : index
-// CHECK:       %[[I2:.*]] = arith.muli %[[I1]], %[[C4]] : index
-// CHECK:       %[[I3:.*]] = arith.muli %[[C1]], %[[C4]] : index
-// CHECK:       scf.for %[[I:.*]] = %[[I0]] to %[[I2]] step %[[I3]] {
-// CHECK:         %[[I4:.*]] = memref.load %[[ARG0]]{{\[}}%[[I]]
-// CHECK:         %[[I5:.*]] = arith.muli %[[I4]], %[[I4]] : i32
-// CHECK:         memref.store %[[I5]], %[[ARG0]]{{\[}}%[[I]]
+// CHECK:       %[[I2:.*]] = arith.muli %[[I0]], %[[C4]] : index
+// CHECK:       %[[I3:.*]] = arith.muli %[[I1]], %[[C4]] : index
+// CHECK:       %[[I4:.*]] = arith.muli %[[C1]], %[[C4]] : index
+// CHECK:       scf.for %[[I:.*]] = %[[I2]] to %[[I3]] step %[[I4]] {
+// CHECK:         %[[I5:.*]] = memref.load %[[ARG0]]{{\[}}%[[I]]
+// CHECK:         %[[I6:.*]] = arith.muli %[[I5]], %[[I5]] : i32
+// CHECK:         memref.store %[[I6]], %[[ARG0]]{{\[}}%[[I]]
 
 func.func @fold_one_loop2(%arg0: memref<?xi32>, %arg1: index, %arg2: index) {
   %c0 = arith.constant 0 : index
@@ -54,12 +55,13 @@ func.func @fold_one_loop2(%arg0: memref<?xi32>, %arg1: index, %arg2: index) {
 // CHECK:       scf.for %[[J:.*]] = %[[C0]] to %[[C10]] step %[[C1]] {
 // CHECK:         %[[I0:.*]] = arith.addi %[[ARG2]], %[[C0]] : index
 // CHECK:         %[[I1:.*]] = arith.addi %[[ARG2]], %[[ARG1]] : index
-// CHECK:         %[[I2:.*]] = arith.muli %[[I1]], %[[C4]] : index
-// CHECK:         %[[I3:.*]] = arith.muli %[[C1]], %[[C4]] : index
-// CHECK:         scf.for %[[I:.*]] = %[[I0]] to %[[I2]] step %[[I3]] {
-// CHECK:           %[[I4:.*]] = memref.load %[[ARG0]]{{\[}}%[[I]]
-// CHECK:           %[[I5:.*]] = arith.muli %[[I4]], %[[I4]] : i32
-// CHECK:           memref.store %[[I5]], %[[ARG0]]{{\[}}%[[I]]
+// CHECK:         %[[I2:.*]] = arith.muli %[[I0]], %[[C4]] : index
+// CHECK:         %[[I3:.*]] = arith.muli %[[I1]], %[[C4]] : index
+// CHECK:         %[[I4:.*]] = arith.muli %[[C1]], %[[C4]] : index
+// CHECK:         scf.for %[[I:.*]] = %[[I2]] to %[[I3]] step %[[I4]] {
+// CHECK:           %[[I5:.*]] = memref.load %[[ARG0]]{{\[}}%[[I]]
+// CHECK:           %[[I6:.*]] = arith.muli %[[I5]], %[[I5]] : i32
+// CHECK:           memref.store %[[I6]], %[[ARG0]]{{\[}}%[[I]]
 
 func.func @fold_two_loops(%arg0: memref<?xi32>, %arg1: index, %arg2: index) {
   %c0 = arith.constant 0 : index
@@ -86,14 +88,17 @@ func.func @fold_two_loops(%arg0: memref<?xi32>, %arg1: index, %arg2: index) {
 // CHECK:       %[[C10:.*]] = arith.constant 10 : index
 // CHECK:       %[[I0:.*]] = arith.addi %[[ARG2]], %[[C0]] : index
 // CHECK:       %[[I1:.*]] = arith.addi %[[ARG2]], %[[C10]] : index
-// CHECK:       scf.for %[[J:.*]] = %[[I0]] to %[[I1]] step %[[C1]] {
-// CHECK:         %[[I1:.*]] = arith.addi %[[ARG2]], %[[ARG1]] : index
-// CHECK:         %[[I2:.*]] = arith.muli %[[I1]], %[[C4]] : index
-// CHECK:         %[[I3:.*]] = arith.muli %[[C1]], %[[C4]] : index
-// CHECK:         scf.for %[[I:.*]] = %[[J]] to %[[I2]] step %[[I3]] {
-// CHECK:           %[[I4:.*]] = memref.load %[[ARG0]]{{\[}}%[[I]]
-// CHECK:           %[[I5:.*]] = arith.muli %[[I4]], %[[I4]] : i32
-// CHECK:           memref.store %[[I5]], %[[ARG0]]{{\[}}%[[I]]
+// CHECK:       %[[I2:.*]] = arith.muli %[[I0]], %[[C4]] : index
+// CHECK:       %[[I3:.*]] = arith.muli %[[I1]], %[[C4]] : index
+// CHECK:       %[[I4:.*]] = arith.muli %[[C1]], %[[C4]] : index
+// CHECK:       scf.for %[[J:.*]] = %[[I2]] to %[[I3]] step %[[I4]] {
+// CHECK:         %[[I5:.*]] = arith.addi %[[ARG2]], %[[ARG1]] : index
+// CHECK:         %[[I6:.*]] = arith.muli %[[I5]], %[[C4]] : index
+// CHECK:         %[[I7:.*]] = arith.muli %[[C1]], %[[C4]] : index
+// CHECK:         scf.for %[[I:.*]] = %[[J]] to %[[I6]] step %[[I7]] {
+// CHECK:           %[[I8:.*]] = memref.load %[[ARG0]]{{\[}}%[[I]]
+// CHECK:           %[[I9:.*]] = arith.muli %[[I8]], %[[I8]] : i32
+// CHECK:           memref.store %[[I9]], %[[ARG0]]{{\[}}%[[I]]
 
 // If an instruction's operands are not defined outside the loop, we cannot
 // perform the optimization, as is the case with the arith.muli below. (If
diff --git a/mlir/test/Dialect/SparseTensor/codegen.mlir b/mlir/test/Dialect/SparseTensor/codegen.mlir
index df03d871ba3a3e9772c91d4001893f1c5763ff42..af78458f1093298099b9a0bb8dd23ac30b49e0f7 100644
--- a/mlir/test/Dialect/SparseTensor/codegen.mlir
+++ b/mlir/test/Dialect/SparseTensor/codegen.mlir
@@ -826,19 +826,3 @@ func.func @sparse_new_coo_permute_no(%arg0: !llvm.ptr) -> tensor<?x?xf32, #CooPN
   %0 = sparse_tensor.new %arg0 : !llvm.ptr to tensor<?x?xf32, #CooPNo>
   return %0 : tensor<?x?xf32, #CooPNo>
 }
-
-// CHECK-LABEL: func.func @test_tensor_dim_unranked
-//       CHECK: tensor.dim
-func.func @test_tensor_dim_unranked(%arg0: tensor<*xf32>) -> index {
-  %c = arith.constant 0 : index
-  %0 = tensor.dim %arg0, %c : tensor<*xf32>
-  return %0 : index
-}
-
-// CHECK-LABEL: func.func @test_tensor_reshape_unranked
-//       CHECK: tensor.reshape
-func.func @test_tensor_reshape_unranked(%src: tensor<*xf32>, %shape: tensor<1xi32>) -> tensor<?xf32> {
-  %dst = tensor.reshape %src(%shape)
-         : (tensor<*xf32>, tensor<1xi32>) -> tensor<?xf32>
-  return %dst : tensor<?xf32>
-}
diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir
index 737b736ba795fe39d961b5e7c1d9f503d3a36164..908d2d8aa83f7c3a0c7910c5b9294b1c530a1f94 100644
--- a/mlir/test/Dialect/SparseTensor/invalid.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid.mlir
@@ -105,7 +105,7 @@ func.func @invalid_positions_dense(%arg0: tensor<128xf64>) -> memref<?xindex> {
 
 func.func @invalid_positions_unranked(%arg0: tensor<*xf64>) -> memref<?xindex> {
   // expected-error@+1 {{'sparse_tensor.positions' op operand #0 must be sparse tensor of any type values, but got 'tensor<*xf64>'}}
-  %0 = sparse_tensor.positions %arg0 { level = 0 : index } : tensor<*xf64> to memref<?xindex>
+  %0 = "sparse_tensor.positions"(%arg0) { level = 0 : index } : (tensor<*xf64>) -> (memref<?xindex>)
   return %0 : memref<?xindex>
 }
 
@@ -141,7 +141,7 @@ func.func @invalid_indices_dense(%arg0: tensor<10x10xi32>) -> memref<?xindex> {
 
 func.func @invalid_indices_unranked(%arg0: tensor<*xf64>) -> memref<?xindex> {
   // expected-error@+1 {{'sparse_tensor.coordinates' op operand #0 must be sparse tensor of any type values, but got 'tensor<*xf64>'}}
-  %0 = sparse_tensor.coordinates %arg0 { level = 0 : index } : tensor<*xf64> to memref<?xindex>
+  %0 = "sparse_tensor.coordinates"(%arg0) { level = 0 : index } : (tensor<*xf64>) -> (memref<?xindex>)
   return %0 : memref<?xindex>
 }
 
@@ -347,7 +347,7 @@ func.func @sparse_wrong_arity_compression(%arg0: memref<?xf64>,
 // -----
 
 func.func @sparse_convert_unranked(%arg0: tensor<*xf32>) -> tensor<10xf32> {
-  // expected-error@+1 {{unexpected type in convert}}
+  // expected-error@+1 {{invalid kind of type specified}}
   %0 = sparse_tensor.convert %arg0 : tensor<*xf32> to tensor<10xf32>
   return %0 : tensor<10xf32>
 }
diff --git a/mlir/test/Dialect/SparseTensor/no_lowering.mlir b/mlir/test/Dialect/SparseTensor/no_lowering.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..4f21055a13d58a8cb283d671a85c51a133563525
--- /dev/null
+++ b/mlir/test/Dialect/SparseTensor/no_lowering.mlir
@@ -0,0 +1,54 @@
+// RUN: mlir-opt %s --lower-sparse-ops-to-foreach --split-input-file | FileCheck %s
+
+// Ensure that we exit gracefully rather than crashing.
+
+// CHECK-LABEL: func.func @test_tensor_dim_unranked
+//       CHECK: tensor.dim
+func.func @test_tensor_dim_unranked(%arg0: tensor<*xf32>) -> index {
+  %c = arith.constant 0 : index
+  %0 = tensor.dim %arg0, %c : tensor<*xf32>
+  return %0 : index
+}
+
+// -----
+
+#SparseVector = #sparse_tensor.encoding<{
+  map = (d0) -> (d0 : compressed)
+}>
+
+// CHECK-LABEL: func.func @test_no_constant_dim
+//       CHECK: tensor.dim
+func.func @test_no_constant_dim(%arg0: tensor<?xf64, #SparseVector>, %arg1: index) -> index {
+  %0 = tensor.dim %arg0, %arg1 : tensor<?xf64, #SparseVector>
+  return %0 : index
+}
+
+// -----
+
+// CHECK-LABEL: func.func @test_tensor_dim_no_encoding
+//       CHECK: tensor.dim
+func.func @test_tensor_dim_no_encoding(%arg0: tensor<?xf32>) -> index {
+  %c = arith.constant 0 : index
+  %0 = tensor.dim %arg0, %c : tensor<?xf32>
+  return %0 : index
+}
+
+// -----
+
+// CHECK-LABEL: func.func @test_tensor_reshape_unranked
+//       CHECK: tensor.reshape
+func.func @test_tensor_reshape_unranked(%src: tensor<*xf32>, %shape: tensor<1xi32>) -> tensor<?xf32> {
+  %dst = tensor.reshape %src(%shape)
+         : (tensor<*xf32>, tensor<1xi32>) -> tensor<?xf32>
+  return %dst : tensor<?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func.func @test_tensor_reshape_no_encoding
+//       CHECK: tensor.reshape
+func.func @test_tensor_reshape_no_encoding(%src: tensor<?x?xf32>, %shape: tensor<1xi32>) -> tensor<?xf32> {
+  %dst = tensor.reshape %src(%shape)
+         : (tensor<?x?xf32>, tensor<1xi32>) -> tensor<?xf32>
+  return %dst : tensor<?xf32>
+}
diff --git a/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir b/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir
index 5c11d80178f727510d299bfc2ff2c20ebc07c2dd..c70c940564a264e2bbf1c1a2c5083addfbc7e845 100644
--- a/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir
+++ b/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir
@@ -18,7 +18,7 @@ module attributes {gpu.container_module} {
     return
   }
   gpu.module @gpumodule {
-    gpu.func @kernel_cluster() kernel attributes {gpu.known_block_size = array<i32: 1, 1, 1>, gpu.known_grid_size = array<i32: 2, 2, 1>} {
+    gpu.func @kernel_cluster() kernel attributes {gpu.known_block_size = array<i32: 1, 1, 1>, gpu.known_grid_size = array<i32: 4, 4, 1>} {
       %cidX = gpu.cluster_id  x
       %cidY = gpu.cluster_id  y
       %cidZ = gpu.cluster_id  z
diff --git a/mlir/test/Target/LLVMIR/Import/commandline.ll b/mlir/test/Target/LLVMIR/Import/commandline.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f31aec8cc5aac1b0f40765c249bd8cd6db961b21
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/commandline.ll
@@ -0,0 +1,6 @@
+; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s
+
+; CHECK: module attributes {
+; CHECK-SAME: llvm.commandline = "exec -o infile"
+!llvm.commandline = !{!0}
+!0 = !{!"exec -o infile"}
diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll
index 09909d7d63b2ab52c9c6c41082fc28448e225bdb..9ef1942e0787c67d2e64e15d33cb49c14ca4c7d7 100644
--- a/mlir/test/Target/LLVMIR/Import/debug-info.ll
+++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll
@@ -867,3 +867,27 @@ define void @test() !dbg !3 {
 ; CHECK: #[[FILE:.+]] = #llvm.di_file<"test.f90" in "">
 ; CHECK: #[[SP:.+]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
 ; CHECK: #llvm.di_common_block<scope = #[[SP]], name = "block", file = #[[FILE]], line = 3>
+
+; // -----
+
+@data = external global i64, !dbg !0, !dbg !5
+
+!llvm.module.flags = !{!8}
+!llvm.dbg.cu = !{!2}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !7)
+!2 = distinct !DICompileUnit(language: DW_LANG_C, file: !3, globals: !4)
+!3 = !DIFile(filename: "test.c", directory: "")
+!4 = !{!0, !5}
+!5 = !DIGlobalVariableExpression(var: !6, expr: !DIExpression())
+!6 = distinct !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 3, type: !7)
+!7 = !DIBasicType(name: "int", size: 32)
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+
+
+; CHECK: #[[VAR1:.+]] =  #llvm.di_global_variable<{{.*}}name = "a"{{.*}}>
+; CHECK: #[[VAR2:.+]] =  #llvm.di_global_variable<{{.*}}name = "b"{{.*}}>
+; CHECK: #[[EXP1:.+]] =  #llvm.di_global_variable_expression<var = #[[VAR1]], expr = <>>
+; CHECK: #[[EXP2:.+]] =  #llvm.di_global_variable_expression<var = #[[VAR2]], expr = <>>
+; CHECK: llvm.mlir.global external @data() {{{.*}}dbg_exprs = [#[[EXP1]], #[[EXP2]]]} : i64
diff --git a/mlir/test/Target/LLVMIR/Import/global-variables.ll b/mlir/test/Target/LLVMIR/Import/global-variables.ll
index 879e9135fe4ece670fb972b3b91bbdef2c18369b..fbeda4cd42af82b856c281d32852e91206c67b84 100644
--- a/mlir/test/Target/LLVMIR/Import/global-variables.ll
+++ b/mlir/test/Target/LLVMIR/Import/global-variables.ll
@@ -274,8 +274,8 @@ define void @bar() {
 ; CHECK-DAG: #[[GVAR1:.*]] = #llvm.di_global_variable<scope = #[[SPROG]], name = "bar", linkageName = "bar", file = #[[FILE]], line = 8, type = #[[TYPE]], isLocalToUnit = true>
 ; CHECK-DAG: #[[EXPR0:.*]] = #llvm.di_global_variable_expression<var = #[[GVAR0]], expr = <[DW_OP_LLVM_fragment(0, 16)]>>
 ; CHECK-DAG: #[[EXPR1:.*]] = #llvm.di_global_variable_expression<var = #[[GVAR1]], expr = <[DW_OP_constu(3), DW_OP_plus]>>
-; CHECK-DAG: llvm.mlir.global external @foo() {addr_space = 0 : i32, alignment = 8 : i64, dbg_expr = #[[EXPR0]]} : i32
-; CHECK-DAG: llvm.mlir.global external @bar() {addr_space = 0 : i32, alignment = 8 : i64, dbg_expr = #[[EXPR1]]} : i32
+; CHECK-DAG: llvm.mlir.global external @foo() {addr_space = 0 : i32, alignment = 8 : i64, dbg_exprs = [#[[EXPR0]]]} : i32
+; CHECK-DAG: llvm.mlir.global external @bar() {addr_space = 0 : i32, alignment = 8 : i64, dbg_exprs = [#[[EXPR1]]]} : i32
 
 @foo = external global i32, align 8, !dbg !5
 @bar = external global i32, align 8, !dbg !7
@@ -308,7 +308,7 @@ define void @bar() {
 ; CHECK:  llvm.mlir.global internal constant @one() {addr_space = 0 : i32, dso_local} : !llvm.ptr {
 ; CHECK:    llvm.mlir.addressof @mlir.llvm.nameless_global_3 : !llvm.ptr
 
-; CHECK: llvm.mlir.global external constant @".str.1"() {addr_space = 0 : i32, dbg_expr = #[[GLOBAL_VAR_EXPR]]}
+; CHECK: llvm.mlir.global external constant @".str.1"() {addr_space = 0 : i32, dbg_exprs = [#[[GLOBAL_VAR_EXPR]]]}
 
 @0 = private unnamed_addr constant [2 x i8] c"0\00"
 @1 = private unnamed_addr constant [2 x i8] c"1\00"
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 2fc2c3c6c32ffa9c9edab84c51ce5f6e5505017d..28a1bd21c82a38d75d8d0b028c68eb3deefadb84 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -109,6 +109,24 @@ define void @cos_test(float %0, <8 x float> %1) {
   %4 = call <8 x float> @llvm.cos.v8f32(<8 x float> %1)
   ret void
 }
+; CHECK-LABEL:  llvm.func @hyperbolic_trig_test
+define void @hyperbolic_trig_test(float %0, <8 x float> %1) {
+  ; CHECK: llvm.intr.sinh(%{{.*}}) : (f32) -> f32
+  %3 = call float @llvm.sinh.f32(float %0)
+  ; CHECK: llvm.intr.sinh(%{{.*}}) : (vector<8xf32>) -> vector<8xf32>
+  %4 = call <8 x float> @llvm.sinh.v8f32(<8 x float> %1)
+
+  ; CHECK: llvm.intr.cosh(%{{.*}}) : (f32) -> f32
+  %5 = call float @llvm.cosh.f32(float %0)
+  ; CHECK: llvm.intr.cosh(%{{.*}}) : (vector<8xf32>) -> vector<8xf32>
+  %6 = call <8 x float> @llvm.cosh.v8f32(<8 x float> %1)
+
+  ; CHECK: llvm.intr.tanh(%{{.*}}) : (f32) -> f32
+  %7 = call float @llvm.tanh.f32(float %0)
+  ; CHECK: llvm.intr.tanh(%{{.*}}) : (vector<8xf32>) -> vector<8xf32>
+  %8 = call <8 x float> @llvm.tanh.v8f32(<8 x float> %1)
+  ret void
+}
 
 ; CHECK-LABEL:  llvm.func @copysign_test
 define void @copysign_test(float %0, float %1, <8 x float> %2, <8 x float> %3) {
@@ -959,6 +977,12 @@ declare float @llvm.floor.f32(float)
 declare <8 x float> @llvm.floor.v8f32(<8 x float>)
 declare float @llvm.cos.f32(float)
 declare <8 x float> @llvm.cos.v8f32(<8 x float>)
+declare float @llvm.sinh.f32(float)
+declare <8 x float> @llvm.sinh.v8f32(<8 x float>)
+declare float @llvm.cosh.f32(float)
+declare <8 x float> @llvm.cosh.v8f32(<8 x float>)
+declare float @llvm.tanh.f32(float)
+declare <8 x float> @llvm.tanh.v8f32(<8 x float>)
 declare float @llvm.copysign.f32(float, float)
 declare <8 x float> @llvm.copysign.v8f32(<8 x float>, <8 x float>)
 declare float @llvm.pow.f32(float, float)
diff --git a/mlir/test/Target/LLVMIR/commandline.mlir b/mlir/test/Target/LLVMIR/commandline.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..817a5ac2164e9f283ed5ee2a168127b507c22f5f
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/commandline.mlir
@@ -0,0 +1,6 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK: !llvm.commandline = !{![[S:[0-9]+]]}
+// CHECK: ![[S]] = !{!"exec -o infile"}
+module attributes {llvm.commandline = "exec -o infile"} {
+}
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index 826fda60c5efef1f214e8bf635026187aefcc776..b97bdcbc772d8072c850bceffd9be61ac8a7066d 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -327,8 +327,8 @@ llvm.func @dbg_intrinsics_with_no_location(%arg0: i32) -> (i32) {
 #di_file_2 = #llvm.di_file<"not" in "existence">
 #di_compile_unit_2 = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_C, file = #di_file_2, producer = "MLIR", isOptimized = true, emissionKind = Full>
 #di_basic_type_2 = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "uint64_t", sizeInBits = 64, encoding = DW_ATE_unsigned>
-llvm.mlir.global external @global_with_expr_1() {addr_space = 0 : i32, dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_compile_unit_2, name = "global_with_expr_1", linkageName = "global_with_expr_1", file = #di_file_2, line = 370, type = #di_basic_type_2>, expr = <>>} : i64
-llvm.mlir.global external @global_with_expr_2() {addr_space = 0 : i32, dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_compile_unit_2, name = "global_with_expr_2", linkageName = "global_with_expr_2", file = #di_file_2, line = 371, type = #di_basic_type_2, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <>>} : i64
+llvm.mlir.global external @global_with_expr_1() {addr_space = 0 : i32, dbg_exprs = [#llvm.di_global_variable_expression<var = <scope = #di_compile_unit_2, name = "global_with_expr_1", linkageName = "global_with_expr_1", file = #di_file_2, line = 370, type = #di_basic_type_2>, expr = <>>]} : i64
+llvm.mlir.global external @global_with_expr_2() {addr_space = 0 : i32, dbg_exprs = [#llvm.di_global_variable_expression<var = <scope = #di_compile_unit_2, name = "global_with_expr_2", linkageName = "global_with_expr_2", file = #di_file_2, line = 371, type = #di_basic_type_2, isLocalToUnit = true, isDefined = true, alignInBits = 8>, expr = <>>]} : i64
 
 // -----
 
@@ -347,7 +347,7 @@ llvm.mlir.global external @global_with_expr_2() {addr_space = 0 : i32, dbg_expr
 #di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "MLIR", isOptimized = true, emissionKind = Full>
 #di_basic_type = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
 #di_module = #llvm.di_module<file = #di_file, scope = #di_compile_unit, name = "module2", configMacros = "", includePath = "", apinotes = "", line = 120, isDecl = false >
-llvm.mlir.global external @module_global() {dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_module, name = "module_global", linkageName = "module_global", file = #di_file, line = 121, type = #di_basic_type, isLocalToUnit = false, isDefined = true>, expr = <>>} : i64
+llvm.mlir.global external @module_global() {dbg_exprs = [#llvm.di_global_variable_expression<var = <scope = #di_module, name = "module_global", linkageName = "module_global", file = #di_file, line = 121, type = #di_basic_type, isLocalToUnit = false, isDefined = true>, expr = <>>]} : i64
 
 // -----
 
@@ -362,7 +362,7 @@ llvm.mlir.global external @module_global() {dbg_expr = #llvm.di_global_variable_
 #cu = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #file, producer = "MLIR", isOptimized = true, emissionKind = Full>
 #ty1 = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
 #sp = #llvm.di_subprogram<compileUnit = #cu, scope = #file, name = "fn_with_gl", file = #file, subprogramFlags = "Definition|Optimized">
-llvm.mlir.global @func_global() {dbg_expr = #llvm.di_global_variable_expression<var = <scope = #sp, name = "func_global", linkageName = "func_global", file = #file, line = 121, type = #ty1, isLocalToUnit = true, isDefined = true>, expr = <>>} : i64
+llvm.mlir.global @func_global() {dbg_exprs = [#llvm.di_global_variable_expression<var = <scope = #sp, name = "func_global", linkageName = "func_global", file = #file, line = 121, type = #ty1, isLocalToUnit = true, isDefined = true>, expr = <>>]} : i64
 
 llvm.func @fn_with_gl() {
   llvm.return
@@ -419,7 +419,7 @@ llvm.func @imp_fn() {
 #di_global_variable = #llvm.di_global_variable<file = #di_file, line = 268, type = #di_composite_type, isLocalToUnit = true, isDefined = true>
 #di_global_variable_expression = #llvm.di_global_variable_expression<var = #di_global_variable, expr = <>>
 
-llvm.mlir.global external constant @".str.1"() {addr_space = 0 : i32, dbg_expr = #di_global_variable_expression} : !llvm.array<10 x i8>
+llvm.mlir.global external constant @".str.1"() {addr_space = 0 : i32, dbg_exprs = [#di_global_variable_expression]} : !llvm.array<10 x i8>
 
 // -----
 
@@ -514,7 +514,7 @@ llvm.func @class_method() {
 #di_global_variable = #llvm.di_global_variable<file = #di_file, line = 1, type = #di_composite_type>
 #di_global_variable_expression = #llvm.di_global_variable_expression<var = #di_global_variable>
 
-llvm.mlir.global @global_variable() {dbg_expr = #di_global_variable_expression} : !llvm.struct<()>
+llvm.mlir.global @global_variable() {dbg_exprs = [#di_global_variable_expression]} : !llvm.struct<()>
 
 // CHECK: distinct !DIGlobalVariable({{.*}}type: ![[COMP:[0-9]+]],
 // CHECK: ![[COMP]] = distinct !DICompositeType({{.*}}scope: ![[SCOPE:[0-9]+]]
@@ -544,7 +544,7 @@ llvm.mlir.global @global_variable() {dbg_expr = #di_global_variable_expression}
 #di_global_variable = #llvm.di_global_variable<file = #di_file, line = 1, type = #di_var_type>
 #di_global_variable_expression = #llvm.di_global_variable_expression<var = #di_global_variable>
 
-llvm.mlir.global @global_variable() {dbg_expr = #di_global_variable_expression} : !llvm.struct<()>
+llvm.mlir.global @global_variable() {dbg_exprs = [#di_global_variable_expression]} : !llvm.struct<()>
 
 // CHECK: distinct !DIGlobalVariable({{.*}}type: ![[VAR:[0-9]+]],
 // CHECK: ![[VAR]] = !DISubroutineType(types: ![[COMPS:[0-9]+]])
@@ -616,7 +616,7 @@ llvm.func @fn_with_composite() {
 #loc1 = loc("test.f90": 1:1)
 #loc2 = loc(fused<#sp>[#loc1])
 
-llvm.mlir.global external @gv() {dbg_expr = #gve} : i64
+llvm.mlir.global external @gv() {dbg_exprs = [#gve]} : i64
 
 llvm.func @subranges(%arg: !llvm.ptr) {
   llvm.intr.dbg.declare #lvar2 = %arg : !llvm.ptr
@@ -678,7 +678,7 @@ llvm.func @string_ty(%arg0: !llvm.ptr) {
  expr = <>>
 
 llvm.mlir.global common @block_(dense<0> : tensor<8xi8>)
-  {dbg_expr = #var_expression} : !llvm.array<8 x i8>
+  {dbg_exprs = [#var_expression]} : !llvm.array<8 x i8>
 
 llvm.func @test() {
   llvm.return
@@ -690,3 +690,27 @@ llvm.func @test() {
 // CHECK: !DICommonBlock(scope: ![[SCOPE:[0-9]+]], declaration: null, name: "block", file: ![[FILE:[0-9]+]], line: 3)
 // CHECK: ![[SCOPE]] = {{.*}}!DISubprogram(name: "test"{{.*}})
 // CHECK: ![[FILE]] = !DIFile(filename: "test.f90"{{.*}})
+
+// -----
+
+// Test multiple DIGlobalVariableExpression on a global.
+#bt = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "int", sizeInBits = 32>
+#file = #llvm.di_file<"test.f90" in "">
+#cu = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_C,
+ file = #file, isOptimized = false, emissionKind = Full>
+#global_var = #llvm.di_global_variable<scope = #cu, name = "a",
+ file = #file, line = 2, type = #bt>
+#var_expression = #llvm.di_global_variable_expression<var = #global_var,
+ expr = <>>
+#global_var1 = #llvm.di_global_variable<scope = #cu, name = "b",
+ file = #file, line = 3, type = #bt>
+#var_expression1 = #llvm.di_global_variable_expression<var = #global_var1,
+ expr = <>>
+
+llvm.mlir.global @data() {dbg_exprs = [#var_expression, #var_expression1]} : i64
+
+// CHECK: @data = external global i64, !dbg ![[EXP1:[0-9]+]], !dbg ![[EXP2:[0-9]+]]
+// CHECK: ![[EXP1]] = !DIGlobalVariableExpression(var: ![[VAR1:[0-9]+]], expr: !DIExpression())
+// CHECK: ![[VAR1]] = {{.*}}!DIGlobalVariable(name: "a"{{.*}})
+// CHECK: ![[EXP2]] = !DIGlobalVariableExpression(var: ![[VAR2:[0-9]+]], expr: !DIExpression())
+// CHECK: ![[VAR2]] = {{.*}}!DIGlobalVariable(name: "b"{{.*}})
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index de0dc8d21584fefc36d1af5cb171d86389c0df71..0634a7ba907f1e00ab92dad838334aa77b723839 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -112,6 +112,25 @@ llvm.func @cos_test(%arg0: f32, %arg1: vector<8xf32>) {
   llvm.return
 }
 
+// CHECK-LABEL: @hyperbolic_trig_test
+llvm.func @hyperbolic_trig_test(%arg0: f32, %arg1: vector<8xf32>) {
+  // CHECK: call float @llvm.sinh.f32
+  llvm.intr.sinh(%arg0) : (f32) -> f32
+  // CHECK: call <8 x float> @llvm.sinh.v8f32
+  llvm.intr.sinh(%arg1) : (vector<8xf32>) -> vector<8xf32>
+
+  // CHECK: call float @llvm.cosh.f32
+  llvm.intr.cosh(%arg0) : (f32) -> f32
+  // CHECK: call <8 x float> @llvm.cosh.v8f32
+  llvm.intr.cosh(%arg1) : (vector<8xf32>) -> vector<8xf32>
+
+  // CHECK: call float @llvm.tanh.f32
+  llvm.intr.tanh(%arg0) : (f32) -> f32
+  // CHECK: call <8 x float> @llvm.tanh.v8f32
+  llvm.intr.tanh(%arg1) : (vector<8xf32>) -> vector<8xf32>
+  llvm.return
+}
+
 // CHECK-LABEL: @copysign_test
 llvm.func @copysign_test(%arg0: f32, %arg1: f32, %arg2: vector<8xf32>, %arg3: vector<8xf32>) {
   // CHECK: call float @llvm.copysign.f32
diff --git a/mlir/test/Target/LLVMIR/omptarget-nowait-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-nowait-llvm.mlir
index 1e2fbe86d13c4728bb889242ed1e39c044d29bb0..b487b31d5447771e6e11c65f9f965b1b5e007281 100644
--- a/mlir/test/Target/LLVMIR/omptarget-nowait-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-nowait-llvm.mlir
@@ -1,39 +1,31 @@
-// RUN: not mlir-translate -mlir-to-llvmir -split-input-file %s 2>&1 | FileCheck %s
+// RUN: mlir-translate -mlir-to-llvmir %s 2>&1 | FileCheck %s
 
-llvm.func @_QPopenmp_target_data_update() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+// Set a dummy target triple to enable target region outlining.
+module attributes {omp.target_triples = ["dummy-target-triple"]} {
+  llvm.func @_QPfoo() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(implicit) capture(ByCopy) -> !llvm.ptr
+    omp.target nowait map_entries(%2 -> %arg0 : !llvm.ptr) {
+      %3 = llvm.mlir.constant(2 : i32) : i32
+      llvm.store %3, %arg0 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
+  }
 
-  // CHECK: error: `nowait` is not supported yet
-  omp.target_update map_entries(%2 : !llvm.ptr) nowait
-
-  llvm.return
-}
-
-// -----
-
-llvm.func @_QPopenmp_target_data_enter() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
-
-  // CHECK: error: `nowait` is not supported yet
-  omp.target_enter_data map_entries(%2 : !llvm.ptr) nowait
-
-  llvm.return
-}
 
+// CHECK: define void @_QPfoo() {
 
-// -----
+// CHECK:   %[[TASK:.*]] = call ptr @__kmpc_omp_target_task_alloc
+// CHECK-SAME:     (ptr @{{.*}}, i32 %{{.*}}, i32 {{.*}}, i64 {{.*}}, i64 {{.*}}, ptr
+// CHECK-SAME:     @[[TASK_PROXY_FUNC:.*]], i64 {{.*}})
 
-llvm.func @_QPopenmp_target_data_exit() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+// CHECK:   call i32 @__kmpc_omp_task(ptr {{.*}}, i32 %{{.*}}, ptr %[[TASK]])
+// CHECK: }
 
-  // CHECK: error: `nowait` is not supported yet
-  omp.target_exit_data map_entries(%2 : !llvm.ptr) nowait
 
-  llvm.return
+// CHECK: define internal void @[[TASK_PROXY_FUNC]](i32 %{{.*}}, ptr %{{.*}}) {
+// CHECK:   call void @_QPfoo..omp_par(i32 %{{.*}}, ptr %{{.*}})
+// CHECK: }
 }
diff --git a/mlir/test/Target/LLVMIR/omptarget-nowait-unsupported-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-nowait-unsupported-llvm.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..1e2fbe86d13c4728bb889242ed1e39c044d29bb0
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-nowait-unsupported-llvm.mlir
@@ -0,0 +1,39 @@
+// RUN: not mlir-translate -mlir-to-llvmir -split-input-file %s 2>&1 | FileCheck %s
+
+llvm.func @_QPopenmp_target_data_update() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+
+  // CHECK: error: `nowait` is not supported yet
+  omp.target_update map_entries(%2 : !llvm.ptr) nowait
+
+  llvm.return
+}
+
+// -----
+
+llvm.func @_QPopenmp_target_data_enter() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+
+  // CHECK: error: `nowait` is not supported yet
+  omp.target_enter_data map_entries(%2 : !llvm.ptr) nowait
+
+  llvm.return
+}
+
+
+// -----
+
+llvm.func @_QPopenmp_target_data_exit() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+
+  // CHECK: error: `nowait` is not supported yet
+  omp.target_exit_data map_entries(%2 : !llvm.ptr) nowait
+
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
index 360b3b0c0e60c1791e7fed19c8cf1761f4bce881..649210795ff5c72764a653be6a565676077dac37 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
@@ -18,7 +18,6 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
           llvm.store %loop_cnt, %gep : i32, !llvm.ptr
           omp.yield
         }
-        omp.terminator
       }
      omp.terminator
     }
diff --git a/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir b/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir
index 13d34b7e58f77e19b1dd2ab25645f78e0f1f73c2..b7aecec308ef35519a12c55f8bf074be79760b29 100644
--- a/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir
@@ -17,7 +17,6 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
         llvm.store %1, %4 : i32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
index ee851eaf71ac0b726bb52a74c36a5f90a6f94975..e2a8d88bd181afe8acdfec4f95730125e38cc5c0 100644
--- a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
@@ -14,7 +14,6 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
           llvm.store %loop_cnt, %gep : i32, !llvm.ptr
           omp.yield
         }
-        omp.terminator
       }
     llvm.return
   }
@@ -27,7 +26,6 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
         omp.loop_nest (%loop_cnt) : i32 = (%loop_lb) to (%loop_ub) inclusive step (%loop_step) {
           omp.yield
         }
-        omp.terminator
       }
     llvm.return
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir b/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
index 3dad3e9eee60d95ead7c5d2e15139240db5a90d2..717a77e61b9a11507f7d5552e27f5ffebd2dfe17 100644
--- a/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
@@ -46,7 +46,6 @@ module attributes { } {
               llvm.store %21, %30 : i32, !llvm.ptr
               omp.yield
             }
-            omp.terminator
           }
           omp.terminator
         }
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 5d76e87472dfe4ab36b4af324f427e87b6781ddd..c7ddbda8e8a9212428089e06964cc4da143d4df6 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -351,7 +351,6 @@ llvm.func @wsloop_simple(%arg0: !llvm.ptr) {
         llvm.store %3, %4 : f32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
       // CHECK: call void @__kmpc_for_static_fini(ptr @[[$loc_struct]],
     }) : () -> ()
     omp.terminator
@@ -374,7 +373,6 @@ llvm.func @wsloop_inclusive_1(%arg0: !llvm.ptr) {
       llvm.store %3, %4 : f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }) : () -> ()
   llvm.return
 }
@@ -394,7 +392,6 @@ llvm.func @wsloop_inclusive_2(%arg0: !llvm.ptr) {
       llvm.store %3, %4 : f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }) : () -> ()
   llvm.return
 }
@@ -412,7 +409,6 @@ llvm.func @test_omp_wsloop_static_defchunk(%lb : i32, %ub : i32, %step : i32) ->
       llvm.call @body(%iv) : (i32) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -431,7 +427,6 @@ llvm.func @test_omp_wsloop_static_1(%lb : i32, %ub : i32, %step : i32) -> () {
       llvm.call @body(%iv) : (i32) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -450,7 +445,6 @@ llvm.func @test_omp_wsloop_static_2(%lb : i32, %ub : i32, %step : i32) -> () {
       llvm.call @body(%iv) : (i32) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -469,7 +463,6 @@ llvm.func @test_omp_wsloop_dynamic(%lb : i64, %ub : i64, %step : i64) -> () {
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -489,7 +482,6 @@ llvm.func @test_omp_wsloop_dynamic_chunk_const(%lb : i64, %ub : i64, %step : i64
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -512,7 +504,6 @@ llvm.func @test_omp_wsloop_dynamic_chunk_var(%lb : i32, %ub : i32, %step : i32)
       llvm.call @body(%iv) : (i32) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -535,7 +526,6 @@ llvm.func @test_omp_wsloop_dynamic_chunk_var2(%lb : i32, %ub : i32, %step : i32)
       llvm.call @body(%iv) : (i32) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -554,7 +544,6 @@ llvm.func @test_omp_wsloop_dynamic_chunk_var3(%lb : i32, %ub : i32, %step : i32,
       llvm.call @body(%iv) : (i32) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -573,7 +562,6 @@ llvm.func @test_omp_wsloop_auto(%lb : i64, %ub : i64, %step : i64) -> () {
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -592,7 +580,6 @@ llvm.func @test_omp_wsloop_runtime(%lb : i64, %ub : i64, %step : i64) -> () {
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -611,7 +598,6 @@ llvm.func @test_omp_wsloop_guided(%lb : i64, %ub : i64, %step : i64) -> () {
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -630,7 +616,6 @@ llvm.func @test_omp_wsloop_dynamic_nonmonotonic(%lb : i64, %ub : i64, %step : i6
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -649,7 +634,6 @@ llvm.func @test_omp_wsloop_dynamic_monotonic(%lb : i64, %ub : i64, %step : i64)
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -668,7 +652,6 @@ llvm.func @test_omp_wsloop_runtime_simd(%lb : i64, %ub : i64, %step : i64) -> ()
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -687,7 +670,6 @@ llvm.func @test_omp_wsloop_guided_simd(%lb : i64, %ub : i64, %step : i64) -> ()
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -706,7 +688,6 @@ llvm.func @simd_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) {
       llvm.store %3, %4 : f32, !llvm.ptr
       omp.yield
     }
-    "omp.terminator"() : () -> ()
   }) : () -> ()
 
   llvm.return
@@ -746,7 +727,6 @@ llvm.func @simd_simple_multiple(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64
       llvm.store %3, %5 : f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -770,7 +750,6 @@ llvm.func @simd_simple_multiple_simdlen(%lb1 : i64, %ub1 : i64, %step1 : i64, %l
       llvm.store %3, %5 : f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -791,7 +770,6 @@ llvm.func @simd_simple_multiple_safelen(%lb1 : i64, %ub1 : i64, %step1 : i64, %l
       llvm.store %3, %5 : f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -811,7 +789,6 @@ llvm.func @simd_simple_multiple_simdlen_safelen(%lb1 : i64, %ub1 : i64, %step1 :
       llvm.store %3, %5 : f32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -840,7 +817,6 @@ llvm.func @simd_if(%arg0: !llvm.ptr {fir.bindc_name = "n"}, %arg1: !llvm.ptr {fi
       llvm.store %arg2, %1 : i32, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -861,7 +837,6 @@ llvm.func @simd_order() {
       llvm.store %arg0, %2 : i64, !llvm.ptr
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -885,7 +860,6 @@ llvm.func @test_omp_wsloop_ordered(%lb : i64, %ub : i64, %step : i64) -> () {
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -905,7 +879,6 @@ llvm.func @test_omp_wsloop_static_ordered(%lb : i64, %ub : i64, %step : i64) ->
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -926,7 +899,6 @@ llvm.func @test_omp_wsloop_static_chunk_ordered(%lb : i32, %ub : i32, %step : i3
       llvm.call @body(%iv) : (i32) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -946,7 +918,6 @@ llvm.func @test_omp_wsloop_dynamic_ordered(%lb : i64, %ub : i64, %step : i64) ->
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -966,7 +937,6 @@ llvm.func @test_omp_wsloop_auto_ordered(%lb : i64, %ub : i64, %step : i64) -> ()
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -986,7 +956,6 @@ llvm.func @test_omp_wsloop_runtime_ordered(%lb : i64, %ub : i64, %step : i64) ->
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -1006,7 +975,6 @@ llvm.func @test_omp_wsloop_guided_ordered(%lb : i64, %ub : i64, %step : i64) ->
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -1026,7 +994,6 @@ llvm.func @test_omp_wsloop_dynamic_nonmonotonic_ordered(%lb : i64, %ub : i64, %s
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -1046,7 +1013,6 @@ llvm.func @test_omp_wsloop_dynamic_monotonic_ordered(%lb : i64, %ub : i64, %step
       llvm.call @body(%iv) : (i64) -> ()
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
@@ -1223,7 +1189,6 @@ llvm.func @collapse_wsloop(
         llvm.store %34, %20 : i32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -1286,7 +1251,6 @@ llvm.func @collapse_wsloop_dynamic(
         llvm.store %34, %20 : i32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -1320,7 +1284,6 @@ llvm.func @omp_ordered(%arg0 : i32, %arg1 : i32, %arg2 : i32, %arg3 : i64,
       }
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop ordered(1) {
@@ -1341,7 +1304,6 @@ llvm.func @omp_ordered(%arg0 : i32, %arg1 : i32, %arg2 : i32, %arg3 : i64,
 
       omp.yield
     }
-    omp.terminator
   }
 
   omp.wsloop ordered(2) {
@@ -1373,7 +1335,6 @@ llvm.func @omp_ordered(%arg0 : i32, %arg1 : i32, %arg2 : i32, %arg3 : i64,
 
       omp.yield
     }
-    omp.terminator
   }
 
   llvm.return
@@ -2387,7 +2348,6 @@ llvm.func @repeated_successor(%arg0: i64, %arg1: i64, %arg2: i64, %arg3: i1) {
     ^bb1(%0: i64):  // 2 preds: ^bb0, ^bb0
       omp.yield
     }
-    omp.terminator
   }
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-nested.mlir b/mlir/test/Target/LLVMIR/openmp-nested.mlir
index ce5f22f10d7dce1534652bd74cb8e0f83802b30d..a5224540691f32bc5d6eae08d52bee25361b0e80 100644
--- a/mlir/test/Target/LLVMIR/openmp-nested.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-nested.mlir
@@ -19,7 +19,6 @@ module {
                 llvm.store %2, %12 : i64, !llvm.ptr
                 omp.yield
               }
-              omp.terminator
             }
             omp.terminator
           }
@@ -30,7 +29,6 @@ module {
           %21 = llvm.call @printf(%6, %20, %20) vararg(!llvm.func<i32 (ptr, ...)>): (!llvm.ptr, i32, i32) -> i32
           omp.yield
         }
-        omp.terminator
       }
       omp.terminator
     }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction.mlir b/mlir/test/Target/LLVMIR/openmp-reduction.mlir
index dcac402d5fa28567dd57e7649e0c3c6711774e31..6d74a925b87b5c9289080198f6d906b9b616ba90 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction.mlir
@@ -34,7 +34,6 @@ llvm.func @simple_reduction(%lb : i64, %ub : i64, %step : i64) {
         llvm.store %3, %prv : f32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -118,7 +117,6 @@ llvm.func @reuse_declaration(%lb : i64, %ub : i64, %step : i64) {
         llvm.store %6, %prv1 : f32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -207,7 +205,6 @@ llvm.func @missing_omp_reduction(%lb : i64, %ub : i64, %step : i64) {
         llvm.store %4, %prv0 : f32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -297,7 +294,6 @@ llvm.func @double_reference(%lb : i64, %ub : i64, %step : i64) {
         llvm.store %5, %prv : f32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -393,7 +389,6 @@ llvm.func @no_atomic(%lb : i64, %ub : i64, %step : i64) {
         llvm.store %6, %prv1 : f32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
@@ -549,7 +544,6 @@ llvm.func @parallel_nested_workshare_reduction(%ub : i64) {
         llvm.store %add, %prv : i32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     omp.terminator
   }
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
index 7726b980581fa6a43a05bc0bf28ae04284ef1a47..1a5065fec026ea6dab7667cd752ebddf681c72ef 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
@@ -36,7 +36,6 @@
         llvm.store %0, %arg1 : i32, !llvm.ptr
         omp.yield
       }
-      omp.terminator
     }
     llvm.return
   }
diff --git a/mlir/test/lib/Analysis/DataFlow/TestDeadCodeAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestDeadCodeAnalysis.cpp
index d02efaaa3fe320bc211b95e9350a218a3a79426d..2dc77c9705d3509513aa1b711e1b611be7aa0462 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestDeadCodeAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestDeadCodeAnalysis.cpp
@@ -29,7 +29,8 @@ static void printAnalysisResults(DataFlowSolver &solver, Operation *op,
         os << "  ";
         block.printAsOperand(os);
         os << " = ";
-        auto *live = solver.lookupState<Executable>(&block);
+        auto *live = solver.lookupState<Executable>(
+            solver.getProgramPointBefore(&block));
         if (live)
           os << *live;
         else
@@ -49,12 +50,14 @@ static void printAnalysisResults(DataFlowSolver &solver, Operation *op,
         }
       }
       if (!region.empty()) {
-        auto *preds = solver.lookupState<PredecessorState>(&region.front());
+        auto *preds = solver.lookupState<PredecessorState>(
+            solver.getProgramPointBefore(&region.front()));
         if (preds)
           os << "region_preds: " << *preds << "\n";
       }
     }
-    auto *preds = solver.lookupState<PredecessorState>(op);
+    auto *preds =
+        solver.lookupState<PredecessorState>(solver.getProgramPointAfter(op));
     if (preds)
       os << "op_preds: " << *preds << "\n";
   });
@@ -68,15 +71,15 @@ struct ConstantAnalysis : public DataFlowAnalysis {
 
   LogicalResult initialize(Operation *top) override {
     WalkResult result = top->walk([&](Operation *op) {
-      if (failed(visit(op)))
+      if (failed(visit(getProgramPointAfter(op))))
         return WalkResult::interrupt();
       return WalkResult::advance();
     });
     return success(!result.wasInterrupted());
   }
 
-  LogicalResult visit(ProgramPoint point) override {
-    Operation *op = point.get<Operation *>();
+  LogicalResult visit(ProgramPoint *point) override {
+    Operation *op = point->getPrevOp();
     Attribute value;
     if (matchPattern(op, m_Constant(&value))) {
       auto *constant = getOrCreate<Lattice<ConstantValue>>(op->getResult(0));
diff --git a/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp
index 6794cbbbd899419571c72f8faea1f519d7e1c0f3..fa6223aa9168be05ff946782c4698a4551dbde04 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp
@@ -115,7 +115,8 @@ LogicalResult NextAccessAnalysis::visitOperation(Operation *op,
     std::optional<Value> underlyingValue =
         UnderlyingValueAnalysis::getMostUnderlyingValue(
             value, [&](Value value) {
-              return getOrCreateFor<UnderlyingValueLattice>(op, value);
+              return getOrCreateFor<UnderlyingValueLattice>(
+                  getProgramPointBefore(op), value);
             });
 
     // If the underlying value is not known yet, don't propagate.
@@ -151,7 +152,7 @@ void NextAccessAnalysis::visitCallControlFlowTransfer(
           UnderlyingValueAnalysis::getMostUnderlyingValue(
               operand, [&](Value value) {
                 return getOrCreateFor<UnderlyingValueLattice>(
-                    call.getOperation(), value);
+                    getProgramPointBefore(call.getOperation()), value);
               });
       if (!underlyingValue)
         return;
@@ -283,9 +284,8 @@ struct TestNextAccessPass
       if (!tag)
         return;
 
-      const NextAccess *nextAccess = solver.lookupState<NextAccess>(
-          op->getNextNode() == nullptr ? ProgramPoint(op->getBlock())
-                                       : op->getNextNode());
+      const NextAccess *nextAccess =
+          solver.lookupState<NextAccess>(solver.getProgramPointAfter(op));
       op->setAttr(kNextAccessAttrName,
                   makeNextAccessAttribute(op, solver, nextAccess));
 
@@ -300,9 +300,8 @@ struct TestNextAccessPass
         if (!successor.getSuccessor() || successor.getSuccessor()->empty())
           continue;
         Block &successorBlock = successor.getSuccessor()->front();
-        ProgramPoint successorPoint = successorBlock.empty()
-                                          ? ProgramPoint(&successorBlock)
-                                          : &successorBlock.front();
+        ProgramPoint *successorPoint =
+            solver.getProgramPointBefore(&successorBlock);
         entryPointNextAccess.push_back(makeNextAccessAttribute(
             op, solver, solver.lookupState<NextAccess>(successorPoint)));
       }
diff --git a/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp
index 301d2a20978c845c34c886fc4619db2f1cc3ebff..89b5c835744fdb4d90de41410da25d1d04e19534 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp
@@ -117,7 +117,8 @@ LogicalResult LastModifiedAnalysis::visitOperation(
     std::optional<Value> underlyingValue =
         UnderlyingValueAnalysis::getMostUnderlyingValue(
             value, [&](Value value) {
-              return getOrCreateFor<UnderlyingValueLattice>(op, value);
+              return getOrCreateFor<UnderlyingValueLattice>(
+                  getProgramPointAfter(op), value);
             });
 
     // If the underlying value is not yet known, don't propagate yet.
@@ -157,7 +158,7 @@ void LastModifiedAnalysis::visitCallControlFlowTransfer(
           UnderlyingValueAnalysis::getMostUnderlyingValue(
               operand, [&](Value value) {
                 return getOrCreateFor<UnderlyingValueLattice>(
-                    call.getOperation(), value);
+                    getProgramPointAfter(call.getOperation()), value);
               });
       if (!underlyingValue)
         return;
@@ -243,7 +244,7 @@ struct TestLastModifiedPass
         return;
       os << "test_tag: " << tag.getValue() << ":\n";
       const LastModification *lastMods =
-          solver.lookupState<LastModification>(op);
+          solver.lookupState<LastModification>(solver.getProgramPointAfter(op));
       assert(lastMods && "expected a dense lattice");
       for (auto [index, operand] : llvm::enumerate(op->getOperands())) {
         os << " operand #" << index << "\n";
diff --git a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
index 2445b58452bd60a751a49b3ca781c7169730d3f0..1f3cab1e02adbf7861b3c80b241c99946f068fa8 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
@@ -108,7 +108,7 @@ WrittenToAnalysis::visitOperation(Operation *op, ArrayRef<WrittenTo *> operands,
     for (WrittenTo *operand : operands) {
       meet(operand, *r);
     }
-    addDependency(const_cast<WrittenTo *>(r), op);
+    addDependency(const_cast<WrittenTo *>(r), getProgramPointAfter(op));
   }
   return success();
 }
diff --git a/mlir/test/lib/Analysis/TestDataFlowFramework.cpp b/mlir/test/lib/Analysis/TestDataFlowFramework.cpp
index 9573ec1d1432574545dcd4285b3b79758ab595ca..3eb39fc950dadba441e5aa9337121472d62a1292 100644
--- a/mlir/test/lib/Analysis/TestDataFlowFramework.cpp
+++ b/mlir/test/lib/Analysis/TestDataFlowFramework.cpp
@@ -75,7 +75,7 @@ public:
   using DataFlowAnalysis::DataFlowAnalysis;
 
   LogicalResult initialize(Operation *top) override;
-  LogicalResult visit(ProgramPoint point) override;
+  LogicalResult visit(ProgramPoint *point) override;
 
 private:
   void visitBlock(Block *block);
@@ -100,7 +100,8 @@ LogicalResult FooAnalysis::initialize(Operation *top) {
     return top->emitError("expected at least one block in the region");
 
   // Initialize the top-level state.
-  (void)getOrCreate<FooState>(&top->getRegion(0).front())->join(0);
+  (void)getOrCreate<FooState>(getProgramPointBefore(&top->getRegion(0).front()))
+      ->join(0);
 
   // Visit all nested blocks and operations.
   for (Block &block : top->getRegion(0)) {
@@ -114,11 +115,11 @@ LogicalResult FooAnalysis::initialize(Operation *top) {
   return success();
 }
 
-LogicalResult FooAnalysis::visit(ProgramPoint point) {
-  if (auto *op = llvm::dyn_cast_if_present<Operation *>(point))
-    visitOperation(op);
+LogicalResult FooAnalysis::visit(ProgramPoint *point) {
+  if (!point->isBlockStart())
+    visitOperation(point->getPrevOp());
   else
-    visitBlock(point.get<Block *>());
+    visitBlock(point->getBlock());
   return success();
 }
 
@@ -127,27 +128,26 @@ void FooAnalysis::visitBlock(Block *block) {
     // This is the initial state. Let the framework default-initialize it.
     return;
   }
-  FooState *state = getOrCreate<FooState>(block);
+  ProgramPoint *point = getProgramPointBefore(block);
+  FooState *state = getOrCreate<FooState>(point);
   ChangeResult result = ChangeResult::NoChange;
   for (Block *pred : block->getPredecessors()) {
     // Join the state at the terminators of all predecessors.
-    const FooState *predState =
-        getOrCreateFor<FooState>(block, pred->getTerminator());
+    const FooState *predState = getOrCreateFor<FooState>(
+        point, getProgramPointAfter(pred->getTerminator()));
     result |= state->join(*predState);
   }
   propagateIfChanged(state, result);
 }
 
 void FooAnalysis::visitOperation(Operation *op) {
-  FooState *state = getOrCreate<FooState>(op);
+  ProgramPoint *point = getProgramPointAfter(op);
+  FooState *state = getOrCreate<FooState>(point);
   ChangeResult result = ChangeResult::NoChange;
 
   // Copy the state across the operation.
   const FooState *prevState;
-  if (Operation *prev = op->getPrevNode())
-    prevState = getOrCreateFor<FooState>(op, prev);
-  else
-    prevState = getOrCreateFor<FooState>(op, op->getBlock());
+  prevState = getOrCreateFor<FooState>(point, getProgramPointBefore(op));
   result |= state->set(*prevState);
 
   // Modify the state with the attribute, if specified.
@@ -172,7 +172,8 @@ void TestFooAnalysisPass::runOnOperation() {
     auto tag = op->getAttrOfType<StringAttr>("tag");
     if (!tag)
       return;
-    const FooState *state = solver.lookupState<FooState>(op);
+    const FooState *state =
+        solver.lookupState<FooState>(solver.getProgramPointAfter(op));
     assert(state && !state->isUninitialized());
     os << tag.getValue() << " -> " << state->getValue() << "\n";
   });
diff --git a/mlir/test/mlir-rewrite/simple.mlir b/mlir/test/mlir-rewrite/simple.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..ab6bfe24fccf034f7883091682eda57414005cdf
--- /dev/null
+++ b/mlir/test/mlir-rewrite/simple.mlir
@@ -0,0 +1,11 @@
+// RUN: mlir-opt %s | mlir-rewrite --simple-rename --simple-rename-op-name="test.concat" --simple-rename-match="axis" --simple-rename-replace="bxis" | FileCheck %s -check-prefix=RENAME
+// RUN: mlir-opt %s | mlir-rewrite --mark-ranges | FileCheck %s -check-prefix=RANGE
+// Note: running through mlir-opt to just strip out comments & avoid self matches.
+
+func.func @two_dynamic_one_direct_shape(%arg0: tensor<?x4x?xf32>, %arg1: tensor<2x4x?xf32>) -> tensor<?x4x?xf32> {
+  // RENAME: "test.concat"({{.*}}) {bxis = 0 : i64}
+  // RANGE: 《%{{.*}} = 〖"test.concat"〗({{.*}}) {axis = 0 : i64} : (tensor<?x4x?xf32>, tensor<2x4x?xf32>) -> tensor<?x4x?xf32>》
+  %5 = "test.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<?x4x?xf32>, tensor<2x4x?xf32>) -> tensor<?x4x?xf32>
+  return %5 : tensor<?x4x?xf32>
+}
+
diff --git a/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir b/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir
index 160c388cedf75654df158285efb5c6d78fda1aec..f0d1571d5838776dca40bee2f7b99ad536e6768a 100644
--- a/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir
+++ b/mlir/test/mlir-tblgen/attr-or-type-format-roundtrip.mlir
@@ -23,10 +23,10 @@ attributes {
   attr7 = #test.custom_anchor<5>,
   // CHECK: #test.custom_anchor<5, true>
   attr8 = #test.custom_anchor<5, true>,
-  // CHECK: #test.attr_with_optional_signed<-12>
-  attr9 = #test.attr_with_optional_signed<-12>,
-  // CHECK: #test.attr_with_optional_unsigned<22>
-  attr_10 = #test.attr_with_optional_unsigned<22>
+  // CHECK: #test.attr_with_optional_signed<-9223372036854775808>
+  attr9 = #test.attr_with_optional_signed<-9223372036854775808>,
+  // CHECK: #test.attr_with_optional_unsigned<18446744073709551615>
+  attr_10 = #test.attr_with_optional_unsigned<18446744073709551615>
 }
 
 // CHECK-LABEL: @test_roundtrip_default_parsers_struct
diff --git a/mlir/test/mlir-tblgen/op-decl-and-defs.td b/mlir/test/mlir-tblgen/op-decl-and-defs.td
index f6fe45007d9e05a70817dceb2b143a75f145af21..31dd53725c59ac54263013a4cc534b830710d13f 100644
--- a/mlir/test/mlir-tblgen/op-decl-and-defs.td
+++ b/mlir/test/mlir-tblgen/op-decl-and-defs.td
@@ -72,8 +72,9 @@ def NS_AOp : NS_Op<"a_op", [IsolatedFromAbove, IsolatedFromAbove]> {
 // CHECK: template <typename RangeT>
 // CHECK: class AOpGenericAdaptor : public detail::AOpGenericAdaptorBase {
 // CHECK: public:
-// CHECK:   AOpGenericAdaptor(RangeT values,
-// CHECK-SAME: odsOperands(values)
+// CHECK:   AOpGenericAdaptor(RangeT values, ::mlir::DictionaryAttr attrs = {}, const ::mlir::EmptyProperties &properties = {}, ::mlir::RegionRange regions = {}) : Base(attrs, properties, regions), odsOperands(values) {}
+// CHECK:   AOpGenericAdaptor(RangeT values, ::mlir::DictionaryAttr attrs, ::mlir::OpaqueProperties properties, ::mlir::RegionRange regions = {}) : AOpGenericAdaptor(values, attrs, (properties ? *properties.as<::mlir::EmptyProperties *>() : ::mlir::EmptyProperties{}), regions) {}
+// CHECK:   AOpGenericAdaptor(RangeT values, const AOpGenericAdaptorBase &base) : Base(base), odsOperands(values) {}
 // CHECK:   RangeT getODSOperands(unsigned index) {
 // CHECK:   ValueT getA() {
 // CHECK:   RangeT getB() {
diff --git a/mlir/test/python/dialects/linalg/ops.py b/mlir/test/python/dialects/linalg/ops.py
index 72045a07b2da8008c1e5a520bfd1c2a74006cce0..3bfbcf7d7f7c81d4f0a473e9e7c0ae4c3e74a731 100644
--- a/mlir/test/python/dialects/linalg/ops.py
+++ b/mlir/test/python/dialects/linalg/ops.py
@@ -84,6 +84,81 @@ def testNamedStructuredOpCustomForm():
 
     print(module)
 
+
+# CHECK-LABEL: TEST: testNamedStructuredOpGenericForm
+@run
+def testNamedStructuredOpGenericForm():
+    with Context() as ctx, Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        with InsertionPoint(module.body):
+
+            @func.FuncOp.from_py_func(
+                RankedTensorType.get((4, 16), f32), RankedTensorType.get((16, 8), f32)
+            )
+            def named_form(lhs, rhs):
+                init_result = tensor.empty([4, 8], f32)
+                #      CHECK: "linalg.matmul"(%{{.*}})
+                # CHECK-SAME:    cast = #linalg.type_fn<cast_signed>
+                # CHECK-SAME:    operandSegmentSizes = array<i32: 2, 1>
+                # CHECK-NEXT:  ^bb0(%{{.*}}: f32, %{{.*}}: f32, %{{.*}}: f32):
+                # CHECK-NEXT:    arith.mulf{{.*}} (f32, f32) -> f32
+                # CHECK-NEXT:    arith.addf{{.*}} (f32, f32) -> f32
+                # CHECK-NEXT:    linalg.yield{{.*}} (f32) -> ()
+                # CHECK-NEXT: (tensor<4x16xf32>, tensor<16x8xf32>, tensor<4x8xf32>) -> tensor<4x8xf32>
+                return linalg.matmul(lhs, rhs, outs=[init_result])
+
+    module.operation.print(print_generic_op_form=True)
+
+
+# CHECK-LABEL: TEST: testNamedStructuredAsGenericOp
+@run
+def testNamedStructuredAsGenericOp():
+    with Context() as ctx, Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        with InsertionPoint(module.body):
+
+            @func.FuncOp.from_py_func(
+                RankedTensorType.get((4, 16), f32), RankedTensorType.get((16, 8), f32)
+            )
+            def generic_form(lhs, rhs):
+                init_result = tensor.EmptyOp([4, 8], f32)
+                # CHECK: linalg.generic
+                return linalg.matmul(
+                    lhs, rhs, outs=[init_result.result], emit_generic=True
+                )
+
+    print(module)
+
+
+# CHECK-LABEL: TEST: testOpResultFromOtherOp
+@run
+def testOpResultFromOtherOp():
+    with Context(), Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        with InsertionPoint(module.body):
+
+            @func.FuncOp.from_py_func(
+                RankedTensorType.get((4, 16), f32), RankedTensorType.get((16, 8), f32)
+            )
+            def pass_an_op_directly(arg0, arg1):
+                one = arith.ConstantOp(F32Type.get(), 1.0)
+                # CHECK: %[[LHS:.*]] = linalg.fill
+                lhs = linalg.fill(one, outs=[arg0])
+                # CHECK: %[[RHS:.*]] = linalg.fill
+                rhs = linalg.fill(one, outs=[arg1])
+                # CHECK: %[[INIT:.*]] = tensor.empty
+                init = tensor.EmptyOp([4, 8], f32)
+                # CHECK: linalg.matmul
+                # CHECK: ins(%[[LHS]], %[[RHS]]
+                # CHECK: outs(%[[INIT]]
+                return linalg.matmul(lhs, rhs, outs=init)
+
+    print(module)
+
+
 # CHECK-LABEL: TEST: testIdentityRegionOps
 @run
 def testIdentityRegionOps():
diff --git a/mlir/tools/CMakeLists.txt b/mlir/tools/CMakeLists.txt
index 9b474385fdae187937c7fce2d3be73d6ce851f7e..0a2d0ff291509957e1196ac502f96226f6d0eb02 100644
--- a/mlir/tools/CMakeLists.txt
+++ b/mlir/tools/CMakeLists.txt
@@ -4,6 +4,7 @@ add_subdirectory(mlir-parser-fuzzer)
 add_subdirectory(mlir-pdll-lsp-server)
 add_subdirectory(mlir-query)
 add_subdirectory(mlir-reduce)
+add_subdirectory(mlir-rewrite)
 add_subdirectory(mlir-shlib)
 add_subdirectory(mlir-spirv-cpu-runner)
 add_subdirectory(mlir-translate)
diff --git a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
index f820cb7ee8c3c4978658307f466b22b73c868954..aa5a52a21f1251c1bf1c75954d5230747225e84d 100644
--- a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
+++ b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
@@ -681,11 +681,7 @@ ParseResult {0}::parse(OpAsmParser &parser, OperationState &result) {{
     {0}::getNumRegionArgs(), {0}::getRegionBuilder());
 }
 void {0}::print(OpAsmPrinter &p) {{
-  SmallVector<StringRef, 3> elidedAttrs = {{"operandSegmentSizes",
-                                           "linalg.memoized_indexing_maps",
-                                           "indexing_maps"};
-  ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs(),
-                           elidedAttrs);
+  ::printNamedStructuredOp(p, getOperation(), getInputs(), getOutputs());
 }
 )FMT";
 
diff --git a/mlir/tools/mlir-rewrite/CMakeLists.txt b/mlir/tools/mlir-rewrite/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5b8c1cd45539978363a77d6ad76e7a3c22c7105a
--- /dev/null
+++ b/mlir/tools/mlir-rewrite/CMakeLists.txt
@@ -0,0 +1,35 @@
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+set(LIBS
+  ${dialect_libs}
+  ${test_libs}
+
+  MLIRAffineAnalysis
+  MLIRAnalysis
+  MLIRCastInterfaces
+  MLIRDialect
+  MLIRParser
+  MLIRPass
+  MLIRTransforms
+  MLIRTransformUtils
+  MLIRSupport
+  MLIRIR
+  )
+
+include_directories(../../../clang/include)
+
+add_mlir_tool(mlir-rewrite
+  mlir-rewrite.cpp
+
+  DEPENDS
+  ${LIBS}
+  SUPPORT_PLUGINS
+  )
+target_link_libraries(mlir-rewrite PRIVATE ${LIBS})
+llvm_update_compile_flags(mlir-rewrite)
+
+mlir_check_all_link_libraries(mlir-rewrite)
+export_executable_symbols_for_plugins(mlir-rewrite)
diff --git a/mlir/tools/mlir-rewrite/mlir-rewrite.cpp b/mlir/tools/mlir-rewrite/mlir-rewrite.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..308e6490726c86c3a8bdba9a2345dc43440b4c10
--- /dev/null
+++ b/mlir/tools/mlir-rewrite/mlir-rewrite.cpp
@@ -0,0 +1,392 @@
+//===- mlir-rewrite.cpp - MLIR Rewrite Driver -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Main entry function for mlir-rewrite.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/AsmParser/AsmParser.h"
+#include "mlir/AsmParser/AsmParserState.h"
+#include "mlir/IR/AsmState.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/InitAllDialects.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Support/FileUtilities.h"
+#include "mlir/Tools/ParseUtilities.h"
+#include "llvm/ADT/RewriteBuffer.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+using namespace mlir;
+
+namespace mlir {
+using OperationDefinition = AsmParserState::OperationDefinition;
+
+/// Return the source code associated with the OperationDefinition.
+SMRange getOpRange(const OperationDefinition &op) {
+  const char *startOp = op.scopeLoc.Start.getPointer();
+  const char *endOp = op.scopeLoc.End.getPointer();
+
+  for (auto res : op.resultGroups) {
+    SMRange range = res.definition.loc;
+    startOp = std::min(startOp, range.Start.getPointer());
+  }
+  return {SMLoc::getFromPointer(startOp), SMLoc::getFromPointer(endOp)};
+}
+
+/// Helper to simplify rewriting the source file.
+class RewritePad {
+public:
+  static std::unique_ptr<RewritePad> init(StringRef inputFilename,
+                                          StringRef outputFilename);
+
+  /// Return the context the file was parsed into.
+  MLIRContext *getContext() { return &context; }
+
+  /// Return the OperationDefinition's of the operation's parsed.
+  iterator_range<AsmParserState::OperationDefIterator> getOpDefs() {
+    return asmState.getOpDefs();
+  }
+
+  /// Insert the specified string at the specified location in the original
+  /// buffer.
+  void insertText(SMLoc pos, StringRef str, bool insertAfter = true) {
+    rewriteBuffer.InsertText(pos.getPointer() - start, str, insertAfter);
+  }
+
+  /// Replace the range of the source text with the corresponding string in the
+  /// output.
+  void replaceRange(SMRange range, StringRef str) {
+    rewriteBuffer.ReplaceText(range.Start.getPointer() - start,
+                              range.End.getPointer() - range.Start.getPointer(),
+                              str);
+  }
+
+  /// Replace the range of the operation in the source text with the
+  /// corresponding string in the output.
+  void replaceDef(const OperationDefinition &opDef, StringRef newDef) {
+    replaceRange(getOpRange(opDef), newDef);
+  }
+
+  /// Return the source string corresponding to the source range.
+  StringRef getSourceString(SMRange range) {
+    return StringRef(range.Start.getPointer(),
+                     range.End.getPointer() - range.Start.getPointer());
+  }
+
+  /// Return the source string corresponding to operation definition.
+  StringRef getSourceString(const OperationDefinition &opDef) {
+    auto range = getOpRange(opDef);
+    return getSourceString(range);
+  }
+
+  /// Write to stream the result of applying all changes to the
+  /// original buffer.
+  /// Note that it isn't safe to use this function to overwrite memory mapped
+  /// files in-place (PR17960).
+  ///
+  /// The original buffer is not actually changed.
+  raw_ostream &write(raw_ostream &stream) const {
+    return rewriteBuffer.write(stream);
+  }
+
+  /// Return lines that are purely comments.
+  SmallVector<SMRange> getSingleLineComments() {
+    unsigned curBuf = sourceMgr.getMainFileID();
+    const llvm::MemoryBuffer *curMB = sourceMgr.getMemoryBuffer(curBuf);
+    llvm::line_iterator lineIterator(*curMB);
+    SmallVector<SMRange> ret;
+    for (; !lineIterator.is_at_end(); ++lineIterator) {
+      StringRef trimmed = lineIterator->ltrim();
+      if (trimmed.starts_with("//")) {
+        ret.emplace_back(
+            SMLoc::getFromPointer(trimmed.data()),
+            SMLoc::getFromPointer(trimmed.data() + trimmed.size()));
+      }
+    }
+    return ret;
+  }
+
+  /// Return the IR from parsed file.
+  Block *getParsed() { return &parsedIR; }
+
+  /// Return the definition for the given operation, or nullptr if the given
+  /// operation does not have a definition.
+  const OperationDefinition &getOpDef(Operation *op) const {
+    return *asmState.getOpDef(op);
+  }
+
+private:
+  // The context and state required to parse.
+  MLIRContext context;
+  llvm::SourceMgr sourceMgr;
+  DialectRegistry registry;
+  FallbackAsmResourceMap fallbackResourceMap;
+
+  // Storage of textual parsing results.
+  AsmParserState asmState;
+
+  // Parsed IR.
+  Block parsedIR;
+
+  // The RewriteBuffer  is doing most of the real work.
+  llvm::RewriteBuffer rewriteBuffer;
+
+  // Start of the original input, used to compute offset.
+  const char *start;
+};
+
+std::unique_ptr<RewritePad> RewritePad::init(StringRef inputFilename,
+                                             StringRef outputFilename) {
+  std::unique_ptr<RewritePad> r = std::make_unique<RewritePad>();
+
+  // Register all the dialects needed.
+  registerAllDialects(r->registry);
+
+  // Set up the input file.
+  std::string errorMessage;
+  std::unique_ptr<llvm::MemoryBuffer> file =
+      openInputFile(inputFilename, &errorMessage);
+  if (!file) {
+    llvm::errs() << errorMessage << "\n";
+    return nullptr;
+  }
+  r->sourceMgr.AddNewSourceBuffer(std::move(file), SMLoc());
+
+  // Set up the MLIR context and error handling.
+  r->context.appendDialectRegistry(r->registry);
+
+  // Record the start of the buffer to compute offsets with.
+  unsigned curBuf = r->sourceMgr.getMainFileID();
+  const llvm::MemoryBuffer *curMB = r->sourceMgr.getMemoryBuffer(curBuf);
+  r->start = curMB->getBufferStart();
+  r->rewriteBuffer.Initialize(curMB->getBuffer());
+
+  // Parse and populate the AsmParserState.
+  ParserConfig parseConfig(&r->context, /*verifyAfterParse=*/true,
+                           &r->fallbackResourceMap);
+  // Always allow unregistered.
+  r->context.allowUnregisteredDialects(true);
+  if (failed(parseAsmSourceFile(r->sourceMgr, &r->parsedIR, parseConfig,
+                                &r->asmState)))
+    return nullptr;
+
+  return r;
+}
+
+/// Return the source code associated with the operation name.
+SMRange getOpNameRange(const OperationDefinition &op) { return op.loc; }
+
+/// Return whether the operation was printed using generic syntax in original
+/// buffer.
+bool isGeneric(const OperationDefinition &op) {
+  return op.loc.Start.getPointer()[0] == '"';
+}
+
+inline int asMainReturnCode(LogicalResult r) {
+  return r.succeeded() ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+/// Reriter function to invoke.
+using RewriterFunction = std::function<mlir::LogicalResult(
+    mlir::RewritePad &rewriteState, llvm::raw_ostream &os)>;
+
+/// Structure to group information about a rewriter (argument to invoke via
+/// mlir-tblgen, description, and rewriter function).
+class RewriterInfo {
+public:
+  /// RewriterInfo constructor should not be invoked directly, instead use
+  /// RewriterRegistration or registerRewriter.
+  RewriterInfo(StringRef arg, StringRef description, RewriterFunction rewriter)
+      : arg(arg), description(description), rewriter(std::move(rewriter)) {}
+
+  /// Invokes the rewriter and returns whether the rewriter failed.
+  LogicalResult invoke(mlir::RewritePad &rewriteState, raw_ostream &os) const {
+    assert(rewriter && "Cannot call rewriter with null rewriter");
+    return rewriter(rewriteState, os);
+  }
+
+  /// Returns the command line option that may be passed to 'mlir-rewrite' to
+  /// invoke this rewriter.
+  StringRef getRewriterArgument() const { return arg; }
+
+  /// Returns a description for the rewriter.
+  StringRef getRewriterDescription() const { return description; }
+
+private:
+  // The argument with which to invoke the rewriter via mlir-tblgen.
+  StringRef arg;
+
+  // Description of the rewriter.
+  StringRef description;
+
+  // Rewritererator function.
+  RewriterFunction rewriter;
+};
+
+static llvm::ManagedStatic<std::vector<RewriterInfo>> rewriterRegistry;
+
+/// Adds command line option for each registered rewriter.
+struct RewriterNameParser : public llvm::cl::parser<const RewriterInfo *> {
+  RewriterNameParser(llvm::cl::Option &opt);
+
+  void printOptionInfo(const llvm::cl::Option &o,
+                       size_t globalWidth) const override;
+};
+
+/// RewriterRegistration provides a global initializer that registers a rewriter
+/// function.
+struct RewriterRegistration {
+  RewriterRegistration(StringRef arg, StringRef description,
+                       const RewriterFunction &function);
+};
+
+RewriterRegistration::RewriterRegistration(StringRef arg, StringRef description,
+                                           const RewriterFunction &function) {
+  rewriterRegistry->emplace_back(arg, description, function);
+}
+
+RewriterNameParser::RewriterNameParser(llvm::cl::Option &opt)
+    : llvm::cl::parser<const RewriterInfo *>(opt) {
+  for (const auto &kv : *rewriterRegistry) {
+    addLiteralOption(kv.getRewriterArgument(), &kv,
+                     kv.getRewriterDescription());
+  }
+}
+
+void RewriterNameParser::printOptionInfo(const llvm::cl::Option &o,
+                                         size_t globalWidth) const {
+  RewriterNameParser *tp = const_cast<RewriterNameParser *>(this);
+  llvm::array_pod_sort(tp->Values.begin(), tp->Values.end(),
+                       [](const RewriterNameParser::OptionInfo *vT1,
+                          const RewriterNameParser::OptionInfo *vT2) {
+                         return vT1->Name.compare(vT2->Name);
+                       });
+  using llvm::cl::parser;
+  parser<const RewriterInfo *>::printOptionInfo(o, globalWidth);
+}
+
+} // namespace mlir
+
+// TODO: Make these injectable too in non-global way.
+static llvm::cl::OptionCategory clSimpleRenameCategory{"simple-rename options"};
+static llvm::cl::opt<std::string> simpleRenameOpName{
+    "simple-rename-op-name", llvm::cl::desc("Name of op to match on"),
+    llvm::cl::cat(clSimpleRenameCategory)};
+static llvm::cl::opt<std::string> simpleRenameMatch{
+    "simple-rename-match", llvm::cl::desc("Match string for rename"),
+    llvm::cl::cat(clSimpleRenameCategory)};
+static llvm::cl::opt<std::string> simpleRenameReplace{
+    "simple-rename-replace", llvm::cl::desc("Replace string for rename"),
+    llvm::cl::cat(clSimpleRenameCategory)};
+
+// Rewriter that does simple renames.
+LogicalResult simpleRename(RewritePad &rewriteState, raw_ostream &os) {
+  StringRef opName = simpleRenameOpName;
+  StringRef match = simpleRenameMatch;
+  StringRef replace = simpleRenameReplace;
+  llvm::Regex regex(match);
+
+  rewriteState.getParsed()->walk([&](Operation *op) {
+    if (op->getName().getStringRef() != opName)
+      return;
+
+    const OperationDefinition &opDef = rewriteState.getOpDef(op);
+    SMRange range = getOpRange(opDef);
+    // This is a little bit overkill for simple.
+    std::string str = regex.sub(replace, rewriteState.getSourceString(range));
+    rewriteState.replaceRange(range, str);
+  });
+  return success();
+}
+
+static mlir::RewriterRegistration rewriteSimpleRename("simple-rename",
+                                                      "Perform a simple rename",
+                                                      simpleRename);
+
+// Rewriter that insert range markers.
+LogicalResult markRanges(RewritePad &rewriteState, raw_ostream &os) {
+  for (auto it : rewriteState.getOpDefs()) {
+    auto [startOp, endOp] = getOpRange(it);
+
+    rewriteState.insertText(startOp, "《");
+    rewriteState.insertText(endOp, "》");
+
+    auto nameRange = getOpNameRange(it);
+
+    if (isGeneric(it)) {
+      rewriteState.insertText(nameRange.Start, "〖");
+      rewriteState.insertText(nameRange.End, "〗");
+    } else {
+      rewriteState.insertText(nameRange.Start, "〔");
+      rewriteState.insertText(nameRange.End, "〕");
+    }
+  }
+
+  // Highlight all comment lines.
+  // TODO: Could be replaced if this is kept in memory.
+  for (auto commentLine : rewriteState.getSingleLineComments()) {
+    rewriteState.insertText(commentLine.Start, "❰");
+    rewriteState.insertText(commentLine.End, "❱");
+  }
+
+  return success();
+}
+
+static mlir::RewriterRegistration
+    rewriteMarkRanges("mark-ranges", "Indicate ranges parsed", markRanges);
+
+int main(int argc, char **argv) {
+  static llvm::cl::opt<std::string> inputFilename(
+      llvm::cl::Positional, llvm::cl::desc("<input file>"),
+      llvm::cl::init("-"));
+
+  static llvm::cl::opt<std::string> outputFilename(
+      "o", llvm::cl::desc("Output filename"), llvm::cl::value_desc("filename"),
+      llvm::cl::init("-"));
+
+  llvm::cl::opt<const mlir::RewriterInfo *, false, mlir::RewriterNameParser>
+      rewriter("", llvm::cl::desc("Rewriter to run"));
+
+  std::string helpHeader = "mlir-rewrite";
+
+  llvm::cl::ParseCommandLineOptions(argc, argv, helpHeader);
+
+  // If no rewriter has been selected, exit with error code. Could also just
+  // return but its unlikely this was intentionally being used as `cp`.
+  if (!rewriter) {
+    llvm::errs() << "No rewriter selected!\n";
+    return mlir::asMainReturnCode(mlir::failure());
+  }
+
+  // Set up rewrite buffer.
+  auto rewriterOr = RewritePad::init(inputFilename, outputFilename);
+  if (!rewriterOr)
+    return mlir::asMainReturnCode(mlir::failure());
+
+  // Set up the output file.
+  std::string errorMessage;
+  auto output = openOutputFile(outputFilename, &errorMessage);
+  if (!output) {
+    llvm::errs() << errorMessage << "\n";
+    return mlir::asMainReturnCode(mlir::failure());
+  }
+
+  LogicalResult result = rewriter->invoke(*rewriterOr, output->os());
+  if (succeeded(result)) {
+    rewriterOr->write(output->os());
+    output->keep();
+  }
+  return mlir::asMainReturnCode(result);
+}
diff --git a/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp b/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
index 86ebaf2cf27dfeb241914664447a49a284e273b2..6a3d5a25e28cd9af543212e1e1ea35f476783a87 100644
--- a/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
+++ b/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
@@ -46,8 +46,9 @@ public:
 private:
   /// Emits parse calls to construct given kind.
   void emitParseHelper(StringRef kind, StringRef returnType, StringRef builder,
-                       ArrayRef<Init *> args, ArrayRef<std::string> argNames,
-                       StringRef failure, mlir::raw_indented_ostream &ios);
+                       ArrayRef<const Init *> args,
+                       ArrayRef<std::string> argNames, StringRef failure,
+                       mlir::raw_indented_ostream &ios);
 
   /// Emits print instructions.
   void emitPrintHelper(const Record *memberRec, StringRef kind,
@@ -135,10 +136,12 @@ void Generator::emitParse(StringRef kind, const Record &x) {
       R"(static {0} read{1}(MLIRContext* context, DialectBytecodeReader &reader) )";
   mlir::raw_indented_ostream os(output);
   std::string returnType = getCType(&x);
-  os << formatv(head, kind == "attribute" ? "::mlir::Attribute" : "::mlir::Type", x.getName());
-  DagInit *members = x.getValueAsDag("members");
-  SmallVector<std::string> argNames =
-      llvm::to_vector(map_range(members->getArgNames(), [](StringInit *init) {
+  os << formatv(head,
+                kind == "attribute" ? "::mlir::Attribute" : "::mlir::Type",
+                x.getName());
+  const DagInit *members = x.getValueAsDag("members");
+  SmallVector<std::string> argNames = llvm::to_vector(
+      map_range(members->getArgNames(), [](const StringInit *init) {
         return init->getAsUnquotedString();
       }));
   StringRef builder = x.getValueAsString("cBuilder").trim();
@@ -148,7 +151,7 @@ void Generator::emitParse(StringRef kind, const Record &x) {
 }
 
 void printParseConditional(mlir::raw_indented_ostream &ios,
-                           ArrayRef<Init *> args,
+                           ArrayRef<const Init *> args,
                            ArrayRef<std::string> argNames) {
   ios << "if ";
   auto parenScope = ios.scope("(", ") {");
@@ -159,7 +162,7 @@ void printParseConditional(mlir::raw_indented_ostream &ios,
   };
 
   auto parsedArgs =
-      llvm::to_vector(make_filter_range(args, [](Init *const attr) {
+      llvm::to_vector(make_filter_range(args, [](const Init *const attr) {
         const Record *def = cast<DefInit>(attr)->getDef();
         if (def->isSubClassOf("Array"))
           return true;
@@ -168,7 +171,7 @@ void printParseConditional(mlir::raw_indented_ostream &ios,
 
   interleave(
       zip(parsedArgs, argNames),
-      [&](std::tuple<llvm::Init *&, const std::string &> it) {
+      [&](std::tuple<const Init *&, const std::string &> it) {
         const Record *attr = cast<DefInit>(std::get<0>(it))->getDef();
         std::string parser;
         if (auto optParser = attr->getValueAsOptionalString("cParser")) {
@@ -196,7 +199,7 @@ void printParseConditional(mlir::raw_indented_ostream &ios,
 }
 
 void Generator::emitParseHelper(StringRef kind, StringRef returnType,
-                                StringRef builder, ArrayRef<Init *> args,
+                                StringRef builder, ArrayRef<const Init *> args,
                                 ArrayRef<std::string> argNames,
                                 StringRef failure,
                                 mlir::raw_indented_ostream &ios) {
@@ -210,7 +213,7 @@ void Generator::emitParseHelper(StringRef kind, StringRef returnType,
   // Print decls.
   std::string lastCType = "";
   for (auto [arg, name] : zip(args, argNames)) {
-    DefInit *first = dyn_cast<DefInit>(arg);
+    const DefInit *first = dyn_cast<DefInit>(arg);
     if (!first)
       PrintFatalError("Unexpected type for " + name);
     const Record *def = first->getDef();
@@ -251,13 +254,14 @@ void Generator::emitParseHelper(StringRef kind, StringRef returnType,
     std::string returnType = getCType(def);
     ios << "auto " << listHelperName(name) << " = [&]() -> FailureOr<"
         << returnType << "> ";
-    SmallVector<Init *> args;
+    SmallVector<const Init *> args;
     SmallVector<std::string> argNames;
     if (def->isSubClassOf("CompositeBytecode")) {
-      DagInit *members = def->getValueAsDag("members");
-      args = llvm::to_vector(members->getArgs());
+      const DagInit *members = def->getValueAsDag("members");
+      args = llvm::to_vector(map_range(
+          members->getArgs(), [](Init *init) { return (const Init *)init; }));
       argNames = llvm::to_vector(
-          map_range(members->getArgNames(), [](StringInit *init) {
+          map_range(members->getArgNames(), [](const StringInit *init) {
             return init->getAsUnquotedString();
           }));
     } else {
@@ -332,7 +336,7 @@ void Generator::emitPrint(StringRef kind, StringRef type,
     auto *members = rec->getValueAsDag("members");
     for (auto [arg, name] :
          llvm::zip(members->getArgs(), members->getArgNames())) {
-      DefInit *def = dyn_cast<DefInit>(arg);
+      const DefInit *def = dyn_cast<DefInit>(arg);
       assert(def);
       const Record *memberRec = def->getDef();
       emitPrintHelper(memberRec, kind, kind, name->getAsUnquotedString(), os);
@@ -385,7 +389,7 @@ void Generator::emitPrintHelper(const Record *memberRec, StringRef kind,
     auto *members = memberRec->getValueAsDag("members");
     for (auto [arg, argName] :
          zip(members->getArgs(), members->getArgNames())) {
-      DefInit *def = dyn_cast<DefInit>(arg);
+      const DefInit *def = dyn_cast<DefInit>(arg);
       assert(def);
       emitPrintHelper(def->getDef(), kind, parent,
                       argName->getAsUnquotedString(), ios);
diff --git a/mlir/tools/mlir-tblgen/DialectGen.cpp b/mlir/tools/mlir-tblgen/DialectGen.cpp
index 55c3d9da2590056556d61eea5080583a9ad387a3..414cad5e1dcc2e798472004ccec67c37852cbba7 100644
--- a/mlir/tools/mlir-tblgen/DialectGen.cpp
+++ b/mlir/tools/mlir-tblgen/DialectGen.cpp
@@ -46,10 +46,10 @@ using DialectFilterIterator =
 } // namespace
 
 static void populateDiscardableAttributes(
-    Dialect &dialect, llvm::DagInit *discardableAttrDag,
+    Dialect &dialect, const llvm::DagInit *discardableAttrDag,
     SmallVector<std::pair<std::string, std::string>> &discardableAttributes) {
   for (int i : llvm::seq<int>(0, discardableAttrDag->getNumArgs())) {
-    llvm::Init *arg = discardableAttrDag->getArg(i);
+    const llvm::Init *arg = discardableAttrDag->getArg(i);
 
     StringRef givenName = discardableAttrDag->getArgNameStr(i);
     if (givenName.empty())
@@ -271,7 +271,8 @@ static void emitDialectDecl(Dialect &dialect, raw_ostream &os) {
     if (dialect.hasOperationInterfaceFallback())
       os << operationInterfaceFallbackDecl;
 
-    llvm::DagInit *discardableAttrDag = dialect.getDiscardableAttributes();
+    const llvm::DagInit *discardableAttrDag =
+        dialect.getDiscardableAttributes();
     SmallVector<std::pair<std::string, std::string>> discardableAttributes;
     populateDiscardableAttributes(dialect, discardableAttrDag,
                                   discardableAttributes);
@@ -370,7 +371,7 @@ static void emitDialectDef(Dialect &dialect, const RecordKeeper &records,
   StringRef superClassName =
       dialect.isExtensible() ? "ExtensibleDialect" : "Dialect";
 
-  llvm::DagInit *discardableAttrDag = dialect.getDiscardableAttributes();
+  const llvm::DagInit *discardableAttrDag = dialect.getDiscardableAttributes();
   SmallVector<std::pair<std::string, std::string>> discardableAttributes;
   populateDiscardableAttributes(dialect, discardableAttrDag,
                                 discardableAttributes);
diff --git a/mlir/tools/mlir-tblgen/OmpOpGen.cpp b/mlir/tools/mlir-tblgen/OmpOpGen.cpp
index 1c20a6a9bcf4e89190461d532876a1eb9ecfb7e5..8716667723a3730dc78b68c21467689a4b4a3276 100644
--- a/mlir/tools/mlir-tblgen/OmpOpGen.cpp
+++ b/mlir/tools/mlir-tblgen/OmpOpGen.cpp
@@ -102,11 +102,13 @@ static StringRef extractOmpClauseName(const Record *clause) {
 
 /// Check that the given argument, identified by its name and initialization
 /// value, is present in the \c arguments `dag`.
-static bool verifyArgument(DagInit *arguments, StringRef argName,
-                           Init *argInit) {
+static bool verifyArgument(const DagInit *arguments, StringRef argName,
+                           const Init *argInit) {
   auto range = zip_equal(arguments->getArgNames(), arguments->getArgs());
   return llvm::any_of(
-      range, [&](std::tuple<llvm::StringInit *const &, llvm::Init *const &> v) {
+      range,
+      [&](std::tuple<const llvm::StringInit *const &, const llvm::Init *const &>
+              v) {
         return std::get<0>(v)->getAsUnquotedString() == argName &&
                std::get<1>(v) == argInit;
       });
@@ -141,8 +143,8 @@ static void verifyClause(const Record *op, const Record *clause) {
   StringRef clauseClassName = extractOmpClauseName(clause);
 
   if (!clause->getValueAsBit("ignoreArgs")) {
-    DagInit *opArguments = op->getValueAsDag("arguments");
-    DagInit *arguments = clause->getValueAsDag("arguments");
+    const DagInit *opArguments = op->getValueAsDag("arguments");
+    const DagInit *arguments = clause->getValueAsDag("arguments");
 
     for (auto [name, arg] :
          zip(arguments->getArgNames(), arguments->getArgs())) {
@@ -208,8 +210,9 @@ static void verifyClause(const Record *op, const Record *clause) {
 ///
 /// \return the name of the base type to represent elements of the argument
 ///         type.
-static StringRef translateArgumentType(ArrayRef<SMLoc> loc, StringInit *name,
-                                       Init *init, int &nest, int &rank) {
+static StringRef translateArgumentType(ArrayRef<SMLoc> loc,
+                                       const StringInit *name, const Init *init,
+                                       int &nest, int &rank) {
   const Record *def = cast<DefInit>(init)->getDef();
 
   llvm::StringSet<> superClasses;
@@ -282,7 +285,7 @@ static void genClauseOpsStruct(const Record *clause, raw_ostream &os) {
   StringRef clauseName = extractOmpClauseName(clause);
   os << "struct " << clauseName << "ClauseOps {\n";
 
-  DagInit *arguments = clause->getValueAsDag("arguments");
+  const DagInit *arguments = clause->getValueAsDag("arguments");
   for (auto [name, arg] :
        zip_equal(arguments->getArgNames(), arguments->getArgs())) {
     int nest = 0, rank = 1;
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 67df002ce3818130bdbdac6b05eea3dc4e77d564..ce2b6ed94c39490acdd426dcb6889c42ebf4198e 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -4282,6 +4282,19 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
     }
   }
 
+  // Create a constructor that creates a new generic adaptor by copying
+  // everything from another adaptor, except for the values.
+  {
+    SmallVector<MethodParameter> paramList;
+    paramList.emplace_back("RangeT", "values");
+    paramList.emplace_back("const " + op.getGenericAdaptorName() + "Base &",
+                           "base");
+    auto *constructor =
+        genericAdaptor.addConstructor<Method::Inline>(paramList);
+    constructor->addMemberInitializer("Base", "base");
+    constructor->addMemberInitializer("odsOperands", "values");
+  }
+
   // Create constructors constructing the adaptor from an instance of the op.
   // This takes the attributes, properties and regions from the op instance
   // and the value range from the parameter.
diff --git a/mlir/unittests/Dialect/SPIRV/SerializationTest.cpp b/mlir/unittests/Dialect/SPIRV/SerializationTest.cpp
index 9d2f690ed898afca1078d8c14be9a1ad79fa8f12..ef89c1645d373fe710ad7f5216e65c05017c077b 100644
--- a/mlir/unittests/Dialect/SPIRV/SerializationTest.cpp
+++ b/mlir/unittests/Dialect/SPIRV/SerializationTest.cpp
@@ -176,7 +176,7 @@ TEST_F(SerializationTest, SignlessVsSignedIntegerConstantBitExtension) {
       IntegerType::get(&context, 16, IntegerType::Signless);
   auto signedInt16Type = IntegerType::get(&context, 16, IntegerType::Signed);
   // Check the bit extension of same value under different signedness semantics.
-  APInt signlessIntConstVal(signlessInt16Type.getWidth(), -1,
+  APInt signlessIntConstVal(signlessInt16Type.getWidth(), 0xffff,
                             signlessInt16Type.getSignedness());
   APInt signedIntConstVal(signedInt16Type.getWidth(), -1,
                           signedInt16Type.getSignedness());
diff --git a/mlir/unittests/IR/AttributeTest.cpp b/mlir/unittests/IR/AttributeTest.cpp
index 981e919289c30604170c8f61b11caf84644e06a6..2b2ec056739431c8acfbab83b154f1f514e34a1f 100644
--- a/mlir/unittests/IR/AttributeTest.cpp
+++ b/mlir/unittests/IR/AttributeTest.cpp
@@ -351,6 +351,21 @@ TEST(DenseResourceElementsAttrTest, CheckNoCast) {
   EXPECT_FALSE(isa<DenseBoolResourceElementsAttr>(i32ResourceAttr));
 }
 
+TEST(DenseResourceElementsAttrTest, CheckNotMutableAllocateAndCopy) {
+  MLIRContext context;
+  Builder builder(&context);
+
+  // Create a i32 attribute.
+  std::vector<int32_t> data = {10, 20, 30};
+  auto type = RankedTensorType::get(data.size(), builder.getI32Type());
+  Attribute i32ResourceAttr = DenseI32ResourceElementsAttr::get(
+      type, "resource",
+      HeapAsmResourceBlob::allocateAndCopyInferAlign<int32_t>(
+          data, /*is_mutable=*/false));
+
+  EXPECT_TRUE(isa<DenseI32ResourceElementsAttr>(i32ResourceAttr));
+}
+
 TEST(DenseResourceElementsAttrTest, CheckInvalidData) {
   MLIRContext context;
   Builder builder(&context);
diff --git a/mlir/utils/spirv/gen_spirv_dialect.py b/mlir/utils/spirv/gen_spirv_dialect.py
index 6d82c0121581966465e2edf580b174d4055a83f8..2fb540ef103250be20f7bb81e9db1a49411dcc97 100755
--- a/mlir/utils/spirv/gen_spirv_dialect.py
+++ b/mlir/utils/spirv/gen_spirv_dialect.py
@@ -979,7 +979,8 @@ def extract_td_op_info(op_def):
     # Get instruction category
     prefix = "SPIRV_"
     inst_category = [
-        o[len(prefix) :] for o in re.findall(prefix + r"\w+Op", op_def.split(":", 1)[1])
+        o[len(prefix) :]
+        for o in re.findall(prefix + r"\w+Op\b", op_def.split(":", 1)[1])
     ]
     assert len(inst_category) <= 1, "more than one ops in the same section!"
     inst_category = inst_category[0] if len(inst_category) == 1 else "Op"
diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index b054313d2c1e123a2e2fdcd90c5cd1435e09ebfe..3e3c57c6adb2e68a3e98744c58e1d04bac664455 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -1589,15 +1589,13 @@ kmp_str_buf_t *__kmp_affinity_str_buf_mask(kmp_str_buf_t *buf,
   return buf;
 }
 
-// Return (possibly empty) affinity mask representing the offline CPUs
-// Caller must free the mask
-kmp_affin_mask_t *__kmp_affinity_get_offline_cpus() {
-  kmp_affin_mask_t *offline;
-  KMP_CPU_ALLOC(offline);
-  KMP_CPU_ZERO(offline);
+static kmp_affin_mask_t *__kmp_parse_cpu_list(const char *path) {
+  kmp_affin_mask_t *mask;
+  KMP_CPU_ALLOC(mask);
+  KMP_CPU_ZERO(mask);
 #if KMP_OS_LINUX
   int n, begin_cpu, end_cpu;
-  kmp_safe_raii_file_t offline_file;
+  kmp_safe_raii_file_t file;
   auto skip_ws = [](FILE *f) {
     int c;
     do {
@@ -1606,29 +1604,29 @@ kmp_affin_mask_t *__kmp_affinity_get_offline_cpus() {
     if (c != EOF)
       ungetc(c, f);
   };
-  // File contains CSV of integer ranges representing the offline CPUs
+  // File contains CSV of integer ranges representing the CPUs
   // e.g., 1,2,4-7,9,11-15
-  int status = offline_file.try_open("/sys/devices/system/cpu/offline", "r");
+  int status = file.try_open(path, "r");
   if (status != 0)
-    return offline;
-  while (!feof(offline_file)) {
-    skip_ws(offline_file);
-    n = fscanf(offline_file, "%d", &begin_cpu);
+    return mask;
+  while (!feof(file)) {
+    skip_ws(file);
+    n = fscanf(file, "%d", &begin_cpu);
     if (n != 1)
       break;
-    skip_ws(offline_file);
-    int c = fgetc(offline_file);
+    skip_ws(file);
+    int c = fgetc(file);
     if (c == EOF || c == ',') {
       // Just single CPU
       end_cpu = begin_cpu;
     } else if (c == '-') {
       // Range of CPUs
-      skip_ws(offline_file);
-      n = fscanf(offline_file, "%d", &end_cpu);
+      skip_ws(file);
+      n = fscanf(file, "%d", &end_cpu);
       if (n != 1)
         break;
-      skip_ws(offline_file);
-      c = fgetc(offline_file); // skip ','
+      skip_ws(file);
+      c = fgetc(file); // skip ','
     } else {
       // Syntax problem
       break;
@@ -1638,13 +1636,19 @@ kmp_affin_mask_t *__kmp_affinity_get_offline_cpus() {
         end_cpu >= __kmp_xproc || begin_cpu > end_cpu) {
       continue;
     }
-    // Insert [begin_cpu, end_cpu] into offline mask
+    // Insert [begin_cpu, end_cpu] into mask
     for (int cpu = begin_cpu; cpu <= end_cpu; ++cpu) {
-      KMP_CPU_SET(cpu, offline);
+      KMP_CPU_SET(cpu, mask);
     }
   }
 #endif
-  return offline;
+  return mask;
+}
+
+// Return (possibly empty) affinity mask representing the offline CPUs
+// Caller must free the mask
+kmp_affin_mask_t *__kmp_affinity_get_offline_cpus() {
+  return __kmp_parse_cpu_list("/sys/devices/system/cpu/offline");
 }
 
 // Return the number of available procs
@@ -3175,6 +3179,37 @@ static inline const char *__kmp_cpuinfo_get_envvar() {
   return envvar;
 }
 
+static bool __kmp_package_id_from_core_siblings_list(unsigned **threadInfo,
+                                                     unsigned num_avail,
+                                                     unsigned idx) {
+  if (!KMP_AFFINITY_CAPABLE())
+    return false;
+
+  char path[256];
+  KMP_SNPRINTF(path, sizeof(path),
+               "/sys/devices/system/cpu/cpu%u/topology/core_siblings_list",
+               threadInfo[idx][osIdIndex]);
+  kmp_affin_mask_t *siblings = __kmp_parse_cpu_list(path);
+  for (unsigned i = 0; i < num_avail; ++i) {
+    unsigned cpu_id = threadInfo[i][osIdIndex];
+    KMP_ASSERT(cpu_id < __kmp_affin_mask_size * CHAR_BIT);
+    if (!KMP_CPU_ISSET(cpu_id, siblings))
+      continue;
+    if (threadInfo[i][pkgIdIndex] == UINT_MAX) {
+      // Arbitrarily pick the first index we encounter, it only matters that
+      // the value is the same for all siblings.
+      threadInfo[i][pkgIdIndex] = idx;
+    } else if (threadInfo[i][pkgIdIndex] != idx) {
+      // Contradictory sibling lists.
+      KMP_CPU_FREE(siblings);
+      return false;
+    }
+  }
+  KMP_ASSERT(threadInfo[idx][pkgIdIndex] != UINT_MAX);
+  KMP_CPU_FREE(siblings);
+  return true;
+}
+
 // Parse /proc/cpuinfo (or an alternate file in the same format) to obtain the
 // affinity map. On AIX, the map is obtained through system SRAD (Scheduler
 // Resource Allocation Domain).
@@ -3550,18 +3585,13 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line,
         return false;
       }
 
-      // Check for missing fields.  The osId field must be there, and we
-      // currently require that the physical id field is specified, also.
+      // Check for missing fields.  The osId field must be there. The physical
+      // id field will be checked later.
       if (threadInfo[num_avail][osIdIndex] == UINT_MAX) {
         CLEANUP_THREAD_INFO;
         *msg_id = kmp_i18n_str_MissingProcField;
         return false;
       }
-      if (threadInfo[0][pkgIdIndex] == UINT_MAX) {
-        CLEANUP_THREAD_INFO;
-        *msg_id = kmp_i18n_str_MissingPhysicalIDField;
-        return false;
-      }
 
       // Skip this proc if it is not included in the machine model.
       if (KMP_AFFINITY_CAPABLE() &&
@@ -3591,6 +3621,18 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line,
   }
   *line = 0;
 
+  // At least on powerpc, Linux may return -1 for physical_package_id. Try
+  // to reconstruct topology from core_siblings_list in that case.
+  for (i = 0; i < num_avail; ++i) {
+    if (threadInfo[i][pkgIdIndex] == UINT_MAX) {
+      if (!__kmp_package_id_from_core_siblings_list(threadInfo, num_avail, i)) {
+        CLEANUP_THREAD_INFO;
+        *msg_id = kmp_i18n_str_MissingPhysicalIDField;
+        return false;
+      }
+    }
+  }
+
 #if KMP_MIC && REDUCE_TEAM_SIZE
   unsigned teamSize = 0;
 #endif // KMP_MIC && REDUCE_TEAM_SIZE
diff --git a/openmp/runtime/test/affinity/kmp-hw-subset.c b/openmp/runtime/test/affinity/kmp-hw-subset.c
index 606fcdfbada95a3a4f596bd120b8581e2d6e81ca..0b49969bd3b10ce97464378ef6286a3f9b3b114c 100644
--- a/openmp/runtime/test/affinity/kmp-hw-subset.c
+++ b/openmp/runtime/test/affinity/kmp-hw-subset.c
@@ -25,7 +25,7 @@ static int compare_hw_subset_places(const place_list_t *openmp_places,
     expected_per_place = nthreads_per_core;
   } else {
     expected_total = nsockets;
-    expected_per_place = ncores_per_socket;
+    expected_per_place = ncores_per_socket * nthreads_per_core;
   }
   if (openmp_places->num_places != expected_total) {
     fprintf(stderr, "error: KMP_HW_SUBSET did not half each resource layer!\n");
diff --git a/polly/lib/CodeGen/IslExprBuilder.cpp b/polly/lib/CodeGen/IslExprBuilder.cpp
index aaafac14bf806562a2ec45466860d96868dc9093..1688c41c624b24f05b456fb70c3a87f21505048e 100644
--- a/polly/lib/CodeGen/IslExprBuilder.cpp
+++ b/polly/lib/CodeGen/IslExprBuilder.cpp
@@ -129,16 +129,16 @@ Value *IslExprBuilder::createBinOp(BinaryOperator::BinaryOps Opc, Value *LHS,
   Module *M = Builder.GetInsertBlock()->getModule();
   switch (Opc) {
   case Instruction::Add:
-    F = Intrinsic::getDeclaration(M, Intrinsic::sadd_with_overflow,
-                                  {LHS->getType()});
+    F = Intrinsic::getOrInsertDeclaration(M, Intrinsic::sadd_with_overflow,
+                                          {LHS->getType()});
     break;
   case Instruction::Sub:
-    F = Intrinsic::getDeclaration(M, Intrinsic::ssub_with_overflow,
-                                  {LHS->getType()});
+    F = Intrinsic::getOrInsertDeclaration(M, Intrinsic::ssub_with_overflow,
+                                          {LHS->getType()});
     break;
   case Instruction::Mul:
-    F = Intrinsic::getDeclaration(M, Intrinsic::smul_with_overflow,
-                                  {LHS->getType()});
+    F = Intrinsic::getOrInsertDeclaration(M, Intrinsic::smul_with_overflow,
+                                          {LHS->getType()});
     break;
   default:
     llvm_unreachable("No overflow intrinsic for binary operator found!");
diff --git a/polly/lib/CodeGen/PerfMonitor.cpp b/polly/lib/CodeGen/PerfMonitor.cpp
index 3cad8537f3ee19c71ecbe7a43b4b485b77bec0dd..1a79161468544390aa2eb9c5838a7d1c4ce568fe 100644
--- a/polly/lib/CodeGen/PerfMonitor.cpp
+++ b/polly/lib/CodeGen/PerfMonitor.cpp
@@ -59,7 +59,7 @@ void PerfMonitor::addToGlobalConstructors(Function *Fn) {
 }
 
 Function *PerfMonitor::getRDTSCP() {
-  return Intrinsic::getDeclaration(M, Intrinsic::x86_rdtscp);
+  return Intrinsic::getOrInsertDeclaration(M, Intrinsic::x86_rdtscp);
 }
 
 PerfMonitor::PerfMonitor(const Scop &S, Module *M)
diff --git a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
index a08ce54653fb1c4ca2a59ec47b02ed7275d59389..44e4bf82f95bd3874e5a23571266b06fe8612cd2 100644
--- a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
@@ -278,6 +278,7 @@ cc_library(
         "//llvm:MC",
         "//llvm:Object",
         "//llvm:Option",
+        "//llvm:ProfileData",
         "//llvm:Support",
         "//llvm:TargetParser",
         "//llvm:config",
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index bbc17624b5472b2e609fcd2bbc20fdbce041c8e3..cea80ce57027df941b76a07b70c439f2b9f3c60d 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -1158,6 +1158,7 @@ cc_library(
     hdrs = glob(["include/llvm/ProfileData/Coverage/*.h"]),
     copts = llvm_copts,
     deps = [
+        ":BinaryFormat",
         ":Object",
         ":ProfileData",
         ":Support",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index e52439f00879fe648b43a965c28147fc255fa05c..779609340d7224cddb5e32f2c8ea542e89674eb4 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -4876,6 +4876,7 @@ cc_library(
         ":NVGPUTransformOps",
         ":NVVMTarget",
         ":NVVMToLLVM",
+        ":OpenMPToLLVM",
         ":ROCDLTarget",
         ":ROCDLToLLVMIRTranslation",
         ":SCFTransformOps",
@@ -9740,6 +9741,28 @@ cc_binary(
     ],
 )
 
+cc_binary(
+    name = "mlir-rewrite",
+    srcs = ["tools/mlir-rewrite/mlir-rewrite.cpp"],
+    deps = [
+        ":AffineAnalysis",
+        ":AllExtensions",
+        ":AllPassesAndDialects",
+        ":Analysis",
+        ":AsmParser",
+        ":CastInterfaces",
+        ":Dialect",
+        ":IR",
+        ":ParseUtilities",
+        ":Parser",
+        ":Pass",
+        ":Support",
+        ":TransformUtils",
+        ":Transforms",
+        "//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "MlirJitRunner",
     srcs = ["lib/ExecutionEngine/JitRunner.cpp"],
@@ -10603,6 +10626,7 @@ cc_library(
         ":AtomicInterfaces",
         ":AtomicInterfacesIncGen",
         ":ControlFlowInterfaces",
+        ":ConvertToLLVMInterface",
         ":FuncDialect",
         ":IR",
         ":LLVMDialect",
@@ -10650,6 +10674,7 @@ cc_library(
         ":ArithToLLVM",
         ":ControlFlowToLLVM",
         ":ConversionPassIncGen",
+        ":ConvertToLLVMInterface",
         ":FuncToLLVM",
         ":LLVMCommonConversion",
         ":LLVMDialect",